diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c065cf..4aaddb6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,18 @@ jobs: enable-cache: true cache-dependency-glob: "uv.lock" - run: uv sync --locked --all-extras --group dev + # Golden leaf roots are per CPU class; record which CPU ran this shard. + - name: Runner CPU + shell: bash + run: | + set -euo pipefail + MODEL=$(lscpu | sed -n 's/^Model name: *//p' | head -n1) + VENDOR=$(lscpu | sed -n 's/^Vendor ID: *//p' | head -n1) + if lscpu | grep -E '^Flags:' | grep -qw avx512f; then AVX512F=yes; else AVX512F=no; fi + TORCH=$(uv run --frozen python -c "import torch;print(torch.__version__, torch.get_num_threads())") + LINE="cpu: ${MODEL} | vendor: ${VENDOR} | avx512f: ${AVX512F} | torch, threads: ${TORCH}" + echo "$LINE" + echo "shard ${{ matrix.shard }} $LINE" >> "$GITHUB_STEP_SUMMARY" - name: Sharded suite env: SHARD: ${{ matrix.shard }} diff --git a/.gitignore b/.gitignore index 6459210..d5e3014 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ __pycache__/ .ruff_cache/ .pytest_cache/ .hypothesis/ +.evidence/ # audit/report/plan artifacts (kept out of the product repo) experiments/results/ diff --git a/.publishignore b/.publishignore index b6283f7..d87eee5 100644 --- a/.publishignore +++ b/.publishignore @@ -14,6 +14,6 @@ id_(ed25519|rsa) \.(pem|key)$ # Local authority, custody and recovery material; public nested data stays publishable. ^\.(omo|herdr-web-ui)/ -(^|/)(private|secrets|roles|private-roles|role-keys|private-keys|authority|authority-state|portable-authority|portable-bundle|portable-bundles|snapshots|preserved-snapshots|evidence|service-export|integration-root|rescue)/ +(^|/)(private|secrets|roles|private-roles|role-keys|private-keys|authority|authority-state|portable-authority|portable-bundle|portable-bundles|snapshots|preserved-snapshots|evidence|\.evidence|service-export|integration-root|rescue)/ \.(seed|priv|p12|pfx|pkcs8)$ (^|/)(private[-_]key|admin\.token|drain\.token|challenge\.db(-wal|-shm)?|bundle\.json)$ diff --git a/README.md b/README.md index 17a31a9..548e908 100644 --- a/README.md +++ b/README.md @@ -184,7 +184,7 @@ CI publishes and attests both images to GHCR after each green build on `main`. T uv run --frozen python scripts/vast_e2e.py --key-file ~/.vast-key --max-usd 6 --max-minutes 90 ``` -The script rents a single 2x RTX 5090 host with the pinned GPU image and syncs the current source. It runs the GPU self-check, the CUDA test selectors and a full Network v2 round trip (service, multi-GPU miner, auditor replay). It then rescues the results and always deletes the instance. Add `--dry-run` to see the selected offer and its worst-case cost. +The script rents a single 2x RTX 5090 host with the pinned GPU image and syncs the current source. It runs the GPU self-check, the CUDA test selectors and a full Network v2 round trip (service, multi-GPU miner, auditor replay). It then rescues the results and always deletes the instance. Add `--dry-run` to see the selected offer and its worst-case cost; it makes read-only provider GET calls (credit and offer search) and creates nothing. ## Components @@ -228,6 +228,7 @@ tests/ one directory per package ## Contributing - `uv sync --frozen --all-extras`, then `bash scripts/check.sh` before opening a pull request. +- `bash scripts/mine-smoke-v2.sh` checks the network-v2 miner CLI chain end to end on CPU, offline (~2 min; see [docs/network-v2.md](docs/network-v2.md#cpu-smoke)). - Verification stays bitwise. Tolerance checks, obfuscation or trusted-hardware assumptions are out of scope for the security model. - When a route or message changes, update [docs/challenge-routes.md](docs/challenge-routes.md) and regenerate schemas with `uv run --frozen python -m hypertrain.protocol.schema docs/schemas`. - Publishing goes through `scripts/publish.sh`, which runs a strict secret scan and requires an out-of-band approval token. diff --git a/deploy/challenge-profile-row.example.toml b/deploy/challenge-profile-row.example.toml new file mode 100644 index 0000000..e467cc6 --- /dev/null +++ b/deploy/challenge-profile-row.example.toml @@ -0,0 +1,24 @@ +# Example challenge profile with hypertrain active (config/challenges.toml in CortexLM/cortex, +# served by the gateway at GET /v1/trust). The live profile is version 1 with bounty = 2000 and +# proof = 8000 bps; here proof drops to 7000 to give hypertrain 1000. The shares must sum to +# exactly 10000. `version` is 3 because a third challenge needs version >= 3; bump it above the +# live value. It is a whole file: replace the live profile with it, never append it (duplicate +# keys). Restart the master between two epochs. +# Leave `introduced_epoch` at or below the current epoch. No key: Cortex signs nothing here. +version = 3 +introduced_epoch = 0 + +[[challenges]] +id = "bounty" +emission_share_bps = 2000 +policy = "all_metagraph_hotkeys" + +[[challenges]] +id = "proof" +emission_share_bps = 7000 +policy = "all_metagraph_hotkeys" + +[[challenges]] +id = "hypertrain" +emission_share_bps = 1000 +policy = "all_metagraph_hotkeys" diff --git a/deploy/registry-row.toml b/deploy/registry-row.toml index ba227f6..2144103 100644 --- a/deploy/registry-row.toml +++ b/deploy/registry-row.toml @@ -1,9 +1,10 @@ # Row to append to deploy/challenges/registry.toml in CortexLM/cortex (unsigned operator -# registry, cortex@d738424 src/cortex/challenges/registry.py). Registered but absent from the -# signed trust root = burn-in without emission. Secrets in +# registry, src/cortex/challenges/registry.py). Registered but absent from the challenge profile +# (config/challenges.toml) = burn-in without emission. Secrets in # $BASE_CHALLENGE_SECRETS_HOST_DIR/hypertrain/ (0700, owner 65532): internal.token, admin.token, -# worker.token, coord.key (K_coord seed, never the Cortex leaf seed). -# HYPERTRAIN_OWNER_HOTKEY is the ss58 of K_owner; replace before enabling. +# worker.token, coord.key (K_coord seed). +# HYPERTRAIN_OWNER_HOTKEY is the ss58 of hypertrain's K_owner (RunManifest signer); replace +# before enabling. version = 1 [[challenge]] diff --git a/deploy/trust-root-v3-row.example.toml b/deploy/trust-root-v3-row.example.toml deleted file mode 100644 index 1b7f323..0000000 --- a/deploy/trust-root-v3-row.example.toml +++ /dev/null @@ -1,12 +0,0 @@ -# UNSIGNED example row for an algorithm-3 trust root (cortex@d738424 -# config/challenges-v3.example.toml, docs/how-to/trust-root.md "Activate container challenges"). -# Add it to the owner's next v>=3 document, rebalance every share so the sum is exactly 10000, -# then sign offline (cortex trust-sign) and verify (cortex trust-verify). public_key is the -# hypertrain LEAF key from `cortex keygen`; its seed lives only at -# deploy/secrets/master/hypertrain.key on the master, never in the container. - -[[challenges]] -id = "hypertrain" -public_key = "REPLACE_WITH_64_HEX_LEAF_PUBLIC_KEY" -emission_share_bps = 0 -policy = "all_metagraph_hotkeys" diff --git a/docs/challenge-routes.md b/docs/challenge-routes.md index 4a15ce1..b996aed 100644 --- a/docs/challenge-routes.md +++ b/docs/challenge-routes.md @@ -33,7 +33,7 @@ Errors have the shape `{"detail": str}`. | `CHALLENGE_ADMIN_TOKEN_FILE` | Bearer for admin, beacon push and aggregator routes | | `CHALLENGE_WORKER_TOKEN_FILE` | Bearer for auditors and referees | | `CHALLENGE_MASTER_URL` | Used for `GET /v1/metagraph/latest`, which returns `{"hotkeys": {ss58: uid}}` and is cached 30 s | -| `HYPERTRAIN_COORD_KEY_FILE` | K_coord sr25519 seed: 32 raw bytes or 64 hex. It is never the Cortex leaf seed | +| `HYPERTRAIN_COORD_KEY_FILE` | K_coord sr25519 seed: 32 raw bytes or 64 hex | | `HYPERTRAIN_OWNER_HOTKEY` | ss58 of K_owner, which signs RunManifest | | `HYPERTRAIN_GENESIS_UNIX` | Ledger `Params`. Default: quicknet genesis | | `HYPERTRAIN_EPOCH_SECONDS` | Ledger `Params`. Default 4320 | diff --git a/docs/findings/v2-weights-bridge.md b/docs/findings/v2-weights-bridge.md new file mode 100644 index 0000000..66451d9 --- /dev/null +++ b/docs/findings/v2-weights-bridge.md @@ -0,0 +1,84 @@ +# Finding: network-v2 rewards never reach `get_weights` + +Status: **bridged** (fix option 2 below). Before the fix (base `1818f9e`) a matured +contract-2 reward was never paid as weight. End-to-end proof: +`tests/challenge/test_v2_weights_bridge.py::test_v2_matured_reward_reaches_get_weights` +(slow, no longer xfail). Unit proofs: `tests/challenge/test_v2_weights_pay.py`. + +## Bridge +`ChallengeStore.weights` computes the v1 answer, then `_pay_v2` fills the mass it left as +`units_burned_this_epoch` from `EscrowV2.payable(round)`. `round` is the drand round of +`epoch_at` under the run's beacon (`(epoch_at - genesis_time) // period + 1`), so the answer +never reads the wall clock. Each row is moved `available -> paid` with `EscrowV2.pay` inside +the transaction that writes `weights_runs`; the weight goes to the origin's hotkey. +- Paid: free (`available`) units of live reward origins (`escrow_finalized`) whose + `mature_at <= round`, oldest maturity first, still held by the owner the origin was issued to. +- Never paid: `reward_pending`, `admission_locked`, `dispute_locked`, `burned`; shadow + (bootstrap) origins and test genesis, which stay collateral (founder decision, 2026-10-19); + units transferred to another owner or credited to a referee. +- Mass: `full_share_mass` stays 10^6 per store; weights + burned == 10^6. Units over the + budget stay `available` and pay in later epochs. v1 entitlements are paid first. +- Idempotent: a stored epoch answer is returned before any v2 work. A failed persist rolls + the pay back; the retry recomputes and pays once. The `weights_runs_from` meta key tells + such a retry (the v1 journal already holds the epoch) from a genuine pre-multi-run answer. +- Deregistered hotkey: `registered_weights` (`app.py`) still drops it after the store + answered. The escrow units are already `paid`, so they burn to UID0 at the master and are + not owed again (the existing `ponytail`). No freeze was added. +- Child runs (`runs//`) pay and commit in their own database before the root commits. + If the root then fails, its retry is not legacy (`weights_runs_from`), so it reads the + child's stored answer and merges it again: the child's units are emitted once. + +## Code path +- `GET /internal/v1/get_weights` (`src/hypertrain/challenge/app.py:864`) calls `store.weights` + (`app.py:874`), then the metagraph filter `registered_weights` (`app.py:884`). +- `ChallengeStore.weights` (`src/hypertrain/challenge/store.py:1992`) returns a stored + `weights_runs` answer first. Otherwise it computes the v1 answer with + `self.ledger.get_weights` (`store.py:2029`, `src/hypertrain/ledger/ledger.py:490`) and, + when a live v2 reward was finalized, fills it with `_pay_v2` (call `store.py:2033`, + definition `store.py:2043`). Child runs merge through `_merge_answers` (`store.py:322`). +- `_pay_v2` reads `EscrowV2.payable` (`src/hypertrain/ledger/escrow_v2.py:973`) and moves each + row with `EscrowV2.pay` (`escrow_v2.py:964`, called at `store.py:2077`), skipping a new + hotkey once the answer holds `MAX_ENTRIES` (`store.py:2064`). +- `finalize_v2` (`store.py:8132`) issues live origins with `escrow.reward_finalize` + (`store.py:8267`); `push_beacon` (`store.py:663`) matures them to `available`. +- `weights_runs_from` is set on the first v2-era call to one past the newest epoch the v1 + journal answered, so no epoch the journal served before multi-run hosting is re-paid. + +Line numbers match the tree of this note's commit; re-grep `def weights`, `_pay_v2`, `def payable` after edits. + +## Not addressed (relay review, one line each) +- B4: grants are idempotent per live assignment; nothing revokes one when the original arrives. +- B5: `relay-disabled` is keyed by registry epoch, so a rotation re-enables a disabled key. +- B6: the relay accepts at `now == d_upload` (`now > deadline`) while the master refuses + `>= d_upload`; no honest relay is disabled by it. + +## Test +Reuses `_graduate_four` + `_live_checkpoint` (`tests/e2e/test_service_network_v2_e2e.py`). +They finalize live round 0 via `finalize_v2` (status `SETTLED`, four origins of 250000). +The test pushes the beacon past `mature_at`, reads every `available` round-0 unit, then calls +the route and asserts the exact weight vector (four hotkeys at 250000), `units_paid` and the +burned remainder. The fixture has no internal token, so the test points the app `Config` at a +tmp token file. Run time ~150 s (measured 149 s on the host), so it is marked `slow`. + +Before the bridge (base `1818f9e`, earlier `> 0` assertion, no xfail marker): +``` +> assert response.json()["weights"].get(hotkey, 0) > 0, response.text +E AssertionError: {"challenge_slug":"hypertrain",...,"epoch":0,"full_share_mass":1000000, +"metadata":{...,"ledger_minted":0,"ledger_paid":0,"ledger_pending":0,"reason":"ok", +"round_at":0,"units_burned_this_epoch":1000000,"units_paid":0},"weights":{}} +1 failed, 3 warnings in 139.48s +``` +With `xfail(strict=True)`: `1 xfailed, 3 warnings in 142.30s`. + +## Fix options (option 2 implemented) +1. Mirror v2 into the v1 ledger. `finalize_v2` adds one ledger entitlement per origin + (new `Ledger` event kind). Vesting stays in the ledger. Touches `challenge/store.py` + (`finalize_v2`), `ledger/ledger.py`. Risk: the same vesting is kept in two places, and an + EscrowV2 dispute or burn must also claw back the ledger. +2. Answer weights from EscrowV2. `ChallengeStore.weights` adds, for v2 runs, the units that + matured up to `epoch_at` per origin hotkey. It moves them `available` -> `paid` with + `EscrowV2.pay` in the same transaction, then merges via `_merge_answers`. Touches + `challenge/store.py` (`weights`, `_merge_answers`), `ledger/escrow_v2.py` (a query by + maturity round). Escrow stays the only source of truth. +3. Product decision: v2 units are not emissions. Document that `get_weights` pays only v1, + and stop promising v2 rewards. Touches `docs/` only. diff --git a/docs/mfu-multigpu.md b/docs/mfu-multigpu.md index 918241a..d30fc63 100644 --- a/docs/mfu-multigpu.md +++ b/docs/mfu-multigpu.md @@ -33,7 +33,7 @@ peak, tokens/s, per-step times, `rank_peak_mem_gib` (whole round), `rank_step_pe |------|--------|--------| | `--reduction zero2o` | Gradients reduced to their ZeRO-2 owner in buckets issued from post-accumulate grad hooks, so communication overlaps the rest of backward. | Bitwise equal to `zero2` (same adds, same order). | | `--act-ckpt` (`HT_ACT_CKPT=1`) | Per-layer activation checkpointing. | Same kernels on recompute: bitwise equal. | -| sdpa-flash | Selected on (sm_120, torch 2.14.0+cu130) from the qualification record in `tests/trainer/data/attn_qualify_sm120.json`. | Qualified by cross-process equality. | +| sdpa-flash | Selected on (sm_120, torch 2.14.0+cu130) from the qualification record in `tests/trainer/data/attn_qualify_sm120.json`. When the flash kernel rejects `enable_gqa`, the first accepting path of `attention.PATHS` is probed once per (device, dtype, group, head dim) and logged as `attention backend: `; a dtype or head dim no path accepts (e.g. fp32) stops the run (set `HT_ATTN_BACKEND=math` for such models); inputs whose layout the chosen kernel still rejects stop the run with the kernel error (no math fallback). | Qualified by cross-process equality (flash path only). A fallback path is a function of hardware, torch and geometry, so ranks agree, but it is not qualified. | | `--compile` (`HT_COMPILE=1`) | One Inductor graph per layer. | Leaves differ from eager (fusion changes rounding), so a compiled run replays only under the same flag. Both modes replay bitwise. | | `--micro-batch N` | Overrides the profile's per-rank micro_batch. | Part of the run; recorded as `HT_MFU_MICRO_BATCH`. | | `--nccl-pcie` | setdefaults `NCCL_P2P_DISABLE=1`, Ring, Simple, 4-8 channels, 8 MiB buffers. | Collectives are copies: no numeric effect. | @@ -65,7 +65,10 @@ host with `pcie_bw` >= 50. - **Host PCIe.** The same dp2 config measured 11.2-11.5 % on a host where NCCL took ~3.8 s of GPU time per step whether P2P was on or off, against 37.2 % on a `pcie_bw` 53.8 (PCIe Gen5) host. `--profile mfu` therefore filters offers on `pcie_bw` and requires `cuda_max_good >= 13.2` - (the image is CUDA 13.2; a 13.0 host refused create with HTTP 400). + (the image is CUDA 13.2; a 13.0 host refused create with HTTP 400). Every profile also keeps + only the drivers in `scripts/genuine_cuda_ci.py` `DRIVERS`; MFU does not compare replays, so + add `--any-driver` to widen the market. Evidence goes to `.evidence/vast/` unless + `--evidence-dir` is given. - **Round-end memory, not step memory.** `max_memory_allocated` over a round (~29.9 GiB at micro_batch 4) is dominated by the round end: full weights and optimizer state, delta, error feedback and the compressed payload. Inside the steps the peak is ~20 GiB, which is why @@ -75,3 +78,27 @@ host with `pcie_bw` >= 50. reduce-scatter-shaped owner reduction and a larger micro_batch at dp4 are the next steps. - **Elementwise work.** In the compiled trace, CUDA time outside GEMM/attention/NCCL (fp32 adds, fills, casts, checkpoint recompute) is still the largest bucket. + +## All GPU proofs on one host (`--profile proofs`) + +One 2x RTX 5090 instance, one upload, one DELETE and one parsed absence proof at the end. The +image is the digest-pinned `IMAGE` of `scripts/vast_e2e.py`; Vast pulls it before ssh is ready, +so the per-step seconds in `out/steps.tsv` (name, rc, seconds) exclude the pull. + +```bash +uv run --frozen python scripts/vast_e2e.py --profile proofs --key-file /var/lib/openbot/.vast-key \ + --max-usd 8 --max-minutes 45 --dry-run +``` + +`--dry-run` prints the chosen offer, then the step list (`PROOF_PLAN`), the pytest node ids +(`PROOF_TESTS`), the caps and the worst-case cost, without creating anything (it does make read-only GET calls: credit and up to 8 offer pages). `remote_env` is printed with values masked (`KEY=***`). A real run exits 4 when it passed on a driver outside `QUALIFIED_DRIVERS`: the result is valid but backs no bitwise-replay claim. Steps: the +gpu-e2e flow (network round, auditor replay, repeat; `HT_ATTN_BACKEND=math` because the fixture +model is fp32 with head_dim 4), then with `HT_ATTN_BACKEND` unset: `select_backend` must pick +sdpa-flash, the GPU-only tests (a skip fails), and a 1B bf16 dp2 round plus replay through +`run_mfu_dist.py`, checked by `check_gpu_evidence.py --kind mfu` and `attention_backend == +sdpa-flash`. `--boot-retries` defaults to 0 here. + +Drivers: the default is `--any-driver`, and the offer quote and `harness-result.json` carry +`driver_qualified` (false when the driver is not in `scripts/genuine_cuda_ci.py` `DRIVERS`, so +no bitwise claim is made). `--eight-gpu-qualified` rents 8x RTX 5090 with a qualified driver +only; nothing else selects 8 GPUs for this profile. diff --git a/docs/miner.md b/docs/miner.md index 9183a99..1eb7068 100644 --- a/docs/miner.md +++ b/docs/miner.md @@ -31,12 +31,18 @@ uv run --frozen hypertrain-miner run --help | grep -q -- '--rounds' uv run --frozen hypertrain-miner join --help | grep -q -- '--config' ``` -Get the dataset and check it against the run's Merkle root: +Get the dataset and check it against the run's Merkle root. A contract-2 `run-v2` job needs no local shards: the service serves each job's `samples` and `sample_proofs` objects, and the island checks every row against the signed `merkle_root`. For a u16 `run`, download `manifest.json` from the operator's base URL into `data_dir`. Save shard `i` as `shard-{i:05d}.u16`; its URL key is `shard_uri_template` with `manifest.shard_sha256s[i]`. Then verify (this covers u16 sets only): ```bash doctest -uv run --frozen python experiments/fetch_data.py --verify-only +uv run --frozen python -m hypertrain.datasets verify --help >/dev/null ``` +```bash doctest-skip:prod +uv run --frozen python -m hypertrain.datasets verify data +``` + +It prints `OK`, or one line per bad shard and exits 1. If `data_dir` holds no shards, `run` fetches each unit it needs by HTTP Range from `HYPERTRAIN_DATA_BASES` through `hypertrain.datasets.fetch.prefetch`, and checks each unit against the signed unit hashes. + The config is a TOML file. Every key can be overridden with `HYPERTRAIN_MINER_` in the environment, and unknown keys are rejected. Relative paths resolve against the config file's directory. | Key | Required | Meaning | @@ -109,7 +115,7 @@ Replay is bitwise, so any of these will turn an honest run into a MISMATCH: - Import `hypertrain.trainer` before anything imports torch. It sets `CUBLAS_WORKSPACE_CONFIG=:4096:8`, deterministic algorithms with no warn-only fallback, cuDNN deterministic and no benchmark, TF32 off for matmul and cuDNN, no reduced-precision BF16 reductions, and one CPU thread. The image also sets `CUDA_DISABLE_PTX_JIT=1`. - Run the pinned image (`reference_spec.image_digest`), not your own build. - Same GPU model and SM count as the reference; driver in the allowlist when there is one. -- Verified shards (`--verify-only` must print `VERIFY OK`). +- Verified shards (`python -m hypertrain.datasets verify ` must print `OK`). - Don't change batch size, accumulation, threads or anything else the manifest fixes. Check the pins in your environment: @@ -163,7 +169,22 @@ hypertrain-miner status --config miner.toml hypertrain-miner update --check --manifest-url "$MANIFEST_URL" ``` -`register --coldkey ... --request-id ... --expiry ...` is the legacy `join-v2` path. A revoked key is rejected on every authenticated miner route. +`register` (no `--coldkey`) posts the server-key grant (`grant.json` next to the keyfile) and submits no contract-2 admission. `register --coldkey COLDFILE --request-id ID --expiry ROUND` posts no grant; instead it signs a join v2 with the configured keyfile (the server key) and the coldkey and posts it to `/v2/runs//join`, so it needs the coldkey file and belongs on the operator machine. For contract 2, prepare the hotkey-signed request on the GPU host using its configured hotkey (not a delegated server key): + +```bash +hypertrain-miner join-v2 --config miner.toml --request-id "$REQUEST_ID" --expiry "$EXPIRY" \ + --coldkey-ss58 "$COLDKEY_SS58" --prepare join.json +``` + +Copy `join.json` to the operator machine. It contains only public fields and the hotkey signature; no coldkey is needed on the GPU host. Co-sign and submit without `miner.toml`, then lock collateral from explicit existing origins: + +```bash +hypertrain-miner join-v2-cosign --request join.json --coldkey ~/cold.key --api "$API" +hypertrain-miner escrow-lock --api "$API" --run-id "$RUN_ID" --coldkey ~/cold.key \ + --hotkey-ss58 "$HOTKEY_SS58" --units "$UNITS" --origin "$ORIGIN_ID" +``` + +`REQUEST_ID`, `RUN_ID` and origin IDs are 64 lowercase hex characters; `EXPIRY` is a beacon round. Inspect the public request before co-signing. Use a new output filename for each preparation. Repeat `--origin` for multiple origins; the CLI never creates origins. `escrow-lock --dry-run` prints only the unsigned operation without posting. Its default expiry is the run's current beacon round plus 1000; `--expiry` overrides it. Identical inputs (including reordered origins) reuse the operation ID. Success reports the event hash, current locked units, total required units and server eligibility; funding alone does not guarantee eligibility. The existing `join-v2 --coldkey ...` path remains available. A revoked key is rejected on every authenticated miner route. Only sr25519-signed release manifests are accepted; unsigned tags and downgrades are rejected. During the announce window (at least 600 s) `run --auto-update` stages the new wheel while training continues. At the first round boundary at or after `changeover_at`, it swaps `~/.hypertrain/miner/current` atomically and re-executes the new version, which resumes from the round journal. A miner still on the old version 60 s after changeover refuses new rounds. `update --apply` does the stage and swap by hand. `HYPERTRAIN_UPDATE_ROOT` and `HYPERTRAIN_UPDATE_MANIFEST_URL` override the defaults. @@ -178,7 +199,7 @@ Only sr25519-signed release manifests are accepted; unsigned tags and downgrades | `HardwareMismatch: SM count ... != reference 170` | wrong GPU model for this run | | `driver ... not in the manifest allowlist` | install an allowed driver | | `this client trains single-GPU layouts only` | the run uses a multi-GPU layout, not supported by this client yet | -| `local shards are not the run's dataset` or `shard verification failed` | re-fetch the data and run `--verify-only` | +| `local shards are not the run's dataset` or `shard verification failed` | re-fetch the data and run `python -m hypertrain.datasets verify ` | | 409 on commit | that round is already committed; keep the journal in `workdir` between restarts | | status `EXCLUDED` on commit | committed after `d_audit`; move closer to the server or start earlier | | MISMATCH on a run you believe is honest | check section 4, then submit your rerun before finalize | diff --git a/docs/network-v2.md b/docs/network-v2.md index ef07ea3..1a4b7b8 100644 --- a/docs/network-v2.md +++ b/docs/network-v2.md @@ -78,13 +78,32 @@ device = "cpu" `api` is the service base, without `/v2/runs/...`. `NetworkMiner` requires pinned `run_id` and `owner_hotkey`; it verifies the signed wrapper before work. `state_source` and `image_digest` remain required shared config fields; v2 round -inputs are fetched through the job/object endpoints. Relative paths resolve +inputs are fetched through the job/object endpoints. Re-requesting the same +round job (a miner restarting after a crash) returns the identical staged job; +staged inputs that differ on the server answer 409, never an overwrite. +Relative paths resolve against the TOML directory. Environment `HYPERTRAIN_MINER_` overrides matching TOML fields; unknown TOML keys reject. Other existing fields: `data_dir`, `poll_seconds` (2.0), `chunk_bytes` (67108864), `allow_file_upload` (false). Do not assume these v1-oriented tuning fields alter the v2 server's deadlines or relay chunk policy. +## CPU smoke + +`bash scripts/mine-smoke-v2.sh` runs this miner path offline on CPU: it starts the +real challenge app (tiny test-ledger fixture run, throwaway fixture keys) on +127.0.0.1 plus a self-signed TLS relay on localhost, then runs `join-v2`, +`escrow-lock`, four `trial_export`/`probe-v2`/`proof-v2` trials and +`run-v2 --round 0` as separate `uv run --frozen hypertrain-miner` processes. A round +needs a four-identity eligible roster, so three peer identities are graduated +in-process by the operator role (`"mode": "in-process"` in the evidence). PASS +requires every stage at rc 0 and the round-0 delta recorded; measured ~132 s, +budget `MINE_SMOKE_V2_BUDGET_SECONDS` (default 300). Evidence (stage, rc, ms) goes +to `$MINE_SMOKE_V2_EVIDENCE` or a fresh 0600 temp file; `MINE_SMOKE_V2_BREAK=` +points one of the four miner stages at a closed port to check failure detection (any +other name exits 2 before running). It proves the CLI +chain, not CUDA, audits, settlement or production funding. + ## Join, prove work, fund, graduate Set `REQUEST_ID` to a fresh 64-hex request ID, `EXPIRY_BEACON` to a future @@ -106,15 +125,21 @@ which includes its nonce; `WorkScreenV2` is reserved by admission and nonce. alone. Retry the exact stored signed operation: changing expiry or signature does not create a new semantic attempt (`protocol/envelope_v2.py:replay_key`). -For each challenge, obtain the coordinator-signed envelope from -`GET /v2/runs/{run_id}/join/{admission_id}/challenge`. The operator must supply the -matching staged `IslandJobV1` and all files in its `object_paths` beside the job. -The CLI does not download/stage an admission job automatically. Given those -existing files: +After `GET /v2/runs/{run_id}/join/{admission_id}/challenge` issues a challenge, +the operator exports the matching staged `IslandJobV1`, its inputs and the signed +challenge into a new or empty folder: + +```bash +uv run --frozen python -m hypertrain.challenge.trial_export \ + --api "$API" --run-id "$RUN_ID" --admission-id "$ADMISSION_ID" \ + --admin-token-file /path/to/admin-token --out trial +``` + +Copy `trial/` to the miner, then run: ```bash uv run --frozen hypertrain-miner probe-v2 --config miner.toml \ - --job trial/job.json --challenge challenge.json > proof.json + --job trial/job.json --challenge trial/challenge.json > proof.json uv run --frozen hypertrain-miner proof-v2 --config miner.toml --proof proof.json ``` @@ -147,6 +172,15 @@ fraud burns collectible exposure rather than redistributing it. Shadow work does not influence global theta; authorized production bootstrap issuance is a separate policy, not proof of market-backed collateral. +Matured live rewards become weight in `GET /internal/v1/get_weights`. Each epoch answer first +pays v1 entitlements, then fills the rest of its 10^6 mass with `available` units of live +reward origins whose `mature_at` is at or before the drand round of `epoch_at`. Those units +move to `paid` in the same transaction that stores the answer, so a repeated epoch request +returns the stored bytes and pays nothing again; units over the mass pay in later epochs. +Locked, disputed, pending, burned, transferred, shadow-bootstrap and genesis units are never +paid. A hotkey missing from the metagraph is dropped after payment and its units burn to UID0 +(see `docs/findings/v2-weights-bridge.md`). + ## Train one live round and watch disputes Only run after the operator has opened an eligible, funded round: @@ -230,6 +264,30 @@ an exact complete receipt or full original chunk coverage, authenticated healthy witnesses and original grant/assignment lineage. The master fetches/hash-checks original bytes before acceptance. Rescue preserves custody without backdating late acceptance or charging the miner for proven storage failure. +Multi-chunk custody without a receipt disables the relay key at once only when +the master holds the original and it proves every chunk hash. Otherwise the key +is disabled only after rescued bytes hash to `delta_hash`: the grant could not +tie those chunk hashes to the commit, so valid chunks that concatenate to the +wrong delta answer 422 and leave the relay enabled. A rescue the relay cannot +serve at all disables the key too: an HTTP error (including a backend 503), a +signed response other than `SERVED`, a bad response signature or binding, a +timeout, or streamed bytes that miss the signed, miner-committed chunk hash. Two +pinned auditors already proved the failure and the relay cannot produce its own +signed custody. The fallback answers `rescued: false, disabled: true`; the +`relay-disabled` id is `relay_id:assignment_epoch:key_id` and earlier evidence wins. +While a grant is live, `upload-grant` returns the same signed grant for the same +chunk list and 409 for another list, unless the master holds the original delta +and it proves the new list: then the honest list gets a new grant. +That grant carries `replaces`, the digest of the live grant it supersedes (an +optional `UploadGrant` field, omitted otherwise, so older grant digests are +unchanged). The relay then drops its unreceipted reservation for that grant; +it refuses the replacement when that upload has a receipt or belongs to another +miner, round or relay, and a wrong `replaces` leaves the old reservation blocking. +Upgrade order: deploy relays before the master. A master that emits `replaces` +in an `UploadGrant` (commit e3e5343 onward) is rejected by a relay built before +that commit (extra fields are forbidden). Roll relays first, then the master. +A relay whose granted chunks do not assemble to `delta_hash` answers 400 +`AssemblyMismatch`, records it on the upload and skips it in later recovery. Retention release waits for finality, disputes and vesting; signed extensions, overlapping old/new keys and drain preserve existing retrieval authority. diff --git a/docs/operator.md b/docs/operator.md index 72f708e..4b82e00 100644 --- a/docs/operator.md +++ b/docs/operator.md @@ -1,6 +1,6 @@ # Operator guide -This guide covers running hypertrain yourself: first locally, then on a Cortex master, then for a training run. Every block tagged `doctest` is run by `bash scripts/doctest_docs.sh` from the tree root against the local stack, in order, in one shell. Blocks tagged `doctest-skip:prod` need a real Cortex master. The script lists them but doesn't run them, and they were checked by reading them against the pinned Cortex docs (cortex@d738424: `deploy/README.md`, `docs/how-to/trust-root.md`, `docs/CHALLENGES.md`). +This guide covers running hypertrain yourself: first locally, then on a Cortex master, then for a training run. Every block tagged `doctest` is run by `bash scripts/doctest_docs.sh` from the tree root against the local stack, in order, in one shell. Blocks tagged `doctest-skip:prod` need a real Cortex master. The script lists them but doesn't run them, and they were checked by reading them against the pinned Cortex docs (Cortex `deploy/README.md` and `docs/CHALLENGES.md`). Contents: @@ -23,13 +23,23 @@ You need Python 3.12 through [uv](https://docs.astral.sh/uv/), `curl`, `jq`, `op uv sync --frozen --all-extras ``` -The training data is one pinned FineWeb-Edu parquet (ODC-By 1.0, revision `87f09149`), packed into byte-level u32 shards with a Merkle manifest under `data/`. The first run of `uv run --frozen python experiments/fetch_data.py` downloads and builds it (about 770 MB). After that, re-hash it offline with: +The training data is a registry mix (`src/hypertrain/datasets/registry.json`, for example `od-a-proxy-v1`), tokenized with the pinned tokenizer into u16 shards with a Merkle manifest. A contract-2 run needs three steps. `build` reads the pinned sources you already fetched into `cache//` plus `cache/tokenizer.json`; it never downloads. `pin` re-verifies the shards and writes the `samples` and `proofs` objects the service slices for every trial. It also prints the `training.dataset` fragment for the manifest; `--holdout-dir` adds `holdout_commit` from an existing holdout shard set. `register` copies both objects into the run's store under `CHALLENGE_STATE_DIR/runs//objects` and links them with `PUT /v2/runs/{run_id}/admin/dataset`. So it runs on the service host as the service's uid, after the run is created. The first run of a fresh state dir lives in the root store and is refused. ```bash doctest -uv run --frozen python experiments/fetch_data.py --verify-only +uv run --frozen python -m hypertrain.datasets pin --help >/dev/null +uv run --frozen python -m hypertrain.datasets register --help >/dev/null ``` -It prints `merkle_root=8bd23b4f...`, `n_samples=196608` and `VERIFY OK`. Auditors and miners must hold the same bytes, since replay is bitwise. +```bash doctest-skip:prod +uv run --frozen python -m hypertrain.datasets build --mix od-a-proxy-v1 --out data --cache cache +uv run --frozen python -m hypertrain.datasets pin --data-dir data --out-objects pinned \ + --holdout-dir data/holdout | tee pin.json +uv run --frozen python -m hypertrain.datasets register --api "$API" --run-id "$RUN" \ + --admin-token-file /secrets/admin.token --objects pinned --state-dir "$CHALLENGE_STATE_DIR" \ + --samples-hash "$(jq -r .samples_hash pin.json)" --proofs-hash "$(jq -r .proofs_hash pin.json)" +``` + +`python -m hypertrain.datasets verify data` re-hashes a shard set offline and prints `OK`. `pin` and `verify` exit 1 on any shard, unit or `merkle_root` mismatch, and `register` exits 1 on any 4xx/5xx and prints the service's answer. `proofs` holds `n_samples` paths of `depth` hashes (about `n_samples * depth * 67` bytes). Auditors and miners must hold the same bytes, since replay is bitwise. The protocol pieces each ship a QA script that runs the real code on small inputs. They are the quickest check that a checkout is sound: @@ -72,7 +82,7 @@ The server is a FastAPI app, `hypertrain.challenge.app:app`. It reads its settin | `CHALLENGE_INTERNAL_TOKEN_FILE` | unset | master bearer for `get_weights` | | `CHALLENGE_ADMIN_TOKEN_FILE` | unset | operator bearer for `/v1/admin/*` and `/v1/aggregator/*` | | `CHALLENGE_WORKER_TOKEN_FILE` | unset | auditor bearer for `/v1/worker/*` | -| `HYPERTRAIN_COORD_KEY_FILE` | unset | K_coord seed (32 raw bytes or 64 hex), never the Cortex leaf seed | +| `HYPERTRAIN_COORD_KEY_FILE` | unset | K_coord seed (32 raw bytes or 64 hex) | | `HYPERTRAIN_OWNER_HOTKEY` | unset | ss58 of K_owner, the only key that may sign a RunManifest | | `HYPERTRAIN_EPOCH_SECONDS` | `4320` | Cortex epoch length | | `HYPERTRAIN_EPOCHS_PER_ROUND` | `1` | must equal the manifest `budget.epochs_per_round` | @@ -132,6 +142,8 @@ uv run --frozen python -m hypertrain.beacon check --live `beacon check --live` fetches and verifies the latest round from the default relays; pass `--relay URL` (repeatable) to use your own. +The server refuses v2 operations once it is more than two rounds (6 s) behind, so keep `uv run --frozen python -m hypertrain.beacon push --api "$API" --admin-token-file "$HT/secrets/admin_token"` running next to it. Every `--interval` (3 s) it asks the relays (`--relay URL`, repeatable, at least two distinct ones, all must agree) for the latest round, verifies it, and pushes the rounds the server is missing, oldest first, at most the last `--catch-up` (20). It prints one JSON line per tick, `{"pushed": [...], "latest": n, "known": n}`; `--once` runs a single tick, for cron. Relay, network, 408, 429 and 5xx failures are retried; exit codes: 0 after a good `--once` tick or on SIGTERM; 1 on any other service status (a 409 means a conflicting round, never overwritten), a failed `--once` tick or `--max-failures` (10) failures in a row; 2 on a usage error, a 401/403, or a token file that is not one `[A-Za-z0-9._~-]+` token. + ### The manifest `hypertrain.protocol.example.example_manifest()` builds the example CPU manifest. Its `PROVENANCE` dict says where each hyperparameter comes from in the plan and its advisories: @@ -178,6 +190,24 @@ curl -fsS -H "$ADMIN" -H 'Content-Type: application/json' --data-binary @"$HT/ma "$API/v1/admin/runs" | jq -e '.status == "created"' ``` +### Contract-2 manifest CLI + +Create editable CPU test-mode inputs, then build the owner-signed manifest and stage its six dependency objects before starting the service: + +```bash +uv run --frozen python -m hypertrain.ops.manifest_v2 example --out /tmp/ex +uv run --frozen python -m hypertrain.ops.manifest_v2 build \ + --training /tmp/ex/training.json --policies /tmp/ex/policies.json \ + --relays /tmp/ex/relays.json --owner-seed /private/owner.seed \ + --exp-round 1000000 --state-dir /tmp/state --out /tmp/out +``` + +The example's `coord_pubkey`, two auditors, referees, `reward_authority` and relay public key are **placeholder SS58 public identities, not operational keys**. Replace them before a real run; `coord_pubkey` must match the service's K_coord. The example uses a tiny decoder, `ledger_mode="test"`, a pinned genesis allocation hash and one `.test` relay. It creates no dataset, genesis funding or training round. Custom CPU/GPU runs use the same three JSON shapes; provide valid model dimensions and parameter count, an audit budget of `2H`, and the H/step-bound-derived audit lease. Build computes `init_state_hash`; an existing value must match. It also rejects economics whose `reward_authority` differs from `training.coord_pubkey` (`PINNED_AUTHORITIES`) or whose `round_reward_units` differs from `training.budget.epochs_per_round * 1_000_000` (`MANIFEST_REWARD_BUDGET`). The service's configured auditor set must match `training.auditors`; the offline CLI cannot validate that configuration. Building needs the existing `trainer` extra (`uv sync --frozen --extra trainer`); help and example do not. + +The owner seed is an existing private, current-user-owned 0600 file containing 32 raw bytes or 64 hex characters; it is read through the miner's checked loader, never printed or copied. The shared loader strips leading/trailing ASCII whitespace bytes: provide hex for any raw seed whose first or last byte is ASCII whitespace. Choose an expiry drand round later than submission. Both commands require a new or empty output directory. Build writes `manifest.json`, `manifest-envelope.json`, five `policy-.json` files, `relay-registry.json` and `index.json`. The index records `run_id`, `owner_ss58`, six canonical object hashes and hashes of the eight other files; it excludes its own hash. + +`--state-dir` is optional; when supplied, the six objects are stored under `/tmp/state/objects//`. Start the service with `CHALLENGE_STATE_DIR=/tmp/state`, the matching K_coord and `HYPERTRAIN_OWNER_HOTKEY` set to `index.json`'s owner. Submit `manifest-envelope.json` to `POST /v2/admin/runs` with the admin bearer after the service has a valid beacon round. The CLI makes no HTTP requests. Success prints JSON; exit codes: 0 success, 2 input/validation error, 1 unexpected error. + ### Configure, admit, unpause, pause `train_rounds` is the number of drand rounds between `d_assign` and `d_commit` (the training window). `final_after_upload` is how many rounds after the later of `d_audit` and `d_upload` a round may finalize. @@ -267,7 +297,7 @@ Relays per region: what is built is the budget line and the beacon side. The man ## 7. Production on a Cortex master -These steps follow the pinned Cortex deploy guide. None of them run in the doc-test. +These steps follow the Cortex deploy guide. None of them run in the doc-test. ### Secrets @@ -281,19 +311,7 @@ install -d -m 0700 -o 65532 -g 65532 "$dir" chown 65532:65532 "$dir"/*.token "$dir/coord.key" ``` -`internal.token` is the master's bearer for `get_weights`. `admin.token` is yours. `worker.token` goes to auditors. `coord.key` is K_coord, which signs receipts, RoundOpen and Forfeit messages; its ss58 goes in the manifest `coord_pubkey`. It is not the Cortex leaf key. - -### Leaf key - -The leaf key signs the challenge's weight leaves for the trust root. Generate it offline; `keygen` refuses to overwrite a file: - -```bash doctest-skip:prod -cortex keygen \ - --seed-out /private/cortex-ceremony/hypertrain.seed \ - --public-out /private/cortex-ceremony/hypertrain.pubkey -``` - -Install the seed on the master at `deploy/secrets/master/hypertrain.key` as a private regular file. The master fails closed if it's missing, group-readable, a symlink, or doesn't match the signed public key. +`internal.token` is the master's bearer for `get_weights`. `admin.token` is yours. `worker.token` goes to auditors. `coord.key` is K_coord, which signs receipts, RoundOpen and Forfeit messages inside hypertrain; its ss58 goes in the manifest `coord_pubkey`. Cortex needs no key for hypertrain: it scores through `get_weights` and the gateway is the single source of truth. ### Registry row and burn-in @@ -305,33 +323,17 @@ docker compose --project-directory . --env-file deploy/env/master.env \ curl -fsS https:///challenge/hypertrain/version ``` -A registered id that the trust root doesn't list runs without emission. That's the burn-in: check `/challenge/hypertrain/version`, `/health` and the logs, launch a small run, and let at least one round finalize before you sign. - -### Trust root activation (owner-signed, version 3 or higher) - -Add [deploy/trust-root-v3-row.example.toml](../deploy/trust-root-v3-row.example.toml) to the owner's next challenges document (version 3 or higher, algorithm 3), with `public_key` set to the leaf public key and every share rebalanced so the total is exactly 10000 bps. Raise `version`, set `introduced_epoch`, then sign and verify offline: +A registered id that the challenge profile doesn't list runs without emission. That's the burn-in: check `/challenge/hypertrain/version`, `/health` and the logs, launch a small run, and let at least one round finalize before you add the profile row. -```bash doctest-skip:prod -cortex trust-sign \ - --kind challenges \ - --input config/challenges.toml \ - --seed-file /private/cortex-ceremony/owner.seed \ - --signature-out config/challenges.toml.sig -cortex trust-verify \ - --challenges config/challenges.toml \ - --measurements config/measurements.toml \ - --owner-public config/owner.pubkey \ - --gateway-public "$(tr -d '\n' = 3`) and leave `introduced_epoch` at or below the current epoch: a future value stops the master from sealing older epochs and the gateway from decoding stored seals, which burns the emission. There is no signing step: the gateway is the source of truth, the master reads this file and the gateway serves it at `GET /v1/trust`, which validators fetch over HTTPS. Restart the master, then check the gateway's `/v1/trust` lists hypertrain, a new `sealed: true` latest response carries hypertrain weights, and an independent validator recomputation matches. Hypertrain returns `full_share_mass = 10^6` per epoch, so any unverified or unvested mass burns rather than going to other miners. ### Rollback - Image: the supervisor already rolls back a digest that fails `/version`. To hold a known-good image, set `pin = "sha256:..."` in the registry row. -- Trust root: restoring an older signed file is rejected by version watermarks. Stop submission and sign a higher-version recovery profile through the same ceremony, for example one without the hypertrain row. -- Retire: sign a higher version without the hypertrain row first, then remove the registry row. Deleting the row stops the container and keeps its volume. +- Challenge profile: restoring an older `version` is rejected by the gateway and the validator watermark. Publish a higher-version profile instead, for example one without the hypertrain row. +- Retire: publish a higher profile version without the hypertrain row first, then remove the registry row. Deleting the row stops the container and keeps its volume. - A training round: `POST /v1/aggregator/runs/{id}/rounds/{w}/rollback` before finalize. After finality, fraud is handled with money only (burn and clawback), never by subtracting a delta. ## 8. Budget and teardown diff --git a/pyproject.toml b/pyproject.toml index db62419..7828ccc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,11 @@ testpaths = ["tests"] # the `pytest` console script does not put the rootdir on sys.path, `python -m pytest` does. pythonpath = ["."] addopts = "-ra --strict-markers" -markers = ["od: builds a real od-base OpenDecision model (excluded from PR CI)"] +markers = [ + "od: builds a real od-base OpenDecision model (excluded from PR CI)", + "fast: CPU mining-path subset run by scripts/check-fast.sh in under 2 minutes (set in tests/conftest.py)", + "slow: end-to-end test over ~100 s (run explicitly)", +] [tool.ruff] target-version = "py312" diff --git a/scripts/check-fast.sh b/scripts/check-fast.sh new file mode 100755 index 0000000..3d14226 --- /dev/null +++ b/scripts/check-fast.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Fast gate: the `fast`-marked CPU mining-path subset (see tests/conftest.py) in under 2 minutes. +# Paths are passed explicitly so pytest only collects those files (collecting the whole tree +# under -m costs ~45 s on a loaded host). Prints the measured wall time; exits 1 when tests +# fail or the budget is exceeded. Extra arguments go to pytest. +# Not a bitwise gate: the fast set excludes golden replay, codec v2, tests/parity/, trace parity +# and island replay v2; scripts/check-proof.sh runs those, run it before merging +# protocol-adjacent changes. +set -euo pipefail +cd "$(dirname "$0")/.." +export PATH="$HOME/.local/bin:$PATH" + +budget="${CHECK_FAST_BUDGET_SECONDS:-120}" +mapfile -t paths < <(uv run --frozen python - <<'EOF' +import sys +sys.path.insert(0, "tests") +import conftest # noqa: E402 +print("\n".join([*conftest.FAST_DIRS, *conftest.FAST_FILES, "tests/test_mine_smoke_v2.py"])) +EOF +) +start=$(date +%s%N) +rc=0 +timeout -k 5 "$budget" uv run --frozen python -m pytest -q -p no:cacheprovider \ + -m "fast and not od" --durations=10 "${paths[@]}" "$@" || rc=$? +end=$(date +%s%N) +elapsed_ms=$(( (end - start) / 1000000 )) +printf 'check-fast: exit=%s elapsed=%sms budget=%ss\n' "$rc" "$elapsed_ms" "$budget" + +if [ "$rc" -eq 124 ]; then + echo "check-fast: FAIL (budget exceeded)" >&2 + exit 1 +fi +[ "$rc" -eq 0 ] || exit "$rc" +if [ "$elapsed_ms" -gt $(( budget * 1000 )) ]; then + echo "check-fast: FAIL (${elapsed_ms}ms > ${budget}s)" >&2 + exit 1 +fi +echo "check-fast: NOTE bitwise suites (golden replay, codec v2, parity, trace parity, island replay v2) are NOT in this gate; run scripts/check-proof.sh before merging protocol-adjacent changes" +echo "check-fast: PASS" diff --git a/scripts/check-proof.sh b/scripts/check-proof.sh new file mode 100755 index 0000000..7a01501 --- /dev/null +++ b/scripts/check-proof.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Bitwise proof gate: golden replay and its GOLDEN consumers (act ckpt, fp8 off), codec v2, +# tests/parity, trace parity and island replay v2, the suites scripts/check-fast.sh leaves out. +# One pytest run per suite so each prints its own wall time; all suites share one budget +# (default 600 s). `od` tests are excluded as in PR CI (they need the opendecision extra). +# Exits 1 when a suite fails or the budget is exceeded. +# A skip is never silent: -rs prints each reason and any skip outside ALLOWED_SKIPS fails the +# gate (a skipped proof proves nothing). Extra arguments go to every pytest run. +# A golden xfail fails the gate too (-rsx prints it): an unmatched leaf root is not a successful +# golden comparison. It is matched on the reason of tests/trainer/test_golden_replay.py +# _expect_golden, so every GOLDEN consumer is covered. CHECK_PROOF_ALLOW_GOLDEN_XFAIL=1 lets it +# pass: diagnostic only, never for a release. +set -euo pipefail +cd "$(dirname "$0")/.." +export PATH="$HOME/.local/bin:$PATH" + +budget="${CHECK_PROOF_BUDGET_SECONDS:-600}" +# Reasons (extended regex) of skips that do not weaken a proof: the optional full-grid artifact. +# Matched against the reason only (after "SKIPPED [n] file:line: "), anchored at both ends. +ALLOWED_SKIPS="${CHECK_PROOF_ALLOWED_SKIPS:-full grid summary not produced yet}" +log="$(mktemp)" +trap 'rm -f "$log"' EXIT +suites=( + tests/trainer/test_golden_replay.py + tests/trainer/test_act_ckpt.py + tests/trainer/test_engine_flags.py::test_fp8_off_by_default + tests/trainer/test_codec_v2.py + tests/parity/test_parity.py + tests/auditor/test_trace_parity.py + tests/auditor/test_island_replay_v2.py +) +start=$(date +%s%N) +failed=() +for suite in "${suites[@]}"; do + left_ms=$(( budget * 1000 - ($(date +%s%N) - start) / 1000000 )) + if [ "$left_ms" -le 0 ]; then + echo "check-proof: FAIL (budget exceeded before ${suite})" >&2 + echo "check-proof: FAIL" + exit 1 + fi + s=$(date +%s%N) + rc=0 + PY_COLORS=0 NO_COLOR=1 timeout -k 5 "$(( (left_ms + 999) / 1000 ))" uv run --frozen python -m pytest -q -rsx \ + --color=no -p no:cacheprovider -m "not od" --durations=5 "$suite" "$@" 2>&1 | tee "$log" || rc=${PIPESTATUS[0]} + if unexpected=$(sed 's/\x1b\[[0-9;]*m//g' "$log" | grep '^SKIPPED' \ + | grep -Ev -- "^SKIPPED \[[0-9]+\] [^ ]+:[0-9]+: (${ALLOWED_SKIPS})\$"); then + printf 'check-proof: unexpected skip in %s:\n%s\n' "$suite" "$unexpected" >&2 + [ "$rc" -ne 0 ] || rc=1 + fi + if [ "${CHECK_PROOF_ALLOW_GOLDEN_XFAIL:-0}" != 1 ] && xfailed=$(sed 's/\x1b\[[0-9;]*m//g' "$log" \ + | grep '^XFAIL .* - golden leaf root differs on this CPU class'); then + printf 'check-proof: golden replay xfail is not a proof (CHECK_PROOF_ALLOW_GOLDEN_XFAIL=1 for diagnostics only):\n%s\n' \ + "$xfailed" >&2 + [ "$rc" -ne 0 ] || rc=1 + fi + printf 'check-proof: suite=%s exit=%s elapsed=%sms\n' \ + "$suite" "$rc" "$(( ($(date +%s%N) - s) / 1000000 ))" + [ "$rc" -eq 0 ] || failed+=("$suite") +done +elapsed_ms=$(( ($(date +%s%N) - start) / 1000000 )) +printf 'check-proof: elapsed=%sms budget=%ss\n' "$elapsed_ms" "$budget" + +if [ "${#failed[@]}" -gt 0 ]; then + echo "check-proof: failed suites: ${failed[*]}" >&2 + echo "check-proof: FAIL" + exit 1 +fi +if [ "$elapsed_ms" -gt $(( budget * 1000 )) ]; then + echo "check-proof: FAIL (${elapsed_ms}ms > ${budget}s)" >&2 + echo "check-proof: FAIL" + exit 1 +fi +echo "check-proof: PASS" diff --git a/scripts/mine-smoke-v2.sh b/scripts/mine-smoke-v2.sh new file mode 100755 index 0000000..d9a4e48 --- /dev/null +++ b/scripts/mine-smoke-v2.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# CPU network-v2 mining smoke: challenge service + TLS relay on 127.0.0.1, one miner driving +# join-v2 -> probe-v2 -> proof-v2 -> run-v2 through the real CLI (see scripts/mine_smoke_v2.py). +# PASS = every stage exits 0 and run-v2 uploads round 0 inside the budget +# (default 300 s: measured ~132 s on CPU, plus margin for a loaded host). +set -euo pipefail +cd "$(dirname "$0")/.." +export PATH="$HOME/.local/bin:$PATH" + +budget="${MINE_SMOKE_V2_BUDGET_SECONDS:-300}" +evidence="${MINE_SMOKE_V2_EVIDENCE:-$(mktemp -d)/mine-smoke-v2-evidence.json}" +export MINE_SMOKE_V2_EVIDENCE="$evidence" + +start=$(date +%s%N) +rc=0 +timeout -k 5 "$budget" uv run --frozen python scripts/mine_smoke_v2.py || rc=$? +end=$(date +%s%N) +elapsed_ms=$(( (end - start) / 1000000 )) + +printf 'mine-smoke-v2: exit=%s elapsed=%sms budget=%ss evidence=%s\n' \ + "$rc" "$elapsed_ms" "$budget" "$evidence" + +if [ "$rc" -eq 124 ]; then + echo "mine-smoke-v2: FAIL (budget exceeded)" >&2 + exit 1 +fi +[ "$rc" -eq 0 ] || exit "$rc" + +budget_ms=$(( budget * 1000 )) +if [ "$elapsed_ms" -gt "$budget_ms" ]; then + echo "mine-smoke-v2: FAIL (${elapsed_ms}ms > ${budget_ms}ms)" >&2 + exit 1 +fi +echo "mine-smoke-v2: PASS" diff --git a/scripts/mine-smoke.sh b/scripts/mine-smoke.sh new file mode 100755 index 0000000..6a297c5 --- /dev/null +++ b/scripts/mine-smoke.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# CPU mining smoke: challenge service + one miner + one auditor, one complete test-mode round. +# PASS = the round reaches FINAL with a MATCH verdict inside the budget (default 120 s). +set -euo pipefail +cd "$(dirname "$0")/.." +export PATH="$HOME/.local/bin:$PATH" + +budget="${MINE_SMOKE_BUDGET_SECONDS:-120}" +evidence="${MINE_SMOKE_EVIDENCE:-$(mktemp -d)/mine-smoke-evidence.json}" +export MINE_SMOKE_EVIDENCE="$evidence" + +start=$(date +%s%N) +rc=0 +timeout -k 5 "$budget" uv run --frozen python scripts/mine_smoke.py || rc=$? +end=$(date +%s%N) +elapsed_ms=$(( (end - start) / 1000000 )) + +printf 'mine-smoke: exit=%s elapsed=%sms budget=%ss evidence=%s\n' \ + "$rc" "$elapsed_ms" "$budget" "$evidence" + +if [ "$rc" -eq 124 ]; then + echo "mine-smoke: FAIL (budget exceeded)" >&2 + exit 1 +fi +[ "$rc" -eq 0 ] || exit "$rc" + +budget_ms=$(( budget * 1000 )) +if [ "$elapsed_ms" -gt "$budget_ms" ]; then + echo "mine-smoke: FAIL (${elapsed_ms}ms > ${budget_ms}ms)" >&2 + exit 1 +fi +echo "mine-smoke: PASS" diff --git a/scripts/mine_multi_round_v2.py b/scripts/mine_multi_round_v2.py new file mode 100644 index 0000000..e31d7c0 --- /dev/null +++ b/scripts/mine_multi_round_v2.py @@ -0,0 +1,402 @@ +"""CPU network-v2 multi-round mining through the real CLI, up to a positive get_weights. + +Reuses scripts/mine_smoke_v2.py: its chain (real challenge app on 127.0.0.1, TLS relay, fixture +keys, join-v2 -> escrow-lock -> 4 probation trials -> run-v2 --round 0 for miner 0) runs first. +Then, for rounds 0..ROUNDS-1, every roster miner (4 fixture hotkeys) runs +`uv run --frozen hypertrain-miner run-v2 --round w` as its own process; the operator role leases +and completes the 4 audits, aggregates, finalizes and opens the next round. After the last +round, beacon rounds pass the reward maturity and GET /internal/v1/get_weights (internal token ++ slug header, over HTTP) must return a positive weight for miner 0's hotkey. + +Two more rounds probe restart: miner 0 reruns run-v2 after UPLOADED (the accepted delta and +relay receipts must not change), then is SIGKILLed once its island job is staged (training) +and rerun (must UPLOAD). + +Writes evidence JSON to $MINE_MULTI_ROUND_V2_EVIDENCE (default: a fresh private temp file); +exit 0 = PASS. MINE_SMOKE_V2_BREAK=run-v2 points every run-v2 at a closed port; +MINE_MULTI_ROUND_V2_BREAK=mature asks get_weights before round 0 matures, which freezes that +epoch's answer unpaid (the final check must then FAIL). +""" + +from __future__ import annotations + +import contextlib +import json +import os +import secrets +import signal +import subprocess +import sys +import tempfile +import time +import tomllib +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import mine_smoke_v2 as smoke + +ROUNDS = 2 + + +def capture(held: dict[str, Any]) -> Any: + """The smoke's service module, with its network fixture handing back the live Network.""" + svc = smoke._load_service() + fixture = svc.network.__wrapped__ + + def network(path: Path, request: Any) -> Any: + gen = fixture(path, request) + try: + held["n"] = n = next(gen) + yield n + finally: + gen.close() + + svc.network = SimpleNamespace(__wrapped__=network) + return svc + + +def miner_configs(tmp: Path) -> list[Path]: + """miner.toml from the chain for HOT[0]; same keys, own fixture hotkey, for HOT[1..3].""" + base = tomllib.loads((tmp / "miner.toml").read_text()) + out = [tmp / "miner.toml"] + for i in (1, 2, 3): + key = smoke._write_key(tmp / f"hot-{i}.key", bytes([80 + i]) * 32) + conf = tmp / f"miner-{i}.toml" + conf.write_text( + "\n".join(f"{k} = {json.dumps(str(v))}" for k, v in {**base, "keyfile": key}.items()) + ) + out.append(conf) + return out + + +def run_v2(stages: list[dict[str, Any]], conf: Path, w: int, i: int) -> None: + out = json.loads( + smoke.stage( + stages, + "run-v2", + ["hypertrain-miner", "run-v2", "--config", str(conf), "--round", str(w)], + ) + ) + stages[-1].update(miner=i, w=w, result=out) + if out != {"status": "UPLOADED"}: + raise RuntimeError(f"run-v2 miner {i} round {w}: {out}") + + +def open_round(svc: Any, n: Any, w: int, prev_final: str) -> Any: + """Operator role: signed RoundOpenV2 for w >= 1 from the applied state and restored anchors.""" + import torch + + from hypertrain.aggregator.core import load_state + from hypertrain.auditor.replay import AnchorCache, optimizer_hash, pack_state + from hypertrain.data.store import put_large + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.messages import f32hex + from hypertrain.protocol.messages_v2 import PolicyHashes, RoundOpenV2, StartStateV2 + from hypertrain.trainer.compress import state_hash + + rid = n.manifest.run_id() + applied = n.store._record_v2(rid, "applied", str(w - 1)) + theta = { + k: torch.from_numpy(v.copy()) + for k, v in load_state(n.store.objects, applied["out_state"]).theta.items() + } + cache = AnchorCache() + starts, roster = [], [] + for i, hot in enumerate(svc.HOT): + record = n.store._services(rid)[1].status(hot.ss58, now=n.now).record + anchor = n.store._restore_anchor_v2(rid, hot.ss58, w - 1, cache) + starts.append( + StartStateV2( + run_id=rid, + w=w, + hotkey=hot.ss58, + theta_hash=state_hash(theta), + state_object_sha256=put_large(pack_state(theta, anchor.state), sha256_hex), + opt_state_hash=optimizer_hash(anchor.state), + ef_object_sha256=put_large(pack_state(anchor.ef), sha256_hex), + ef_hash=state_hash(anchor.ef), + parent_anchor_hash=anchor.anchor_hash, + global_step0=anchor.state.step, + anchor_verdict_hash=anchor.proof_hash, + ) + ) + roster.append( + { + "hotkey": hot.ss58, + "slot": i, + "q_i": f32hex(1), + "admission_id": record.admission_id, + "coldkey_group": record.coldkey, + "state": record.state, + "eligible_weight": 4194304, + } + ) + opening = RoundOpenV2( + w=w, + prev_final_hash=prev_final, + theta_hash=state_hash(theta), + outer_state_hash="0" * 64, + center_hash="0" * 64, + roster_hash=sha256_hex(canonicalize(roster)), + honeypot_commit="0" * 64, + d_open=n.now + 1, + d_assign=n.now + 2, + d_commit=n.now + 30, + d_audit=n.now + 31, + d_upload=n.now + 200, + d_final=n.now + 300, + contract_version=2, + policy_hashes=PolicyHashes.model_validate( + {k: getattr(n.manifest.network, k) for k in PolicyHashes.model_fields} + ), + registry_epoch=0, + start_state_index_hash=sha256_hex(canonicalize([s.body() for s in starts])), + audit_mode="anchored-full", + roster=roster, + ) + ok( + n.client.post( + n.url + "/admin/rounds", + json=n.signed(svc.COORD, "RoundOpenV2", opening), + headers=svc.admin(), + ), + f"round {w} open", + ) + n.push(opening.d_assign) + return opening + + +def ok(response: Any, what: str) -> Any: + if response.status_code != 200: + raise RuntimeError(f"{what}: {response.status_code} {response.text[:500]}") + return response.json() + + +def settle(svc: Any, n: Any, w: int) -> str: + """Operator role: 4 anchored audits (MATCH), aggregate, finalize; returns the finalize hash.""" + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.messages import Finalize + + rid = n.manifest.run_id() + n.push(n.store._record_v2(rid, "round", str(w))["body"]["d_audit"]) + ok(n.client.post(n.url + f"/admin/rounds/{w}/audits", headers=svc.admin()), "audits") + for index in range(len(svc.HOT)): + auditor = svc.AUDITORS[index % 2] + + def receipt(nonce: str, auditor: Any = auditor) -> Any: + body = {"w": w, "commit_hash": nonce, "received_round": n.now} + return n.signed(auditor, "Receipt", body) + + job = ok( + n.client.post(n.url + "/worker/lease", json=receipt(f"{w * 4 + index + 1:064x}")), + "lease", + )["body"] + url = n.url + "/worker/jobs/" + job["job_id"] + verdict = ok(n.client.post(url + "/execute", json=receipt(job["lease_nonce"])), "execute") + if verdict["verdict"]["result"] != "MATCH": + raise RuntimeError(f"round {w} audit {index}: {verdict['verdict']['result']}") + signed = n.signed(auditor, "ReplayVerdict", verdict["verdict"]) + ok(n.client.post(url + "/complete", json=signed), "complete") + applied = ok(n.client.post(n.url + f"/admin/rounds/{w}/aggregate", headers=svc.admin()), "agg") + final = Finalize( + w=w, + final_theta_hash_w1=applied["theta_hash"], + included=sorted(k.ss58 for k in svc.HOT), + entitlements_root="0" * 64, + ) + settled = ok( + n.client.post( + n.url + f"/admin/rounds/{w}/finalize", + json=n.signed(svc.COORD, "Finalize", final), + headers=svc.admin(), + ), + "finalize", + ) + if settled.get("status") != "SETTLED": + raise RuntimeError(f"round {w} finalize: {settled.get('status')}") + return sha256_hex(canonicalize(n.store._record_v2(rid, "finalize", str(w))["body"])) + + +def get_weights(svc: Any, n: Any, api: str, tmp: Path) -> dict[str, Any]: + """GET /internal/v1/get_weights over HTTP, as Cortex asks: internal token + slug header.""" + import httpx + + route = next( + r for r in n.client.app.routes if getattr(r, "path", "") == "/internal/v1/get_weights" + ) + code = route.endpoint.__code__ + config = route.endpoint.__closure__[code.co_freevars.index("config")].cell_contents + token = secrets.token_hex(16) # throwaway, never written to the evidence + (tmp / "internal").write_text(token) + config.internal_token_file = tmp / "internal" + epoch_at = int(n.store.clock()) + epoch = (epoch_at - n.store.params.genesis_unix) // n.store.params.epoch_seconds + response = httpx.get( + f"{api}/internal/v1/get_weights", + params={"epoch": epoch, "epoch_at": epoch_at}, + headers={"Authorization": f"Bearer {token}", "x-platform-challenge-slug": config.slug}, + timeout=60, + ) + return {"status_code": response.status_code, "epoch": epoch, "answer": response.json()} + + +def restart_probes( + stages: list[dict[str, Any]], + svc: Any, + n: Any, + configs: list[Path], + prev: str, + tmp: Path, + probe: dict[str, Any], +) -> None: + """Round ROUNDS: miner 0 reruns run-v2 after UPLOADED (a fresh process, as after a kill + between rounds); the accepted delta and relay receipts must not change. Round ROUNDS+1: + miner 0's run-v2 is SIGKILLed once its island job is staged (it trains), then rerun; it + must end UPLOADED (resume).""" + rid, hot = n.manifest.run_id(), svc.HOT[0].ss58 + env = {**os.environ, "OMP_NUM_THREADS": "1", "MKL_NUM_THREADS": "1"} + + def argv(w: int) -> list[str]: + conf = ["--config", str(configs[0]), "--round", str(w)] + return ["uv", "run", "--frozen", "hypertrain-miner", "run-v2", *conf] + + def count(kind: str, identity: str | None = None) -> int: + sql = "SELECT COUNT(*) FROM records_v2 WHERE run_id=? AND kind=?" + args = (rid, kind) if identity is None else (rid, kind, identity) + return n.store._db.execute(sql + (" AND id=?" if identity else ""), args).fetchone()[0] + + w = open_round(svc, n, ROUNDS, prev).w + for i, conf in enumerate(configs): + run_v2(stages, conf, w, i) + delta, receipts = n.store._record_v2(rid, "delta", f"{w}:{hot}"), count("relay-receipt") + replay = subprocess.run(argv(w), env=env, capture_output=True, text=True) # noqa: S603 + stages.append({"name": "run-v2-replay", "mode": "subprocess", "probe": True, "w": w}) + stages[-1].update(rc=replay.returncode, stderr_tail=replay.stderr[-1500:]) + probe["after_upload"] = { + "rc": replay.returncode, + "stdout": replay.stdout.strip()[-200:], + "delta_unchanged": n.store._record_v2(rid, "delta", f"{w}:{hot}") == delta, + "relay_receipts_unchanged": count("relay-receipt") == receipts, + } + if not ( + probe["after_upload"]["delta_unchanged"] + and probe["after_upload"]["relay_receipts_unchanged"] + ): + raise RuntimeError(f"rerun after UPLOADED changed the accepted delta: {probe}") + + w = open_round(svc, n, ROUNDS + 1, settle(svc, n, w)).w + for i, conf in enumerate(configs[1:], 1): + run_v2(stages, conf, w, i) + proc = subprocess.Popen( # noqa: S603 - fixed argv, no shell + argv(w), + env=env, + start_new_session=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + deadline = time.monotonic() + 120 + while ( + not count("island-job", f"{w}:{hot}") + and proc.poll() is None + and time.monotonic() < deadline + ): + time.sleep(0.1) + alive = proc.poll() is None + if alive: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + smoke.reap(str(tmp)) + probe["kill_mid_round"] = { + "killed_after_job_staged": alive and bool(count("island-job", f"{w}:{hot}")) + } + try: + run_v2(stages, configs[0], w, 0) + probe["kill_mid_round"]["resume"] = "UPLOADED" + except RuntimeError as e: + probe["kill_mid_round"]["resume"] = str(e)[:200] + raise RuntimeError(f"resume after SIGKILL mid-round: {e}") from None + finally: + probe["kill_mid_round"]["delta_recorded"] = bool(count("delta", f"{w}:{hot}")) + + +def main() -> int: + ev: dict[str, Any] = {"stages": []} + env = os.environ.get("MINE_MULTI_ROUND_V2_EVIDENCE") + fd = os.open(env, os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600) if env else None + out = Path(env) if env else None + if fd is None: + fd, name = tempfile.mkstemp(prefix="mine-multi-round-v2-evidence-", suffix=".json") + out = Path(name) + started = time.monotonic() + signal.signal(signal.SIGTERM, smoke._budget_exceeded) + held: dict[str, Any] = {} + try: + with tempfile.TemporaryDirectory() as d, contextlib.ExitStack() as stack: + tmp = Path(d) + svc = capture(held) + smoke._load_service = lambda: svc + smoke.chain(tmp, ev, stack) + n, stages = held["n"], ev["stages"] + api = tomllib.loads((tmp / "miner.toml").read_text())["api"] + configs = miner_configs(tmp) + prev_final, rounds = "", [] + for w in range(ROUNDS): + if w: + open_round(svc, n, w, prev_final) + for i, conf in enumerate(configs): + if (w, i) != (0, 0): # the chain already ran miner 0 for round 0 + run_v2(stages, conf, w, i) + prev_final = settle(svc, n, w) + if w == 0 and os.environ.get("MINE_MULTI_ROUND_V2_BREAK") == "mature": + # An epoch answer is final: asked before round 0 matures, it pays nothing. + ev["get_weights_immature"] = get_weights(svc, n, api, tmp) + rounds.append({"w": w, "uploaded": len(configs), "finalize_hash": prev_final}) + ev["rounds"] = rounds + rid = n.manifest.run_id() + mature_at = n.store._db.execute( + "SELECT MAX(mature_at) FROM escrow_origins " + "WHERE reward_round IS NOT NULL AND issued>0" + ).fetchone()[0] + ev["mature_at"], ev["now_before_maturity"] = mature_at, n.now + n.push(max(n.now, mature_at) + 1) + weights = get_weights(svc, n, api, tmp) + ev["get_weights"] = weights + hot0 = svc.HOT[0].ss58 + paid = weights["answer"].get("weights", {}) + ev["miner_hotkey"], ev["miner_weight"] = hot0, paid.get(hot0, 0) + if weights["status_code"] != 200 or not paid.get(hot0, 0) > 0: + raise RuntimeError(f"get_weights: no positive weight for {hot0}: {weights}") + if not set(paid) <= {k.ss58 for k in svc.HOT}: + raise RuntimeError(f"get_weights pays a non-roster hotkey: {sorted(paid)}") + ev["run_id"] = rid + ev["restart"] = {} + restart_probes(stages, svc, n, configs, prev_final, tmp, ev["restart"]) + except Exception as e: # noqa: BLE001 - the evidence must record every failure + ev["failed"] = f"{type(e).__name__}: {e}" + required = [s for s in ev["stages"] if not s.get("probe")] + failure = ev.get("failed") or smoke.verdict(required) + ev["elapsed_seconds"] = round(time.monotonic() - started, 2) + ev["result"] = "PASS" if failure is None else "FAIL" + os.ftruncate(fd, 0) + with os.fdopen(fd, "w") as f: + f.write(json.dumps(ev, indent=1, sort_keys=True, default=str)) + summary = { + "result": ev["result"], + "failed": failure, + "elapsed_seconds": ev["elapsed_seconds"], + "rounds": ev.get("rounds"), + "miner_hotkey": ev.get("miner_hotkey"), + "miner_weight": ev.get("miner_weight"), + "weights": ev.get("get_weights", {}).get("answer", {}).get("weights"), + "restart": ev.get("restart"), + "evidence": str(out), + } + print(json.dumps(summary, sort_keys=True)) + return 0 if failure is None else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/mine_smoke.py b/scripts/mine_smoke.py new file mode 100644 index 0000000..66af12f --- /dev/null +++ b/scripts/mine_smoke.py @@ -0,0 +1,105 @@ +"""CPU mining smoke: real challenge app + one miner + one auditor, one complete test-mode round. + +Unlike `miner_flow_e2e.py` (in-process master, no auditor), this starts the challenge FastAPI app +as its own process, drives a full round through the e2e harness (train -> aggregate -> audit -> +finalize) with the tiny 2-layer CPU model and H/J reduced, and fails unless the round reaches +FINAL with a bitwise MATCH verdict. + +Writes evidence JSON to $MINE_SMOKE_EVIDENCE (default: a fresh private temp file, path printed); +exit 0 = PASS, exit 1 = the round did not complete. +""" + +from __future__ import annotations + +import json +import os +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests" / "e2e")) +sys.path.insert(0, str(ROOT / "tests" / "trainer")) + +import common # type: ignore[import-not-found] # noqa: E402,F401 (determinism before torch) +from common import manifest # noqa: E402 +from harness import World, key # type: ignore[import-not-found] # noqa: E402 + +H, J = 4, 2 +MINER_INDEX = 0x41 + + +def evidence_file() -> tuple[Path, int]: + """$MINE_SMOKE_EVIDENCE (never via a symlink), else a fresh 0600 file, never a fixed name.""" + env = os.environ.get("MINE_SMOKE_EVIDENCE") + if env: + # No O_TRUNC: a failed start must not empty an existing file; main() truncates at write. + fd = os.open(env, os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600) + os.fchmod(fd, 0o600) # O_CREAT keeps the mode of a file that already exists + return Path(env), fd + fd, name = tempfile.mkstemp(prefix="mine-smoke-evidence-", suffix=".json") + return Path(name), fd + + +def main() -> int: + ev: dict[str, Any] = {"checks": {}} + out, fd = evidence_file() + started = time.monotonic() + code = 0 + with tempfile.TemporaryDirectory() as d: + w = None + try: + m = manifest(policy="carry", H=H, J=J, q=1.0, E=1, k_segments=2) + k = key(MINER_INDEX) + w = World(Path(d), m, {k: {"probation": True}}) + miner = w.miner(k) + uploaded = w.run_all([miner], 0) + ev["checks"]["miner_uploaded"] = uploaded == ["UPLOADED"] + assert ev["checks"]["miner_uploaded"], uploaded + w.aggregate(0) + audits = w.audit(0, [miner]) + ev["audits"] = audits + r = w.finalize(0) + ev["finalize"] = {"status": r["status"], "json": r.get("json")} + assert r["status"] == 200, r + verdict = r["json"]["verdicts"][k.ss58] + ev["checks"]["round_finalized"] = True + ev["checks"]["verdict_match"] = verdict == "MATCH" + assert verdict == "MATCH", f"verdict {verdict}" + ev["round_state"] = w.view(0)["state"] + ev["checks"]["state_final"] = ev["round_state"] == "FINAL" + assert ev["checks"]["state_final"], ev["round_state"] + except Exception as e: # noqa: BLE001 - the evidence must record every failure + ev["failed"] = f"{type(e).__name__}: {e}" + code = 1 + finally: + if w is not None: + try: + ev["process_exit_codes"] = w.stop() + except Exception as e: # noqa: BLE001 - a failed cleanup fails the run, not the write + ev["cleanup_failed"] = f"{type(e).__name__}: {e}" + code = 1 + ev["elapsed_seconds"] = round(time.monotonic() - started, 2) + ev["result"] = "PASS" if code == 0 else "FAIL" + os.ftruncate(fd, 0) + with os.fdopen(fd, "w") as f: + f.write(json.dumps(ev, indent=1, sort_keys=True, default=str)) + print( + json.dumps( + { + "result": ev["result"], + "elapsed_seconds": ev["elapsed_seconds"], + "checks": ev["checks"], + "evidence": str(out), + "failed": ev.get("failed"), + }, + sort_keys=True, + ) + ) + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/mine_smoke_v2.py b/scripts/mine_smoke_v2.py new file mode 100644 index 0000000..5367a38 --- /dev/null +++ b/scripts/mine_smoke_v2.py @@ -0,0 +1,569 @@ +"""CPU network-v2 mining smoke: join-v2 -> probe-v2 -> proof-v2 -> run-v2 through the real CLI. + +Starts the real challenge app (the `network` service fixture: tiny CPU model, test ledger, +throwaway fixture keys, fixture beacon) on an ephemeral 127.0.0.1 port, plus a TLS relay on +localhost with a self-signed certificate trusted via SSL_CERT_FILE. One miner then runs every +miner stage as its own `uv run --frozen hypertrain-miner ...` process: + + join-v2 -> escrow-lock -> 4 x (trial_export, probe-v2, proof-v2) -> run-v2 --round 0 + +A round only opens on a roster of four funded eligible miners with at most one PROBATION +(4 clean trials) member, so the three other roster identities are graduated in-process by the +operator role (12 real reference trials each, `"mode": "in-process"` in the evidence). + +Writes evidence JSON to $MINE_SMOKE_V2_EVIDENCE (default: a fresh private temp file, path +printed); exit 0 = PASS (every required stage ran with rc 0), exit 1 = a stage failed, +exit 2 = MINE_SMOKE_V2_BREAK names no breakable stage. +MINE_SMOKE_V2_BREAK= points that stage at a closed port +(failure-detection check). SIGTERM (the shell's budget timeout) is recorded as a failure. +""" + +from __future__ import annotations + +import contextlib +import importlib.util +import json +import os +import secrets +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time +from collections.abc import Iterator +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] + +import hypertrain.trainer # noqa: E402,F401 (determinism before torch) + +REQUIRED = ("join-v2", "probe-v2", "proof-v2", "run-v2") +# PROBATION is round-eligible from 4 clean trials (challenge/admission.py:status). +TRIALS = 4 + + +def evidence_file() -> tuple[Path, int]: + """$MINE_SMOKE_V2_EVIDENCE (never via a symlink), else a fresh 0600 file, never a fixed name.""" + env = os.environ.get("MINE_SMOKE_V2_EVIDENCE") + if env: + # No O_TRUNC: a failed start must not empty an existing file; main() truncates at write. + fd = os.open(env, os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600) + os.fchmod(fd, 0o600) # O_CREAT keeps the mode of a file that already exists + return Path(env), fd + fd, name = tempfile.mkstemp(prefix="mine-smoke-v2-evidence-", suffix=".json") + return Path(name), fd + + +def verdict(stages: list[dict[str, Any]]) -> str | None: + """None = PASS: every stage ran with rc 0 and each required stage ran at least once.""" + for s in stages: + if s.get("skipped") or s.get("rc") is None: + return f"stage {s['name']} skipped" + if s["rc"] != 0: + return f"stage {s['name']} failed rc={s['rc']}" + missing = [n for n in REQUIRED if n not in {s["name"] for s in stages}] + return f"stage(s) not run: {', '.join(missing)}" if missing else None + + +def stage(stages: list[dict[str, Any]], name: str, args: list[str], **extra: Any) -> str: + """Run one CLI stage as its own process; record name/rc/ms; raise unless rc == 0.""" + entry: dict[str, Any] = {"name": name, "mode": "subprocess", "rc": None, **extra} + stages.append(entry) + env = {**os.environ, "OMP_NUM_THREADS": "1", "MKL_NUM_THREADS": "1"} + if name in REQUIRED and os.environ.get("MINE_SMOKE_V2_BREAK") == name: + # Failure-detection check: point this miner stage (it reads miner.toml) at a closed port. + env["HYPERTRAIN_MINER_API"] = "http://127.0.0.1:1" + entry["broken_on_purpose"] = True + started = time.monotonic() + proc = subprocess.run( # noqa: S603 - fixed argv, no shell + ["uv", "run", "--frozen", *args], # noqa: S607 - uv from PATH, as mine-smoke.sh + capture_output=True, + text=True, + env=env, + ) + entry["rc"] = proc.returncode + entry["ms"] = round((time.monotonic() - started) * 1000) + if proc.returncode != 0: + entry["stderr_tail"] = proc.stderr[-2000:] + raise RuntimeError(f"stage {name} failed rc={proc.returncode}") + return proc.stdout + + +def reap(marker: str) -> list[int]: + """SIGKILL every other process whose argv names `marker` (this run's temp dir). + + torchrun islands start in their own session (island_launch.py), so a timed-out stage + leaves them running; their argv carries the attempt directory under the temp dir. + """ + killed = [] + for proc in Path("/proc").iterdir(): + if not proc.name.isdigit() or int(proc.name) == os.getpid(): + continue + try: + argv = (proc / "cmdline").read_bytes().split(b"\0") + except OSError: + continue + if any(marker.encode() in a for a in argv): + with contextlib.suppress(ProcessLookupError): + os.kill(int(proc.name), signal.SIGKILL) + killed.append(int(proc.name)) + return killed + + +def _budget_exceeded(signum: int, frame: Any) -> None: + raise RuntimeError("budget exceeded (SIGTERM)") + + +def _load_service() -> Any: + spec = importlib.util.spec_from_file_location( + "mine_smoke_v2_service", ROOT / "tests/challenge/test_service_network_v2.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +@contextlib.contextmanager +def serve(app: Any, sock: socket.socket, **tls: str) -> Iterator[None]: + import uvicorn + + class Ready(uvicorn.Server): + def __init__(self, config: uvicorn.Config) -> None: + super().__init__(config) + self.ready = threading.Event() + + async def startup(self, sockets: Any = None) -> None: + await super().startup(sockets=sockets) + self.ready.set() + + server = Ready(uvicorn.Config(app, log_level="error", **tls)) + thread = threading.Thread(target=lambda: server.run(sockets=[sock]), daemon=True) + thread.start() + if not server.ready.wait(30): + raise RuntimeError("server did not start") + try: + yield + finally: + server.should_exit = True + thread.join(timeout=10) + + +def _write_key(path: Path, seed: bytes) -> Path: + path.write_bytes(seed) + path.chmod(0o600) + return path + + +def _graduate(svc: Any, n: Any, i: int) -> None: + """Operator role: 12 actual reference trials + signed finality, as _graduate_four does.""" + from hypertrain.auditor.replay import unpack_state + from hypertrain.data.store import get_large + from hypertrain.data.trial_assignment import trial_samples + from hypertrain.miner.island_launch import validate_artifacts + from hypertrain.protocol.messages import Finalize + from hypertrain.protocol.messages_v2 import EscrowLock, JoinChallenge, WorkProof, WorkScreenV2 + from hypertrain.trainer.compress import state_hash + + def ok(response: Any) -> Any: + if response.status_code != 200: + raise RuntimeError(f"peer {i}: {response.status_code} {response.text[:500]}") + return response.json() + + n.push(n.now + 101) # join_burst/quota window, as tests/e2e/_graduate_four + identity = n.join(i).json()["admission_id"] + lock = EscrowLock( + operation_id=f"{i + 1:064x}", + owner=svc.COLD[i].ss58, + units=1000, + origin_ids=[n.origin_ids[i]], + admission_id=identity, + dispute_id=None, + kind="LOCK_ADMISSION", + ) + ok(n.client.post(n.url + "/escrow/lock", json=n.signed(svc.COLD[i], "EscrowLock", lock))) + record: dict[str, Any] = {} + for _ in range(12): + pending = n.store._db.execute( + "SELECT r.outcome FROM admissions_v2 a JOIN admission_trial_results r " + "ON r.epoch=a.trial_epoch WHERE a.admission_id=?", + (identity,), + ).fetchone() + if pending is None or pending["outcome"] != "OPEN": + n.push(n.now + 1) + body = ok(n.client.get(n.url + "/join/" + identity + "/challenge"))["body"] + challenge = JoinChallenge.model_validate(body) + ok(n.client.post(n.url + "/admin/join/" + identity + "/reference", headers=svc.admin())) + row = n.store._db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=?", (identity,) + ).fetchone() + proof = WorkProof.model_validate_json(row["reference_json"]) + samples = trial_samples( + n.manifest, identity, challenge.nonce, n.store._beacon_v2(challenge.seed_beacon) + ) + job, directory = n.store.stage_trial_v2( + n.manifest.run_id(), challenge, tuple(samples), row["trial_epoch"] + ) + artifacts = validate_artifacts(job, directory / "published") + screen = WorkScreenV2( + evidence_kind="work-proof-v1", + admission_id=identity, + nonce=challenge.nonce, + policy_hash=challenge.policy_hash, + image_digest=job.manifest.training.reference_spec.image_digest, + layout=challenge.layout, + challenge_hash=challenge.digest(), + rank_results=list(artifacts.ranks), + artifact_hashes=[r.sha256 for r in proof.artifact_refs], + ) + signed = { + "proof": n.signed(svc.HOT[i], "WorkProof", proof), + "screen": n.signed(svc.HOT[i], "WorkScreenV2", screen), + } + ok(n.client.post(n.url + "/join/proof", json=signed)) + theta, _ = unpack_state(get_large(proof.artifact_refs[0].sha256, n.store.objects.get)) + final = Finalize( + w=row["trial_epoch"], + final_theta_hash_w1=state_hash(theta), + included=[svc.HOT[i].ss58], + entitlements_root=proof.digest(), + ) + record = ok( + n.client.post( + n.url + "/admin/join/" + identity + "/finalize", + json=n.signed(svc.COORD, "Finalize", final), + headers=svc.admin(), + ) + )["record"] + if record.get("state") != "ACTIVE": + raise RuntimeError(f"peer {i} not ACTIVE after 12 trials: {record.get('state')}") + + +def _open_round(svc: Any, n: Any) -> Any: + """Operator role: signed RoundOpenV2 over the four eligible identities, as _live_checkpoint.""" + from hypertrain.auditor.replay import AnchorCache, optimizer_hash, pack_state + from hypertrain.data.store import put_large + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.messages import f32hex + from hypertrain.protocol.messages_v2 import PolicyHashes, RoundOpenV2, StartStateV2 + from hypertrain.trainer.compress import state_hash + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params + + manifest = n.manifest + cache = AnchorCache() + theta = init_params(TrainConfig.from_manifest_v2(manifest).model) + starts, roster = [], [] + for i, hot in enumerate(svc.HOT): + record = n.store._services(manifest.run_id())[1].status(hot.ss58, now=n.now).record + anchor = cache.genesis(manifest, hot.ss58, theta) + starts.append( + StartStateV2( + run_id=manifest.run_id(), + w=0, + hotkey=hot.ss58, + theta_hash=state_hash(theta), + state_object_sha256=put_large(pack_state(theta, anchor.state), sha256_hex), + opt_state_hash=optimizer_hash(anchor.state), + ef_object_sha256=put_large(pack_state(anchor.ef), sha256_hex), + ef_hash=state_hash(anchor.ef), + parent_anchor_hash=anchor.anchor_hash, + global_step0=0, + anchor_verdict_hash=anchor.proof_hash, + ) + ) + roster.append( + { + "hotkey": hot.ss58, + "slot": i, + "q_i": f32hex(1), + "admission_id": record.admission_id, + "coldkey_group": record.coldkey, + "state": record.state, + "eligible_weight": 4194304, + } + ) + policies = manifest.network + opening = RoundOpenV2( + w=0, + prev_final_hash="0" * 64, + theta_hash=state_hash(theta), + outer_state_hash="0" * 64, + center_hash="0" * 64, + roster_hash=sha256_hex(canonicalize(roster)), + honeypot_commit="0" * 64, + d_open=n.now + 1, + d_assign=n.now + 2, + d_commit=n.now + 30, + d_audit=n.now + 31, + d_upload=n.now + 200, + d_final=n.now + 300, + contract_version=2, + policy_hashes=PolicyHashes.model_validate( + {k: getattr(policies, k) for k in PolicyHashes.model_fields} + ), + registry_epoch=0, + start_state_index_hash=sha256_hex(canonicalize([s.body() for s in starts])), + audit_mode="anchored-full", + roster=roster, + ) + response = n.client.post( + n.url + "/admin/rounds", + json=n.signed(svc.COORD, "RoundOpenV2", opening), + headers=svc.admin(), + ) + if response.status_code != 200: + raise RuntimeError(f"round open: {response.status_code} {response.text[:500]}") + n.push(opening.d_assign) + return roster + + +def _relay_app(svc: Any, n: Any, stack: contextlib.ExitStack) -> Any: + import anyio + import httpx + + from hypertrain.data.store import LocalFSStore + from hypertrain.data.stream_store import StreamStore + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.relay_messages import RelayNetworkManifest + from hypertrain.relay.app import create_app + from hypertrain.relay.core import Relay + + rid = n.manifest.run_id() + registry = n.store._registry_v2(n.manifest) + # The manifest store.relay_assignment_v2 signs; its record only exists after run-v2 accepts. + network = RelayNetworkManifest( + run_id=rid, + base_manifest_hash=n.manifest.training.training_hash(), + registry_hash=registry.digest(), + specs_hash=sha256_hex(canonicalize([s.body() for s in registry.specs])), + assignment_policy="master-observed-median3-v1", + ) + backing_http = httpx.AsyncClient() + stack.callback(anyio.run, backing_http.aclose) + + async def now() -> int: + return int(n.now) + + relay = Relay( + run_id=rid, + master=svc.COORD.ss58, + registry=registry, + network_manifest_hash=sha256_hex(canonicalize(network.body())), + observers={a.ss58 for a in svc.AUDITORS}, + relay_id="local", + region="local", + keys={"k1": svc.RELAY}, + active_key="k1", + backing=StreamStore(LocalFSStore(n.store.state_dir / "relay-backing"), backing_http), + now=now, + ) + return create_app(relay, secrets.token_hex(16)) + + +def _tls(directory: Path) -> tuple[Path, Path]: + cert, key = directory / "relay-cert.pem", directory / "relay.key" + # The same self-signed recipe as tests/e2e/test_service_network_v2_e2e.py. + subprocess.run( # noqa: S603 - fixed argv, no shell + [ # noqa: S607 - openssl from PATH + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-keyout", + str(key), + "-out", + str(cert), + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + ], + check=True, + capture_output=True, + ) + return cert, key + + +def chain(tmp: Path, ev: dict[str, Any], stack: contextlib.ExitStack) -> None: + import httpx + + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.relay_messages import RelayRegistryV1 + + stages: list[dict[str, Any]] = ev["stages"] + stack.callback(reap, str(tmp)) # last out: after both servers and the fixture stopped + cert, tls_key = _tls(tmp) + # The miner processes and the master's relay-receipt call trust only this localhost CA. + os.environ["SSL_CERT_FILE"] = str(cert) + relay_sock, master_sock = socket.socket(), socket.socket() + for sock in (relay_sock, master_sock): + sock.bind(("127.0.0.1", 0)) + stack.callback(sock.close) + relay_url = f"https://localhost:{relay_sock.getsockname()[1]}" + + svc = _load_service() + + class LocalRegistry: + """The fixture's relay registry, pointed at this run's TLS relay before signing.""" + + @staticmethod + def model_validate(body: dict[str, Any]) -> RelayRegistryV1: + body["specs"][0]["https_url"] = relay_url + return RelayRegistryV1.model_validate(body) + + svc.RelayRegistryV1 = LocalRegistry + (tmp / "service").mkdir() + fixture = svc.network.__wrapped__(tmp / "service", SimpleNamespace(param=None)) + n = next(fixture) + stack.callback(fixture.close) + stack.enter_context(serve(n.client.app, master_sock)) + api = f"http://127.0.0.1:{master_sock.getsockname()[1]}" + rid = n.manifest.run_id() + ev["run_id"] = rid + + # Throwaway fixture test seeds (the only hotkey/coldkey the fake metagraph and test genesis + # know), written 0600 inside the temp dir; never in the evidence. + hot = _write_key(tmp / "hot.key", bytes([80]) * 32) + cold = _write_key(tmp / "cold.key", bytes([90]) * 32) + config = tmp / "miner.toml" + config.write_text( + "\n".join( + f"{k} = {json.dumps(str(v))}" + for k, v in { + "api": api, + "keyfile": hot, + "workdir": tmp / "work", + "state_source": "cpu", + "image_digest": n.manifest.training.reference_spec.image_digest, + "run_id": rid, + "owner_hotkey": svc.OWNER.ss58, + "device": "cpu", + }.items() + ) + ) + miner = ["hypertrain-miner"] + conf = ["--config", str(config)] + joined = json.loads( + stage( + stages, + "join-v2", + [*miner, "join-v2", *conf, "--coldkey", str(cold)] + + ["--request-id", sha256_hex(b"mine-smoke-v2"), "--expiry", "10000"], + ) + ) + admission = joined["admission_id"] + stage( + stages, + "escrow-lock", + [*miner, "escrow-lock", "--api", api, "--run-id", rid, "--coldkey", str(cold)] + + ["--hotkey-ss58", svc.HOT[0].ss58, "--units", "1000", "--origin", n.origin_ids[0]], + role="operator", + ) + with httpx.Client(base_url=api, timeout=60) as http: + for trial in range(1, TRIALS + 1): + n.push(n.now + 1) + # The challenge GET issues the trial; export reads the stored challenge. + http.get(f"/v2/runs/{rid}/join/{admission}/challenge").raise_for_status() + out = tmp / f"trial-{trial}" + stage( + stages, + "trial-export", + ["python", "-m", "hypertrain.challenge.trial_export", "--api", api] + + ["--run-id", rid, "--admission-id", admission] + + ["--admin-token-file", str(tmp / "service/admin"), "--out", str(out)], + role="operator", + trial=trial, + ) + proof = tmp / f"proof-{trial}.json" + proof.write_text( + stage( + stages, + "probe-v2", + [*miner, "probe-v2", *conf, "--job", str(out / "job.json")] + + ["--challenge", str(out / "challenge.json")], + trial=trial, + ) + ) + record = json.loads( + stage(stages, "proof-v2", [*miner, "proof-v2", *conf, "--proof", str(proof)]) + ) + stages[-1]["trial"] = trial + stages[-1]["record"] = {k: record.get(k) for k in ("state", "clean_count")} + if record.get("clean_count") != trial: + raise RuntimeError( + f"proof-v2 trial {trial}: clean_count {record.get('clean_count')}" + ) + + started = time.monotonic() + stages.append({"name": "graduate-peers", "mode": "in-process", "role": "operator", "rc": None}) + for i in (1, 2, 3): + _graduate(svc, n, i) + ev["roster"] = [{"hotkey": r["hotkey"], "state": r["state"]} for r in _open_round(svc, n)] + stages[-1].update(rc=0, ms=round((time.monotonic() - started) * 1000)) + + stack.enter_context( + serve( + _relay_app(svc, n, stack), relay_sock, ssl_certfile=str(cert), ssl_keyfile=str(tls_key) + ) + ) + result = json.loads(stage(stages, "run-v2", [*miner, "run-v2", *conf, "--round", "0"])) + delta = n.store._record_v2(rid, "delta", "0:" + svc.HOT[0].ss58) + ev["round"] = {"result": result, "delta_recorded": delta is not None} + if result != {"status": "UPLOADED"} or delta is None: + raise RuntimeError(f"run-v2: {result}, delta recorded: {delta is not None}") + + +def main() -> int: + broken = os.environ.get("MINE_SMOKE_V2_BREAK") + if broken and broken not in REQUIRED: + # A typo would otherwise run a green smoke and pass the failure-detection check. + print( + f"MINE_SMOKE_V2_BREAK={broken!r} is not a breakable stage: {', '.join(REQUIRED)}", + file=sys.stderr, + ) + return 2 + ev: dict[str, Any] = {"stages": []} + out, fd = evidence_file() + started = time.monotonic() + # The shell's `timeout` sends SIGTERM: unwind so cleanup runs and the evidence is written. + signal.signal(signal.SIGTERM, _budget_exceeded) + try: + with tempfile.TemporaryDirectory() as d, contextlib.ExitStack() as stack: + chain(Path(d), ev, stack) + except Exception as e: # noqa: BLE001 - the evidence must record every failure + ev["failed"] = f"{type(e).__name__}: {e}" + failure = ev.get("failed") or verdict(ev["stages"]) + ev["elapsed_seconds"] = round(time.monotonic() - started, 2) + ev["result"] = "PASS" if failure is None else "FAIL" + os.ftruncate(fd, 0) + with os.fdopen(fd, "w") as f: + f.write(json.dumps(ev, indent=1, sort_keys=True, default=str)) + print( + json.dumps( + { + "result": ev["result"], + "elapsed_seconds": ev["elapsed_seconds"], + "stages": [(s["name"], s.get("rc"), s.get("ms")) for s in ev["stages"]], + "evidence": str(out), + "failed": failure, + }, + sort_keys=True, + ) + ) + return 0 if failure is None else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_mfu_dist.py b/scripts/run_mfu_dist.py index 6db05bc..1aced42 100644 --- a/scripts/run_mfu_dist.py +++ b/scripts/run_mfu_dist.py @@ -34,7 +34,7 @@ from run_mfu import PROFILES, manifest_body, summarize # isort: skip -from hypertrain.trainer.attention import select_backend +from hypertrain.trainer.attention import chosen_paths, select_backend from hypertrain.trainer.island import DistComm, check_launch, island_from_manifest, train_island from hypertrain.trainer.loop import Assignment from hypertrain.trainer.model import init_params @@ -191,6 +191,7 @@ def after_step(t: int, _theta: Any) -> None: "n_gpus": lay.n_gpus, "dp_size": lay.dp_size, "attention_backend": select_backend(device), + "attention_paths": chosen_paths(), **({"trace": [r["trace"] for r in ranks]} if prof is not None else {}), } diff --git a/scripts/vast_e2e.py b/scripts/vast_e2e.py index 3bd12ae..619ed84 100644 --- a/scripts/vast_e2e.py +++ b/scripts/vast_e2e.py @@ -3,16 +3,19 @@ Flow: credit check -> offer search -> worst-case cost admission -> ONE create -> ssh ready -> upload current tree -> scripts/vast_e2e_remote.sh -> rescue /workspace/out -> DELETE -> parsed absence. A watchdog thread fires at the wall deadline (minus the cleanup reserve) and runs the -same rescue+delete. Every provider call is journaled. `--dry-run` stops after the cost calc. +same rescue+delete. Every provider call is journaled. `--dry-run` stops after the cost calc: it +makes read-only GET calls (users/current, up to OFFER_PAGES offer pages), never a PUT/DELETE. The API key is read from --key-file and never printed or journaled. """ from __future__ import annotations import argparse +import importlib.util import json import os import shlex +import signal import subprocess import sys import threading @@ -30,15 +33,14 @@ "ghcr.io/cortexlm/hypertrain-gpu@sha256:" "c16a28dd29df300eca0cfb5c5aeacb8947681708634e3cbc8639b530321fe2e2" ) -EVIDENCE = Path( - "/root/distributed-decision-training/.omo/evidence/hypertrain-complete-20261010/vast" -) ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / ".evidence" / "vast" # = --evidence-dir default; siblings use EVIDENCE.parent UPLOAD = ["src", "tests", "pyproject.toml", "uv.lock", "scripts", "README.md", "LICENSE"] DISK_GB = 40 # 142018: GPUs at 100% util by a hidden tenant; 146840/141607: image pull >15 min (2026-10-12) EXCLUDED_MACHINES = (137465, 43459, 142018, 146840, 141607) HOURS_PER_MONTH = Decimal(720) +OFFER_PAGES = 8 # offer-search pages (of offer_query()["limit"]) before admission refuses class Refused(Exception): @@ -146,10 +148,39 @@ def dec(value: Any) -> Decimal | None: "450710d89268fcd55632f28fdd80508ee920a53075b3f8e9f828bdc154e3d856" ) NUM_GPUS = 2 -# mfu profile: dp all_gathers cross PCIe (a pcie_bw ~26 host gave 11% MFU vs 37% at 53.8), and -# the cuda-13.2 image needs cuda_max_good >= 13.2 (a 13.0 host answered create with HTTP 400). +# mfu profile: dp all_gathers cross PCIe (a pcie_bw ~26 host gave 11% MFU vs 37% at 53.8). +# Every image is cuda-13.2: cuda_max_good >= 13.2 (a 13.0 host answered create with HTTP 400). MIN_PCIE_BW: Decimal | None = None MIN_CUDA = Decimal("13.2") +# Bitwise replay is only claimed on drivers qualified by genuine CUDA CI (single source of truth). +_ci_spec = importlib.util.spec_from_file_location( + "genuine_cuda_ci", Path(__file__).with_name("genuine_cuda_ci.py") +) +assert _ci_spec and _ci_spec.loader +_ci = importlib.util.module_from_spec(_ci_spec) +_ci_spec.loader.exec_module(_ci) +QUALIFIED_DRIVERS: tuple[str, ...] = tuple(_ci.DRIVERS) +ALLOWED_DRIVERS: tuple[str, ...] | None = QUALIFIED_DRIVERS # None: --any-driver + + +# --profile proofs: every GPU proof on ONE host, in this order (steps.tsv rows). The first four +# are the gpu-e2e flow (uv_install only if pip fails); PROOF_STEPS are required to pass. +PROOF_STEPS = ("attn_select", "gpu_pytest", "mfu_dp2_flash", "mfu_dp2_check") +PROOF_PLAN = ( + ("nvidia_smi", "nvidia-smi"), + ("pip_install", "pip install --no-deps -e . (uv_install fallback)"), + ("selfcheck", "python -m hypertrain.miner.admission selfcheck"), + ("network_e2e", "join-v2, 2 chained rounds, auditor replay, repeat; HT_ATTN_BACKEND=math"), + ("attn_select", "HT_ATTN_BACKEND unset: select_backend(cuda:0) == sdpa-flash"), + ("gpu_pytest", "HT_ATTN_BACKEND unset, HT_GENUINE_CUDA_INDEX=0, any skip fails: PROOF_TESTS"), + ("mfu_dp2_flash", "HT_ATTN_BACKEND unset: torchrun 2 ranks, 1B dp2 bf16, run + replay"), + ("mfu_dp2_check", "check_gpu_evidence --kind mfu; attention_backend == sdpa-flash"), +) +PROOF_TESTS = ( + "tests/trainer/test_attention_qualified.py::test_qualified_backend_repeats_bitwise", + "tests/trainer/test_attention.py::test_backward_bitwise", + "tests/trainer/test_optim_fused.py::test_step_has_no_host_sync", +) def offer_query() -> dict[str, Any]: @@ -162,6 +193,8 @@ def offer_query() -> dict[str, Any]: "reliability": {"gt": 0.98}, "inet_down": {"gte": 1000}, **({"pcie_bw": {"gte": float(MIN_PCIE_BW)}} if MIN_PCIE_BW is not None else {}), + # No driver_version filter: the API answers 400 "driver_vers in None" (2026-10-19); + # eligible() applies ALLOWED_DRIVERS to the returned offers. "type": "on-demand", "order": [["dph_total", "asc"]], "limit": 64, @@ -183,12 +216,10 @@ def eligible(o: Any) -> bool: and r > Decimal("0.98") and (d := dec(o.get("inet_down"))) is not None and d >= 1000 - and ( - MIN_PCIE_BW is None - or ((b := dec(o.get("pcie_bw"))) is not None and b >= MIN_PCIE_BW) - and (c := dec(o.get("cuda_max_good"))) is not None - and c >= MIN_CUDA - ) + and (MIN_PCIE_BW is None or ((b := dec(o.get("pcie_bw"))) is not None and b >= MIN_PCIE_BW)) + and (c := dec(o.get("cuda_max_good"))) is not None + and c >= MIN_CUDA + and (ALLOWED_DRIVERS is None or o.get("driver_version") in ALLOWED_DRIVERS) and dec(o.get("dph_total")) is not None and dec(o.get("storage_cost")) is not None ) @@ -217,11 +248,14 @@ def list_rows(r: Response) -> list[dict[str, Any]] | None: class Harness: # Profile hooks (subclasses such as scripts/vast_parity.py override these; defaults = e2e). label_prefix = "hypertrain-e2e" - offer_desc = "2x RTX 5090" disk_gb = DISK_GB upload_paths: list[str] = UPLOAD rsync_excludes: list[str] = [] + @property + def offer_desc(self) -> str: + return f"{NUM_GPUS}x RTX 5090" + def offer_query(self) -> dict[str, Any]: return offer_query() @@ -237,6 +271,7 @@ def __init__(self, args: argparse.Namespace, api: Api) -> None: self.label = f"{self.label_prefix}-{uuid.uuid4().hex[:8]}" self.instance_id: int | None = None self.create_attempted = False + self.retired: list[tuple[int, str]] = [] # stuck hosts boot() deleted: (id, label) self.ssh: tuple[str, int] | None = None self.cancel = threading.Event() self.cleanup_lock = threading.Lock() @@ -254,14 +289,48 @@ def admit(self) -> dict[str, Any]: self.api.log("credit", credit=str(credit), required=str(need)) if credit < need: raise Refused(f"credit {credit} < max_usd + 5 = {need}") - q = urllib.parse.urlencode({"q": json.dumps(self.offer_query())}) - r = self.api.call("GET", "/api/v0/bundles/?" + q) - offers = r.body.get("offers") if r.ok() else None - if not isinstance(offers, list): - raise Refused(f"offer search failed (HTTP {r.status})") - fit = [o for o in offers if self.eligible(o)] + # Drivers are filtered client-side (the API refuses a driver filter), so the cheapest + # page may hold no qualified offer: page on by price until one is eligible. + query, offers, seen, why = self.offer_query(), [], set(), "" + skip = set(self.a.exclude_offer) # here, not in eligible(): profiles override that + for page in range(OFFER_PAGES): + q = urllib.parse.urlencode({"q": json.dumps(query)}) + r = self.api.call("GET", "/api/v0/bundles/?" + q) + got = r.body.get("offers") if r.ok() else None + if not isinstance(got, list): + raise Refused(f"offer search failed (HTTP {r.status})") + new = [o for o in got if not (isinstance(o, dict) and o.get("id") in seen)] + seen |= {o.get("id") for o in got if isinstance(o, dict)} + offers += new + pool = [o for o in offers if not isinstance(o, dict) or o.get("id") not in skip] + fit = [o for o in pool if self.eligible(o)] + prices = [p for o in got if isinstance(o, dict) and (p := dec(o.get("dph_total")))] + limit = query.get("limit") + if fit or not prices or limit is None or len(got) < limit: + break + if not new: # a whole page at one price: the gte cursor cannot advance + why = f"; search stalled: {limit} offers at {max(prices)} USD/h fill a page" + break + if page == OFFER_PAGES - 1: + why = f"; search stopped after {OFFER_PAGES} pages of {limit}: " + why += "cheaper-first market not exhausted" + # gte, not gt: offers at the boundary price may continue on the next page. + # An existing dph_total bound (lte, or a higher gte) is kept, only the floor moves. + bound = query.get("dph_total", {}) + floor = max(float(max(prices)), bound.get("gte", 0.0)) + query = {**query, "dph_total": {**bound, "gte": floor}} if not fit: - raise Refused(f"no eligible {self.offer_desc} offer among {len(offers)}") + alone = 0 # offers a qualified driver would have made eligible + if ALLOWED_DRIVERS is not None: + alone = sum( + isinstance(o, dict) + and any(self.eligible(o | {"driver_version": v}) for v in ALLOWED_DRIVERS) + for o in pool + ) + raise Refused( + f"no eligible {self.offer_desc} offer among {len(offers)} " + f"({alone} rejected by driver allow-list){why}" + ) best = min(fit, key=self.rank) cost = worst_case(best, float(self.a.max_minutes), self.disk_gb) quote = { @@ -285,6 +354,9 @@ def admit(self) -> dict[str, Any]: ) } quote["worst_case_usd"] = str(cost.quantize(Decimal("0.0001"))) + # Bitwise replay is claimed only on these drivers; --any-driver runs must say so. + quote["driver_qualified"] = best.get("driver_version") in QUALIFIED_DRIVERS + self.quote = quote self.api.log("offer_chosen", seen=len(offers), eligible=len(fit), quote=quote) print( json.dumps( @@ -304,7 +376,11 @@ def admit(self) -> dict[str, Any]: return best def create(self, offer: dict[str, Any]) -> None: + if self.cancel.is_set(): + raise RuntimeError("cancelled before create") self.api.log("create_intent", offer_id=offer["id"], label=self.label, image=IMAGE) + # Per rented offer: boot() may fall back to an offer with another driver. + self.result["driver_qualified"] = offer.get("driver_version") in QUALIFIED_DRIVERS self.create_attempted = True r = self.api.call( "PUT", @@ -346,7 +422,7 @@ def find_by_label(self) -> int | None: def boot(self, offer: dict[str, Any]) -> None: """Create + wait for ssh; a vanished offer or a stuck host moves to the next offer.""" - tried: set[int] = set() + tried: set[int | None] = set() for attempt in range(1 + int(self.a.boot_retries)): tried.add(offer.get("machine_id")) try: @@ -369,6 +445,8 @@ def boot(self, offer: dict[str, Any]) -> None: self.api.log("boot_timeout", attempt=attempt, instance_id=self.instance_id) if not (self.delete() and self.confirm_absent()): raise RuntimeError("stuck host not confirmed absent") + assert self.instance_id is not None + self.retired.append((self.instance_id, self.label)) self.instance_id, self.create_attempted = None, False self.label = f"{self.label_prefix}-{uuid.uuid4().hex[:8]}" self.result["label"] = self.label @@ -383,11 +461,17 @@ def boot(self, offer: dict[str, Any]) -> None: def wait_ready(self) -> None: key = self.identity().with_suffix(".pub").read_text().strip() - attached, last_error = False, None + attached, last_error, last_state = False, None, None while time.time() < self.deadline and not self.cancel.is_set(): rows = self.instances() row = next((i for i in rows or [] if i["id"] == self.instance_id), None) if row is not None: + state = (row.get("actual_status"), row.get("cur_state"), row.get("status_msg")) + if state != last_state: # why a host never gets ssh-ready (image pull, no GPU) + self.api.log( + "boot_state", actual=state[0], cur=state[1], msg=str(state[2])[:200] + ) + last_state = state if ( row.get("actual_status") in ("exited", "offline") or row.get("cur_state") == "stopped" @@ -485,6 +569,8 @@ def sh( stdout=sink, stderr=log, ) + if self.cancel.is_set() and not self.cleaned: # signal landed before self.child + self.child.kill() try: self.child.communicate(stdin, timeout=max(1.0, timeout)) except subprocess.TimeoutExpired: @@ -571,10 +657,13 @@ def cleanup(self, reason: str) -> None: if self.instance_id is None and self.create_attempted: self.instance_id = self.find_by_label() if self.instance_id is None and not self.create_attempted: - self.result["absent"] = True + self.result["absent"] = self.retired_absent() + if self.retired: # every host was stuck: still end on a parsed absence + self.result["deleted"] = True + self.api.log("cleanup_done", deleted=True, absent=self.result["absent"]) return self.result["deleted"] = self.delete() - self.result["absent"] = self.confirm_absent() + self.result["absent"] = self.confirm_absent() and self.retired_absent() self.api.log( "cleanup_done", deleted=self.result["deleted"], absent=self.result["absent"] ) @@ -590,16 +679,43 @@ def delete(self) -> bool: self.api.sleep(min(60.0, 5.0 * 2**attempt)) return False - def confirm_absent(self) -> bool: + def confirm_absent(self, iid: int | None = None, label: str | None = None) -> bool: + iid, label = iid or self.instance_id, label or self.label + q = urllib.parse.urlencode({"select_filters": json.dumps({"label": {"eq": label}})}) for _ in range(int(self.a.absence_attempts)): - rows = self.instances() + rows = list_rows(self.api.call("GET", "/api/v1/instances/?" + q)) if rows is not None and not any( - i["id"] == self.instance_id or i.get("label") == self.label for i in rows + i["id"] == iid or i.get("label") == label for i in rows ): return True self.api.sleep(max(5.0, self.api.min_gap)) return False + def retired_absent(self) -> bool: + """Re-list each host boot() deleted: its absence was seen once, the run proves it last.""" + return all([self.confirm_absent(i, label) for i, label in self.retired]) # list: check all + + def plan(self) -> dict[str, Any]: + return { + "image": IMAGE, + "remote_script": self.a.remote_script, + "remote_env": [kv.partition("=")[0] + "=***" for kv in self.a.remote_env], + "steps": [f"{name}: {what}" for name, what in PROOF_PLAN], + "tests": list(PROOF_TESTS), + "caps": { + "max_usd": self.a.max_usd, + "max_minutes": self.a.max_minutes, + "cleanup_reserve_s": self.a.cleanup_reserve, + "boot_timeout_s": self.a.boot_timeout, + "boot_retries": self.a.boot_retries, + "instances": 1, + "num_gpus": NUM_GPUS, + "drivers": "any" if ALLOWED_DRIVERS is None else list(ALLOWED_DRIVERS), + }, + "worst_case_usd": self.quote["worst_case_usd"], + "driver_qualified": self.quote["driver_qualified"], + } + def watchdog(self) -> None: if not self.cancel.wait(max(0.0, self.deadline - time.time())): self.api.log("watchdog_fired", deadline=self.deadline) @@ -609,6 +725,8 @@ def watchdog(self) -> None: def run(self) -> int: offer = self.admit() if self.a.dry_run: + if self.a.profile == "proofs": + print(json.dumps(self.plan(), sort_keys=True)) return 0 dog = threading.Thread(target=self.watchdog, daemon=True) dog.start() @@ -626,22 +744,27 @@ def run(self) -> int: finally: self.cleanup("finally") dog.join(timeout=5) - self.result["remote_ok"] = rc == 0 and not self.result.get("watchdog") + self.result["remote_ok"] = rc == 0 and not ({"watchdog", "signal"} & self.result.keys()) (self.out / "harness-result.json").write_text(json.dumps(self.result, indent=2)) if not self.result.get("absent"): print(f"ALERT: instance {self.instance_id} absence NOT confirmed", file=sys.stderr) return 3 - return 0 if self.result["remote_ok"] else 1 + if not self.result["remote_ok"]: + return 1 + if self.a.profile == "proofs" and not self.result.get("driver_qualified"): + print("proofs passed on a driver outside QUALIFIED_DRIVERS: exit 4", file=sys.stderr) + return 4 # ran fine, but a bitwise-replay claim is not backed by a qualified driver + return 0 def parse(argv: list[str] | None = None) -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__) + p = argparse.ArgumentParser(description=__doc__, allow_abbrev=False) p.add_argument("--key-file", type=Path, required=True) p.add_argument("--max-usd", type=float, required=True) p.add_argument("--max-minutes", type=float, required=True) p.add_argument("--dry-run", action="store_true") p.add_argument("--api-base", default="https://console.vast.ai") - p.add_argument("--evidence-dir", type=Path, default=EVIDENCE) + p.add_argument("--evidence-dir", type=Path, default=ROOT / ".evidence" / "vast") p.add_argument( "--cleanup-reserve", type=float, @@ -655,25 +778,86 @@ def parse(argv: list[str] | None = None) -> argparse.Namespace: default=900.0, help="seconds a host may take to become ssh-ready before it is replaced", ) - p.add_argument("--boot-retries", type=int, default=2, help="replacement hosts to try") + p.add_argument( + "--boot-retries", type=int, help="replacement hosts to try (default 2; proofs: 0)" + ) + p.add_argument( + "--exclude-offer", + type=int, + action="append", + default=[], + metavar="ID", + help="never rent this offer id (repeatable), e.g. a host whose ssh never accepted the key", + ) p.add_argument("--delete-attempts", type=int, default=6) p.add_argument("--absence-attempts", type=int, default=12) p.add_argument( "--profile", - choices=("gpu-e2e", "miner-flow", "mfu"), + choices=("gpu-e2e", "miner-flow", "mfu", "miner-cycle", "gpu8", "proofs"), default="gpu-e2e", - help="miner-flow: 1x RTX 5090, vastai/pytorch image, scripts/vast_miner_flow_remote.sh; " + help="gpu-e2e: --num-gpus x RTX 5090, pinned GPU image, scripts/vast_e2e_remote.sh; " + "gpu8: gpu-e2e on 8x RTX 5090; " + "proofs: gpu-e2e + PROOF_STEPS (GPU tests, production sdpa-flash attention) on one " + "2x RTX 5090 host, any driver (--eight-gpu-qualified: 8x, qualified drivers only); " + "miner-flow: 1x RTX 5090, vastai/pytorch image, scripts/vast_miner_flow_remote.sh; " + "miner-cycle: --num-gpus (default 1) x RTX 5090, vastai/pytorch image, " + "scripts/vast_miner_cycle_remote.sh; " "mfu: --num-gpus x RTX 5090, vastai/pytorch image, pcie_bw >= --min-pcie-bw, " "scripts/vast_mfu_remote.sh", ) - p.add_argument("--num-gpus", type=int, choices=(1, 2, 4, 8), default=2, help="mfu profile") + p.add_argument( + "--num-gpus", + type=int, + choices=(1, 2, 4, 8), + help="GPUs per host, every profile; default 2 (miner-flow, miner-cycle: 1; gpu8: 8)", + ) + drivers = p.add_mutually_exclusive_group() + drivers.add_argument( + "--driver", + action="append", + metavar="VERSION", + help="only this driver_version (repeatable); must be in scripts/genuine_cuda_ci.py DRIVERS", + ) + drivers.add_argument( + "--any-driver", + action="store_true", + help="no driver_version filter; for --profile mfu, where bitwise replay is not compared", + ) + p.add_argument( + "--eight-gpu-qualified", + action="store_true", + help="proofs profile: rent 8x RTX 5090, only with a driver in QUALIFIED_DRIVERS", + ) p.add_argument("--min-pcie-bw", type=float, default=50.0, help="mfu profile, GB/s") p.add_argument("--remote-env", action="append", default=[], metavar="KEY=VAL") a = p.parse_args(argv) a.remote_script = { "miner-flow": "vast_miner_flow_remote.sh", "mfu": "vast_mfu_remote.sh", + "miner-cycle": "vast_miner_cycle_remote.sh", }.get(a.profile, "vast_e2e_remote.sh") + gpus = {"miner-flow": 1, "miner-cycle": 1, "gpu8": 8}.get(a.profile, 2) + if a.eight_gpu_qualified: + if a.profile != "proofs" or a.any_driver: + p.error("--eight-gpu-qualified needs --profile proofs and excludes --any-driver") + gpus = 8 + if a.profile == "proofs": + a.any_driver = a.any_driver or not (a.eight_gpu_qualified or a.driver) + if a.boot_retries is None: + a.boot_retries = 0 if a.profile == "proofs" else 2 + if a.profile in ("miner-flow", "gpu8", "proofs") and a.num_gpus not in (None, gpus): + p.error(f"--profile {a.profile} supports only --num-gpus {gpus}") + a.num_gpus = a.num_gpus or gpus + if a.remote_script == "vast_e2e_remote.sh": + # Appended: env(1) keeps the last KEY=VAL, so --remote-env cannot contradict the host. + a.remote_env = [*a.remote_env, f"HT_E2E_GPUS={a.num_gpus}"] + if a.profile == "proofs": + a.remote_env += [ + f"HT_E2E_PROOFS={' '.join(PROOF_STEPS)}", + f"HT_E2E_PROOF_TESTS={' '.join(PROOF_TESTS)}", + ] + if unknown := sorted(set(a.driver or ()) - set(QUALIFIED_DRIVERS)): + p.error(f"--driver {', '.join(unknown)} not in scripts/genuine_cuda_ci.py DRIVERS") if any("=" not in kv or not kv.partition("=")[0].isidentifier() for kv in a.remote_env): p.error("--remote-env must be KEY=VAL with an identifier KEY") if ( @@ -686,6 +870,17 @@ def parse(argv: list[str] | None = None) -> argparse.Namespace: return a +def _mask_remote_env(argv, remote_env) -> str: + """argv for the journal with every --remote-env VALUE hidden (KEY=***), any spelling.""" + out = [] + for tok in argv: + for kv in sorted(remote_env, key=len, reverse=True): # TOK=ab must not leave b + if kv in tok: + tok = tok.replace(kv, kv.partition("=")[0] + "=***") + out.append(tok) + return shlex.join(out) + + def main( argv: list[str] | None = None, harness: type[Harness] = Harness, @@ -697,27 +892,43 @@ def main( if not key: print("empty key file", file=sys.stderr) return 2 - global IMAGE, NUM_GPUS, MIN_PCIE_BW - if a.profile == "miner-flow": - IMAGE, NUM_GPUS = MINER_FLOW_IMAGE, 1 + global IMAGE, NUM_GPUS, MIN_PCIE_BW, ALLOWED_DRIVERS + NUM_GPUS = a.num_gpus + ALLOWED_DRIVERS = None if a.any_driver else tuple(a.driver or QUALIFIED_DRIVERS) + if a.profile in ("miner-flow", "miner-cycle", "mfu"): + IMAGE = MINER_FLOW_IMAGE if a.profile == "mfu": - IMAGE, NUM_GPUS = MINER_FLOW_IMAGE, a.num_gpus MIN_PCIE_BW = Decimal(str(a.min_pcie_bw)) api = Api(a.api_base, key, a.evidence_dir / "journal.jsonl", min_gap=a.min_gap, sleep=sleep) + h = harness(a, api) + + def stop(signum: int, _frame: Any) -> None: + # A service stop/restart: cancel like the watchdog does, never raise, so a signal can + # not abort a rescue/delete under way. No api.log: this thread may hold its lock. + h.result["signal"] = signum + h.cancel.set() + child = h.child + if not h.cleaned and child is not None and child.poll() is None: + child.kill() # once cleanup runs, the child is the rescue: let it finish + + saved = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGHUP)} api.log( "start", dry_run=a.dry_run, max_usd=a.max_usd, max_minutes=a.max_minutes, - argv=shlex.join(sys.argv[1:] if argv is None else argv), + argv=_mask_remote_env(sys.argv[1:] if argv is None else argv, a.remote_env), pid=os.getpid(), ) try: - return harness(a, api).run() + return h.run() except Refused as e: api.log("refused", reason=str(e)) print(f"REFUSED: {e}", file=sys.stderr) return 2 + finally: + for s, old in saved.items(): + signal.signal(s, old) if __name__ == "__main__": diff --git a/scripts/vast_e2e_remote.sh b/scripts/vast_e2e_remote.sh index 4c76fdb..735f161 100755 --- a/scripts/vast_e2e_remote.sh +++ b/scripts/vast_e2e_remote.sh @@ -9,12 +9,18 @@ mkdir -p "$OUT" cd "$HT" export CUBLAS_WORKSPACE_CONFIG=:4096:8 CUDA_DISABLE_PTX_JIT=1 OMP_NUM_THREADS=1 PYTHONHASHSEED=0 export HT_IMAGE_DIGEST=sha256:c16a28dd29df300eca0cfb5c5aeacb8947681708634e3cbc8639b530321fe2e2 +# torchrun reads PET_: tee every rank's stdout/stderr into $OUT, which is rescued. The +# launcher's own stderr tail keeps only torchrun's summary, not the failing rank's traceback. +export PET_LOG_DIR="$OUT/torchelastic" PET_TEE=3 +# The fixture model is fp32 with head_dim 4: the sdpa-flash kernel that select_backend qualifies +# for sm_120 rejects both ("No available kernel"). Miner, auditor and repeat all inherit this. +export HT_ATTN_BACKEND=${HT_ATTN_BACKEND:-math} step() { - local name=$1; shift + local name=$1 t0=$SECONDS; shift "$@" >"$OUT/$name.log" 2>&1 local rc=$? - printf '%s\t%s\n' "$name" "$rc" >>"$OUT/steps.tsv" + printf '%s\t%s\t%s\n' "$name" "$rc" "$((SECONDS - t0))" >>"$OUT/steps.tsv" return $rc } : >"$OUT/steps.tsv" @@ -42,8 +48,9 @@ if [ "$BACKEND" = cuda ]; then if "$PY" -c "import hypertrain.miner.admission as a; assert hasattr(a, 'main')" 2>/dev/null; then step selfcheck "$PY" -m hypertrain.miner.admission selfcheck fi - grep -rlE "skipif\([^)]*cuda|requires_cuda|mark\.cuda|cuda\.is_available\(\)" tests \ - --include='test_*.py' | sort >"$OUT/cuda_test_files.txt" || true + # --profile proofs runs its explicit list (gpu_pytest below) instead of this discovery. + [ -n "${HT_E2E_PROOFS:-}" ] || grep -rlE "skipif\([^)]*cuda|requires_cuda|mark\.cuda|cuda\.is_available\(\)" \ + tests --include='test_*.py' | sort >"$OUT/cuda_test_files.txt" || true if [ -s "$OUT/cuda_test_files.txt" ]; then step cuda_pytest "$PY" -m pytest -q -p no:cacheprovider $(cat "$OUT/cuda_test_files.txt") fi @@ -73,12 +80,12 @@ spec = importlib.util.spec_from_file_location("svc", root / "tests/challenge/tes svc = importlib.util.module_from_spec(spec); sys.modules["svc"] = spec.loader and svc spec.loader.exec_module(svc) esc = svc.fixture -N = 2 +N = int(os.environ.get("HT_E2E_GPUS", "2")) if backend == "cuda": driver = subprocess.run(["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader"], check=True, capture_output=True, text=True).stdout.split() sm = torch.cuda.get_device_properties(0).multi_processor_count - assert torch.cuda.device_count() == N, torch.cuda.device_count() + assert torch.cuda.device_count() % N == 0, (torch.cuda.device_count(), N) result.update(driver=driver, sm_count=sm, gpus=[torch.cuda.get_device_name(i) for i in range(N)]) else: driver, sm = ["cpu-test"], 170 @@ -153,7 +160,8 @@ setup_v = esc.Setup(manifest, net.setup.policy, net.setup.admission_policy, net. per = manifest.training.batch_samples() def stage(d, w, start=None, ef=None): - job = esc.stage(setup_v, d, tuple(range(w * per, (w + 1) * per)), w) + # The fixture holds 16 rows: at N=8 a round takes 16 samples, so round 1 wraps around. + job = esc.stage(setup_v, d, tuple(i % len(setup_v.rows) for i in range(w * per, (w + 1) * per)), w) upd = {"deadline": int(time.time()) + 1800} if start is not None: (d / "start_state").write_bytes(start); (d / "ef_in").write_bytes(ef) @@ -204,18 +212,55 @@ print(json.dumps({"ok": result["ok"]})) sys.exit(0 if result["ok"] else 1) PYEOF +# --profile proofs (scripts/vast_e2e.py PROOF_STEPS / PROOF_TESTS): the GPU-only tests and the +# production attention path, i.e. HT_ATTN_BACKEND unset so select_backend picks sdpa-flash on +# sm_120. network_e2e above stays on math (fixture model: fp32, head_dim 4). +gpu_pytest() { + # HT_GENUINE_CUDA_INDEX is the CUDA device index for these tests; a skip is a failure. + [ -n "${HT_E2E_PROOF_TESTS:-}" ] || return 2 + env -u HT_ATTN_BACKEND HT_GENUINE_CUDA_INDEX=0 "$PY" -m pytest -rs -p no:cacheprovider \ + --junitxml="$OUT/gpu-junit.xml" $HT_E2E_PROOF_TESTS || return + ! grep -qi skipped "$OUT/gpu_pytest.log" || return 12 +} +mfu_dp2_check() { + "$PY" scripts/check_gpu_evidence.py --kind mfu "$OUT/mfu-dp2-flash.json" || return + "$PY" -c "import json, sys; e = json.load(open(sys.argv[1])); print(e['attention_backend'], e['attention_paths']) +assert e['attention_backend'] == 'sdpa-flash' and e['replay_match'] is True +assert e['attention_paths'] == ['flash'], e['attention_paths']" "$OUT/mfu-dp2-flash.json" +} +if [ "$BACKEND" = cuda ] && [ -n "${HT_E2E_PROOFS:-}" ]; then + step attn_select env -u HT_ATTN_BACKEND "$PY" -c "import hypertrain.trainer, torch +from hypertrain.trainer.attention import attention, chosen_paths, select_backend +d = torch.device('cuda', 0) +b = select_backend(d) +x = [torch.randn(1, h, 256, 128, device=d, dtype=torch.bfloat16) for h in (16, 8, 8)] +attention(*x, b) # production geometry: bf16, 2 q heads per kv head, head dim 128 +print(b, chosen_paths(), torch.__version__, torch.cuda.get_device_capability(0)) +assert b == 'sdpa-flash' and chosen_paths() == ['flash'], (b, chosen_paths())" + step gpu_pytest gpu_pytest + # 1B bf16 dp2 round + replay (head_dim 128): sdpa-flash inside real training, bitwise replay. + step mfu_dp2_flash env -u HT_ATTN_BACKEND "$PY" -m torch.distributed.run --standalone \ + --nproc-per-node 2 scripts/run_mfu_dist.py --profile mfu-dense-1b-dp2 --device cuda \ + --reduction zero2o --act-ckpt --nccl-pcie --warmup 1 --out "$OUT/mfu-dp2-flash.json" + step mfu_dp2_check mfu_dp2_check +fi + "$PY" - "$OUT" "$BACKEND" <<'PYEOF' -import json, sys +import json, os, sys from pathlib import Path out = Path(sys.argv[1]) -steps = dict(line.split("\t") for line in (out / "steps.tsv").read_text().splitlines() if line) -steps = {k: int(v) for k, v in steps.items()} +rows = [line.split("\t") for line in (out / "steps.tsv").read_text().splitlines() if line] +steps = {r[0]: int(r[1]) for r in rows} +seconds = {r[0]: int(r[2]) for r in rows} e2e = json.loads((out / "e2e.json").read_text()) if (out / "e2e.json").exists() else None files = (out / "cuda_test_files.txt") required = ["network_e2e"] + (["nvidia_smi"] if sys.argv[2] == "cuda" else []) +if sys.argv[2] == "cuda": + required += os.environ.get("HT_E2E_PROOFS", "").split() ok = all(steps.get(k) == 0 for k in required) and bool(e2e and e2e["ok"]) and all( steps[k] == 0 for k in ("selfcheck", "cuda_pytest") if k in steps) -json.dump({"ok": ok, "backend": sys.argv[2], "steps": steps, "e2e": e2e, +json.dump({"ok": ok, "backend": sys.argv[2], "steps": steps, "seconds": seconds, "e2e": e2e, + "required": required, "cuda_test_files": files.read_text().split() if files.exists() else None}, open(out / "result.json", "w"), indent=2, sort_keys=True) print("RESULT ok=%s steps=%s" % (ok, steps)) diff --git a/scripts/vast_miner_cycle_remote.sh b/scripts/vast_miner_cycle_remote.sh new file mode 100755 index 0000000..4c07488 --- /dev/null +++ b/scripts/vast_miner_cycle_remote.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Remote half of scripts/vast_e2e.py --profile miner-cycle: one CPU-safe miner cycle, no paid or +# external service. vast_e2e.py rescues /workspace/out before deleting the instance. +set -euo pipefail +out=${HT_E2E_OUT:-/workspace/out} +mkdir -p "$out" +cd "${HT_E2E_SRC:-/workspace/ht}" +command -v uv >/dev/null || curl -LsSf https://astral.sh/uv/0.12.17/install.sh | sh +export PATH="$HOME/.local/bin:$PATH" +uv sync --locked --all-extras --group dev >"$out/sync.log" 2>&1 +if [ -f scripts/mine-smoke.sh ]; then + MINE_SMOKE_EVIDENCE="$out/mine-smoke.json" bash scripts/mine-smoke.sh 2>&1 \ + | tee "$out/mine-smoke.log" +else + MINER_FLOW_EVIDENCE="$out/miner-flow.json" uv run --frozen python scripts/miner_flow_e2e.py \ + 2>&1 | tee "$out/miner-flow.log" +fi diff --git a/src/hypertrain/beacon/__main__.py b/src/hypertrain/beacon/__main__.py index 0f97677..2c20731 100644 --- a/src/hypertrain/beacon/__main__.py +++ b/src/hypertrain/beacon/__main__.py @@ -3,9 +3,13 @@ import argparse import json import sys +from pathlib import Path + +import httpx from hypertrain.beacon.core import BeaconError, BeaconUnavailable from hypertrain.beacon.drand import RELAYS, DrandQuicknet +from hypertrain.beacon.push import Pusher, read_token, redact_urls def main(argv: list[str] | None = None) -> int: @@ -14,22 +18,35 @@ def main(argv: list[str] | None = None) -> int: c = sub.add_parser("check", help="fetch + verify the latest quicknet round") c.add_argument("--live", action="store_true", required=True) c.add_argument("--relay", action="append", default=None) + u = sub.add_parser("push", help="keep a challenge service fed with verified rounds") + u.add_argument("--api", required=True, help="challenge service base URL") + u.add_argument("--admin-token-file", type=Path, required=True) + u.add_argument("--relay", action="append", default=None, help="repeatable, >= 2") + u.add_argument("--interval", type=float, default=3.0, help="seconds between ticks") + u.add_argument("--catch-up", type=int, default=20, help="push at most the last N rounds") + u.add_argument("--once", action="store_true", help="one tick, then exit") + u.add_argument("--max-failures", type=int, default=10, help="consecutive, then exit 1") args = p.parse_args(argv) + if args.cmd == "push": + return _push(u, args) relays = tuple(args.relay or RELAYS) beacon = DrandQuicknet(relays) + shown = [redact_urls(r) for r in relays] try: info_ok = beacon.info_matches() br = beacon.latest() except BeaconUnavailable as e: - print(json.dumps({"live": "CENSORED(network)", "relays": relays, "error": str(e)})) + print( + json.dumps({"live": "CENSORED(network)", "relays": shown, "error": redact_urls(str(e))}) + ) return 3 except BeaconError as e: - print(json.dumps({"live": "FAIL", "relays": relays, "error": str(e)})) + print(json.dumps({"live": "FAIL", "relays": shown, "error": redact_urls(str(e))})) return 1 out = { "live": "PASS" if info_ok else "FAIL", - "relays": relays, + "relays": shown, "chain_info_matches_pinned": info_ok, "round": br.round, "randomness": br.randomness, @@ -40,5 +57,36 @@ def main(argv: list[str] | None = None) -> int: return 0 if info_ok and br.bls_verified else 1 +def _http_url(p: argparse.ArgumentParser, flag: str, value: str) -> httpx.URL: + try: + url = httpx.URL(value) + except httpx.InvalidURL as e: + p.error(f"{flag}: {redact_urls(str(e))}") + if url.scheme not in ("http", "https") or not url.host: + p.error(f"{flag} must be an http(s) URL") + return url + + +def _push(p: argparse.ArgumentParser, args: argparse.Namespace) -> int: + # Distinct relays, or "all >= 2 relays agree" degrades to trusting one. + relays = tuple(dict.fromkeys(r.rstrip("/") for r in args.relay or RELAYS)) + for relay in relays: + _http_url(p, "--relay", relay) + if len(relays) < 2: + p.error("need >= 2 distinct relays") + # NaN fails every comparison; the upper bound keeps time.sleep() in range. + if not 0 < args.interval <= 3600 or args.catch_up < 1 or args.max_failures < 1: + p.error("--interval must be in (0, 3600], --catch-up and --max-failures >= 1") + api = _http_url(p, "--api", args.api) + try: + token = read_token(args.admin_token_file) + except (OSError, ValueError) as e: + print(f"admin token: {e}", file=sys.stderr) + return 2 + with httpx.Client(base_url=api, timeout=10.0) as service: + pusher = Pusher(service, DrandQuicknet(relays), token, args.catch_up) + return pusher.run(args.interval, args.once, args.max_failures) + + if __name__ == "__main__": sys.exit(main()) diff --git a/src/hypertrain/beacon/push.py b/src/hypertrain/beacon/push.py new file mode 100644 index 0000000..af82010 --- /dev/null +++ b/src/hypertrain/beacon/push.py @@ -0,0 +1,143 @@ +"""`python -m hypertrain.beacon push`: keep a challenge service fed with verified quicknet rounds. + +Every round comes from >= 2 agreeing relays and is BLS-verified (DrandQuicknet) before it is +posted to `/v1/admin/beacon`; the service verifies it again on ingest. Per tick: +- service 2xx: next round; +- service 401/403: exit 2 (token); +- service 408/429/5xx, network error, relays unavailable or failing verification: one failure, + retry after `--interval`, exit 1 after `--max-failures` in a row (or at once with `--once`); +- any other service status (409 conflicting round, never overwritten; 400; 404; 3xx) or a + malformed `/v1/beacon/latest` body: exit 1. +""" + +from __future__ import annotations + +import json +import re +import signal +import sys +import time +from pathlib import Path +from typing import Any + +import httpx + +from hypertrain.beacon.core import BeaconError +from hypertrain.beacon.drand import DrandQuicknet + +# Unchanged by JSON escaping and URL encoding, so redacting an echoed token by exact match holds. +# Also keeps it out of exceptions: httpx quotes an illegal header value (e.g. a newline) verbatim. +_TOKEN = re.compile(rb"[A-Za-z0-9._~-]+") +# Relay URLs (DrandQuicknet errors echo them) may carry userinfo or an `?apikey=` query. +# Userinfo runs to the last `@` before the path: a password may hold quotes or `@`. +_USERINFO = re.compile(r"(\w[\w+.-]*://)[^/\s]*@") +_QUERY = re.compile(r"[?#][^\s'\"]*") +# A path segment (not a `//host`) of >= 24 token chars is taken for a credential (`/key/`). +_PATH_TOKEN = re.compile(r"(?<=[^/]/)[A-Za-z0-9_-]{24,}(?![A-Za-z0-9_-])") +# A rejected (429) or cut-off (408) POST was not applied, so replaying it is safe. +_RETRY = frozenset({408, 429}) + + +def redact_urls(text: str) -> str: + return _PATH_TOKEN.sub("***", _QUERY.sub("", _USERINFO.sub(r"\1***@", text))) + + +class ServiceError(Exception): + """Non-2xx or malformed answer from the challenge service.""" + + def __init__(self, status: int, message: str) -> None: + super().__init__(message) + self.status = status + + +def plan(known: int, latest: int, catch_up: int) -> list[int]: + """Rounds to push, ascending: every round after `known`, at most the last `catch_up`.""" + return list(range(max(known + 1, latest - catch_up + 1), latest + 1)) + + +def read_token(path: Path) -> str: + """The admin bearer in `path`. Errors name the file, never its content.""" + raw = path.read_bytes().strip() + if not _TOKEN.fullmatch(raw): + raise ValueError(f"{path}: token must match [A-Za-z0-9._~-]+") + return raw.decode() + + +class Pusher: + def __init__( + self, service: httpx.Client, relays: DrandQuicknet, token: str, catch_up: int + ) -> None: + self.service = service + self.relays = relays + self.catch_up = catch_up + self._token = token + + def _body(self, r: httpx.Response) -> str: + # Untrusted text: a proxy echoing request headers must not leak the token. Redact, then cut. + return r.text.replace(self._token, "***")[:500] + + def _check(self, r: httpx.Response, what: str) -> None: + if not r.is_success: + raise ServiceError(r.status_code, f"{what}: HTTP {r.status_code}: {self._body(r)}") + + def _known(self) -> int: + r = self.service.get("/v1/beacon/latest") + self._check(r, "GET /v1/beacon/latest") + try: + known = r.json().get("round") + except (ValueError, AttributeError, RecursionError): + known = None + if type(known) is not int or known < 0: + raise ServiceError(r.status_code, f"GET /v1/beacon/latest: bad body {self._body(r)}") + return known + + def tick(self) -> dict[str, Any]: + latest = self.relays.latest() + known = self._known() + pushed: list[int] = [] + # Ascending: the service reports only its highest round, so a gap would never be filled. + for rnd in plan(known, latest.round, self.catch_up): + br = latest if rnd == latest.round else self.relays.get(rnd) + r = self.service.post( + "/v1/admin/beacon", + json={"round": br.round, "signature": br.signature, "randomness": br.randomness}, + headers={"Authorization": f"Bearer {self._token}"}, + ) + self._check(r, f"round {rnd}") + pushed.append(rnd) + return {"pushed": pushed, "latest": latest.round, "known": known} + + def run(self, interval: float, once: bool, max_failures: int) -> int: + """Tick until stopped: 0 on SIGTERM/Ctrl-C or after `once`, 1 protocol, 2 token.""" + failures = 0 + previous = signal.signal(signal.SIGTERM, signal.default_int_handler) + try: + while True: + try: + print(json.dumps(self.tick()), flush=True) + failures = 0 + # DrandQuicknet lets ValueError (relay body without an int round) and + # RecursionError (absurdly nested relay JSON) through. + except ( + ServiceError, + BeaconError, + ValueError, + RecursionError, + httpx.HTTPError, + ) as e: + if isinstance(e, ServiceError) and e.status < 500 and e.status not in _RETRY: + print(e, file=sys.stderr, flush=True) + return 2 if e.status in (401, 403) else 1 + failures += 1 + why = redact_urls(str(e)) + msg = f"failure {failures}/{max_failures}: {type(e).__name__}: {why}" + print(msg, file=sys.stderr, flush=True) + if once or failures >= max_failures: + return 1 + if once: + return 0 + time.sleep(interval) + except KeyboardInterrupt: + return 0 + finally: + signal.signal(signal.SIGTERM, previous) diff --git a/src/hypertrain/challenge/app.py b/src/hypertrain/challenge/app.py index b683ddd..f32a80e 100644 --- a/src/hypertrain/challenge/app.py +++ b/src/hypertrain/challenge/app.py @@ -194,7 +194,7 @@ def _require(path: Path | None, authorization: str | None) -> None: def load_coord_key(path: Path | None) -> Keypair | None: - """K_coord seed file: 32 raw bytes or 64 hex chars. Never the Cortex leaf seed.""" + """K_coord seed file: 32 raw bytes or 64 hex chars.""" if path is None: return None try: @@ -543,6 +543,11 @@ async def dataset_v2(run_id: str, request: Request, authorization: Auth = None) raise ChallengeError(422, "expected dataset object linkage") return await run(store.for_run(run_id).configure_inputs_v2, run_id, data) + @app.post("/v2/runs/{run_id}/admin/trial/{admission_id}/export") + async def export_trial_v2(run_id: str, admission_id: str, authorization: Auth = None) -> Any: + admin(authorization) + return await run(store.for_run(run_id).export_trial_v2, run_id, admission_id) + @app.post("/v2/runs/{run_id}/admin/genesis") async def genesis_v2(run_id: str, request: Request, authorization: Auth = None) -> Any: admin(authorization) diff --git a/src/hypertrain/challenge/store.py b/src/hypertrain/challenge/store.py index 58e4984..b725ed6 100644 --- a/src/hypertrain/challenge/store.py +++ b/src/hypertrain/challenge/store.py @@ -40,6 +40,7 @@ from hypertrain.challenge.disputes_v2 import Contest, DisputesV2 from hypertrain.data.assignment import assign_round from hypertrain.data.store import ( + CorruptObjectError, ObjectNotFound, StoreError, _stream, @@ -102,6 +103,7 @@ AuditPolicyV2, DisputePolicyV2, EconomicsPolicyV2, + EscrowOperation, Hex64, IslandJobV1, JoinChallenge, @@ -317,6 +319,50 @@ def rerun_message(run_id: str, w: int, hotkey: str, rerun_leaves_root: str) -> b return f"hypertrain/1|Rerun|{run_id}|{w}|{hotkey}|{rerun_leaves_root}".encode() +def _stage_once(path: Path, data: bytes) -> None: + """Create-once input for a re-emitted job: identical bytes are kept, others refused (409). + + The final name only ever appears by link(2) of a fully written, fsynced temp file, so it is + complete or absent; a SIGKILL leaves at most a unique `.name.*.part` temp, never reused.""" + import os + + # Best-effort purge of this name's crash leftovers; the age floor spares a concurrent + # emission's live temp, and lstat/unlink never follow a planted symlink. + stale = time.time() - 600 + try: + for entry in path.parent.iterdir(): + name = entry.name + if name.startswith(f".{path.name}.") and name.endswith(".part"): + try: + if not entry.is_symlink() and entry.lstat().st_mtime < stale: + entry.unlink(missing_ok=True) + except OSError: + pass + except OSError: + pass + if not os.path.lexists(path): + fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".part") + try: + with os.fdopen(fd, "wb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + try: + os.link(tmp, path) + except FileExistsError: # concurrent emission: compared below like any repeat + pass + finally: + os.unlink(tmp) + dfd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(dfd) + finally: + os.close(dfd) + with os.fdopen(os.open(path, os.O_RDONLY | os.O_NOFOLLOW), "rb") as stream: + if stream.read() != data: + raise ChallengeError(409, "island job inputs differ from the staged job") + + def _merge_answers(answers: list[bytes]) -> bytes: """One get_weights answer over per-run ledgers: each run owns 10^6 of full_share_mass. @@ -1990,16 +2036,45 @@ def rate(num: int, den: int) -> float | None: def weights(self, epoch: int, epoch_at: int | None, computed_at: int) -> bytes: if epoch_at is None: epoch_at = self.params.genesis_unix + (epoch + 1) * self.params.epoch_seconds + with self._tx() as db: + stored = db.execute( + "SELECT answer FROM weights_runs WHERE epoch=?", (epoch,) + ).fetchone() + if stored is not None: # final: a replay needs no v2 services + return bytes(stored[0]) + # Committed before any journal append: an epoch at or after it that the journal + # already answered is a retry after a failed persist, not a pre-multi-run answer. + # Past the newest journal answer, so an upgrade gap never reopens a served epoch. + since = self._meta(db, "weights_runs_from", -1) + if since < 0 and epoch not in self.ledger._engine.answers: + since = max(self.ledger._engine.answers, default=-1) + 1 + self._set_meta(db, "weights_runs_from", since) + # Only a run that already finalized a live reward can owe v2 weight. Resolved outside + # the transaction: building services may open its own. + live = ( + self._db.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='escrow_finalized'" + ).fetchone() + and self._db.execute("SELECT 1 FROM escrow_finalized LIMIT 1").fetchone() + ) + escrows = [ + self._services(r[0])[0] + for r in self._db.execute("SELECT run_id, manifest FROM runs ORDER BY run_id") + if live and json.loads(r[1]).get("manifest_version") == 2 + ] with self._tx() as db: stored = db.execute( "SELECT answer FROM weights_runs WHERE epoch=?", (epoch,) ).fetchone() if stored is not None: return bytes(stored[0]) - legacy = epoch in self.ledger._engine.answers # answered before multi-run hosting + # Answered before multi-run hosting. + legacy = epoch in self.ledger._engine.answers and not 0 <= since <= epoch answer = self.ledger.get_weights(epoch, epoch_at, computed_at) if legacy: return answer + if escrows: + answer = self._pay_v2(escrows, epoch, epoch_at, answer) last = self._meta(db, "last_answer_at", -1) self._set_meta(db, "last_answer_at", max(last, epoch_at)) answers = [answer] + [ @@ -2009,6 +2084,51 @@ def weights(self, epoch: int, epoch_at: int | None, computed_at: int) -> bytes: db.execute("INSERT INTO weights_runs VALUES(?, ?)", (epoch, merged)) return merged + def _pay_v2(self, escrows: list[EscrowV2], epoch: int, epoch_at: int, text: bytes) -> bytes: + """Fill the mass the v1 answer left unpaid with matured live v2 reward units. + + Runs inside the weights transaction: EscrowV2.pay nests as a savepoint, so the + `available -> paid` moves commit or roll back with the persisted epoch answer. + """ + from hypertrain.ledger import MAX_ENTRIES + from hypertrain.ledger.journal import canonical + + answer = json.loads(text) + meta = answer["metadata"] + if meta["reason"] != "ok": + return text + weights: dict[str, float] = answer["weights"] + budget = meta["units_burned_this_epoch"] + for escrow in escrows: + beacon = escrow.manifest.training.beacon + now = (epoch_at - beacon.genesis_time) // beacon.period + 1 # drand round at epoch_at + for row in escrow.payable(now): + if budget == 0: + break + if row["hotkey"] not in weights and len(weights) >= MAX_ENTRIES: + continue # ponytail: no ranking across the cap; the units pay later + take = min(budget, row["units"]) + op = EscrowOperation( + operation_id=sha256_hex( + f"ht-pay/2|{escrow.run_id}|{epoch}|{row['origin']}|{row['owner']}".encode() + ), + owner=row["owner"], + units=take, + origin_ids=[row["origin"]], + admission_id=None, + dispute_id=None, + ) + escrow.pay(op, signer=row["owner"]) + weights[row["hotkey"]] = weights.get(row["hotkey"], 0.0) + take + budget -= take + paid = meta["units_burned_this_epoch"] - budget + if not paid: + return text + meta["units_paid"] += paid + meta["units_burned_this_epoch"] = budget + answer["weights"] = {h: weights[h] for h in sorted(weights)} + return canonical(answer) + def _run_v2(self, run_id: str) -> RunManifestV2: row = self._db.execute("SELECT manifest FROM runs WHERE run_id=?", (run_id,)).fetchone() if row is None: @@ -2619,6 +2739,52 @@ def stage_trial_v2( deadline=deadline, ), directory + def export_trial_v2(self, run_id: str, admission_id: str) -> dict[str, Any]: + from hypertrain.data.trial_assignment import trial_samples + + self._services(run_id) + with self._lock: + manifest = self._run_v2(run_id) + row = self._db.execute( + "SELECT * FROM admissions_v2 WHERE admission_id=? AND run_id=?", + (admission_id, run_id), + ).fetchone() + if row is None or row["challenge"] is None: + raise ChallengeError(409, "NO_CHALLENGE") + challenge = JoinChallenge.model_validate_json(row["challenge"]) + trial = self._db.execute( + "SELECT outcome FROM admission_trial_results WHERE epoch=?", + (row["trial_epoch"],), + ).fetchone() + if trial is None or trial["outcome"] != "OPEN": + raise ChallengeError(409, "REFERENCE_TRIAL_CLOSED") + try: + beacon = self._beacon_v2(challenge.seed_beacon) + except BeaconError: + raise ChallengeError(503, "beacon infrastructure paused") from None + samples = trial_samples(manifest, admission_id, challenge.nonce, beacon) + job, directory = self.stage_trial_v2(run_id, challenge, samples, row["trial_epoch"]) + path = directory / "job.json" + raw = job.model_dump_json().encode() + if path.exists(): + if path.read_bytes() != raw: + raise ChallengeError(409, "immutable trial job changed") + else: + with tempfile.TemporaryDirectory(dir=directory) as temporary: + staged = Path(temporary) / "job.json" + staged.write_bytes(raw) + staged.replace(path) + objects = { + name: self.objects.put((directory / relative).read_bytes()) + for name, relative in job.object_paths.items() + } + return { + "job": job.body(), + "objects": objects, + "challenge": challenge.body(), + "nonce": challenge.nonce, + } + def _dispute_span_v2( self, run_id: str, contest: Contest, level: Any, ctx: tuple[int, ...] ) -> int: @@ -5491,6 +5657,20 @@ def relay_failure_v2(self, run_id: str, raw: bytes) -> dict[str, Any]: ) if set(evidence.chunk_ack_hashes) != {c.digest() for c in chunks}: raise ChallengeError(422, "failure includes unproved chunk custody") + # upload_grant_v2 cannot tie a multi-chunk list to delta_hash before the master + # holds the original, so a miner could frame an honest relay with chunk hashes no + # receipt could follow. Unless the held original proves each chunk, such a key is + # disabled only once rescue hashes to delta_hash. + deferred = receipt is None and len(chunk_manifest.chunks) > 1 + if deferred: + try: + original = self.objects.get(grant.delta_hash) + deferred = any( + sha256_hex(original[c.off : c.off + c.len]) != c.chunk_sha256 + for c in chunk_manifest.chunks + ) + except (ObjectNotFound, CorruptObjectError): + pass # a corrupt copy proves nothing: treat as absent, rescue decides if min(evidence.observed_beacons) <= opportunity.service_deadline or max( evidence.observed_beacons ) > self._now(self._db): @@ -5529,13 +5709,14 @@ def relay_failure_v2(self, run_id: str, raw: bytes) -> dict[str, Any]: evidence.digest(), {"artifacts": data["artifacts"], "observations": data["observations"]}, ) - self._put_record_v2( - run_id, - "relay-disabled", - f"{grant.relay_id}:{grant.assignment_epoch}:{evidence.key_id}", - {"key_id": evidence.key_id, "evidence_hash": evidence.digest()}, - ) - result = {"result": evidence.result, "disabled": True, "miner_penalty": 0} + if not deferred: + self._put_record_v2( + run_id, + "relay-disabled", + f"{grant.relay_id}:{grant.assignment_epoch}:{evidence.key_id}", + {"key_id": evidence.key_id, "evidence_hash": evidence.digest()}, + ) + result = {"result": evidence.result, "disabled": not deferred, "miner_penalty": 0} self._notify_processes_v2() return result @@ -5543,6 +5724,8 @@ async def relay_fallback_v2( self, run_id: str, failure_hash: str, client: Any ) -> dict[str, Any]: """Rescue exact signed bytes; fallback grants never renew the original cutoff.""" + import httpx + from hypertrain.data.stream_store import spool from hypertrain.protocol import relay_envelope from hypertrain.protocol.relay_messages import ( @@ -5558,6 +5741,7 @@ async def relay_fallback_v2( has_full_custody, ) from hypertrain.relay.client import RelayClient + from hypertrain.relay.core import RelayError with self._tx(notify=True): manifest = self._run_v2(run_id) @@ -5714,19 +5898,50 @@ async def relay_fallback_v2( relay = RelayClient(registry, client) recovered = bytearray() responses = [] - for request_raw, signer in requests: - with spool() as payload: - response = await relay.retrieval(request_raw, payload, relay_public=signer) - if RetrievalResponse.model_validate(response.body).status != "SERVED": - raise ChallengeError(409, "original acknowledged bytes unavailable for rescue") - payload.seek(0) - recovered.extend(payload.read()) - responses.append((request_raw, response)) + disabled = f"{grant.relay_id}:{grant.assignment_epoch}:{failure.key_id}" + + def disable() -> None: # Deferred by relay_failure_v2; earlier evidence wins. + if not self._db.execute( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='relay-disabled' AND id=?", + (run_id, disabled), + ).fetchone(): + self._put_record_v2( + run_id, + "relay-disabled", + disabled, + {"key_id": failure.key_id, "evidence_hash": failure_hash}, + ) + + unavailable = False + try: + for request_raw, signer in requests: + with spool() as payload: + response = await relay.retrieval(request_raw, payload, relay_public=signer) + if RetrievalResponse.model_validate(response.body).status != "SERVED": + unavailable = True + break + payload.seek(0) + recovered.extend(payload.read()) + responses.append((request_raw, response)) + except (httpx.HTTPError, CorruptObjectError, RelayError, SignatureError, TimeoutError): + unavailable = True + if unavailable: + # Two pinned observers proved the contractual failure and the relay cannot serve + # its own signed custody either: no response, a bad signature or binding, a + # timeout, or a chunk whose bytes miss its signed, miner-committed hash. Only + # valid chunks that concatenate to the wrong delta (422 below) can be miner + # framing, so that path never disables. + with self._tx(): + self._snapshot_current_v2(manifest) + disable() + self._notify_processes_v2() + return {"rescued": False, "disabled": True, "relay_id": grant.relay_id} with self._tx(): self._snapshot_current_v2(manifest) if len(recovered) != grant.size or sha256_hex(bytes(recovered)) != grant.delta_hash: raise ChallengeError(422, "rescued original bytes differ from signed commit") self.objects.put(bytes(recovered)) + disable() for request_raw, response in responses: self._put_record_v2( run_id, @@ -6399,7 +6614,6 @@ def _expire_leases_v2(self, run_id: str, now: int) -> None: def island_job_v2(self, run_id: str, w: int, hotkey: str) -> dict[str, Any]: from hypertrain.auditor.replay import pack_state, unpack_state - from hypertrain.gpu_ops.journal import durable_write from hypertrain.protocol.messages_v2 import IslandJobV1, StartStateV2 with self._tx(): @@ -6593,7 +6807,7 @@ def read(key: str) -> bytes: ) hashes = {} for name, value in inputs.items(): - durable_write(directory / name, value) + _stage_once(directory / name, value) hashes[name] = put_large(value, self.objects.put) # ht-parts-v1 above 1 MiB job = IslandJobV1( job_version=1, @@ -6608,7 +6822,14 @@ def read(key: str) -> bytes: object_paths={k: k for k in inputs}, deadline=manifest.training.beacon.genesis_time + (int(body["d_commit"]) - 1) * 3, ) - self._put_record_v2(run_id, "island-job", f"{w}:{hotkey}", job.body()) + stored = self._db.execute( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='island-job' AND id=?", + (run_id, f"{w}:{hotkey}"), + ).fetchone() + if stored is None: + self._put_record_v2(run_id, "island-job", f"{w}:{hotkey}", job.body()) + elif json.loads(stored[0]) != job.body(): + raise ChallengeError(409, "island job inputs differ from the staged job") staged: dict[str, Any] = {"job": job.body(), "objects": hashes} if theta_only is not None: # a miner rebuilds start_state from it + its own m/v staged["theta"] = put_large(theta_only, self.objects.put) @@ -9569,6 +9790,7 @@ def upload_grant_v2( (run_id, "%:" + env.signer), ).fetchall() eligible = [] + held = False for row in assignments: assignment_env = relay_envelope.parse_envelope(json.loads(row["data"])) a = RelayAssignment.model_validate(assignment_env.body) @@ -9608,6 +9830,12 @@ def upload_grant_v2( try: original = self.objects.get(committed_work.delta_hash) except ObjectNotFound: + # Miners commit the delta by ref only (miner/core.py:run_round); the master + # first holds the bytes after relay retrieval, where relay_receipt_v2 rechecks + # delta_hash and size. A lone chunk is the whole delta, so check it here. + whole = chunk.chunks[0].chunk_sha256 == committed_work.delta_hash + if len(chunk.chunks) > 1 or whole: + eligible.append(a) continue if len(original) != chunk.size or sha256_hex(original) != committed_work.delta_hash: raise ChallengeError(409, "accepted commit original delta corrupt") @@ -9625,6 +9853,7 @@ def upload_grant_v2( ) if expected == chunk: eligible.append(a) + held = True if len(eligible) != 1: raise ChallengeError(409, "one current signed relay assignment required") assignment = eligible[0] @@ -9649,6 +9878,34 @@ def upload_grant_v2( commit = self._record_v2(run_id, "commit", f"{assignment.w}:{env.signer}")["body"] if chunk.size != commit["delta_bytes"]: raise ChallengeError(422, "grant size differs from accepted commit") + # One live grant per assignment: a retry gets the stored grant, another chunk + # list waits for expiry. Without the original, the list is miner-chosen; with + # it, this list was just proven honest and replaces any other live grant. + replaces = None + for stored_id, stored in self._db.execute( + "SELECT id, data FROM records_v2 WHERE run_id=? AND kind='grant' " + "AND json_extract(data,'$.body.w')=? AND json_extract(data,'$.body.hotkey')=? " + "AND json_extract(data,'$.body.relay_id')=? " + "AND json_extract(data,'$.body.assignment_epoch')=? " + "AND json_extract(data,'$.body.exp_drand')>?", + ( + run_id, + assignment.w, + env.signer, + assignment.primary_id, + assignment.assignment_epoch, + now, + ), + ): + live = json.loads(stored) + if live["body"]["chunk_manifest_hash"] != chunk.chunk_manifest_hash(): + if held: + # The relay may hold this grant's reservation: name it so the + # relay can release it for the replacement (record id = digest). + replaces = stored_id + continue + raise ChallengeError(409, "live upload grant binds another chunk list") + return live grant = UploadGrant( w=assignment.w, hotkey=env.signer, @@ -9663,6 +9920,7 @@ def upload_grant_v2( + 100, nonce=sha256_hex(canonicalize([run_id, assignment.body(), chunk.body()])), exp_drand=assignment.exp_drand, + replaces=replaces, ) signed = relay_envelope.seal( self._coord(), "UploadGrant", run_id, grant, grant.exp_drand diff --git a/src/hypertrain/challenge/trial_export.py b/src/hypertrain/challenge/trial_export.py new file mode 100644 index 0000000..eb340fc --- /dev/null +++ b/src/hypertrain/challenge/trial_export.py @@ -0,0 +1,160 @@ +"""Export a staged admission trial into a copyable probe-v2 folder.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +import tempfile +from pathlib import Path + +import httpx +from pydantic import TypeAdapter + +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages_v2 import Hex64, IslandJobV1, JoinChallenge + + +def _checked(response: httpx.Response) -> httpx.Response: + if not response.is_success: + response.read() + raise ValueError(f"HTTP {response.status_code} {response.text}") + return response + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--api", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--admission-id", required=True) + parser.add_argument("--admin-token-file", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args(argv) + out: Path = args.out + try: + if out.is_symlink() or out.exists() and (not out.is_dir() or any(out.iterdir())): + raise ValueError("--out must not exist or must be an empty directory") + except (OSError, ValueError) as error: + print(str(error), file=sys.stderr) + return 2 + + token = "" + try: + destination = out.resolve() + run_id = TypeAdapter(Hex64).validate_python(args.run_id) + admission_id = TypeAdapter(Hex64).validate_python(args.admission_id) + token = args.admin_token_file.read_text().strip() + if not re.fullmatch(r"[A-Za-z0-9._~+/-]+=*", token): + raise ValueError("admin token file must contain one ASCII bearer token") + url = args.api.rstrip("/") + f"/v2/runs/{run_id}" + with httpx.Client(timeout=60, follow_redirects=False) as client: + data = _checked( + client.post( + url + f"/admin/trial/{admission_id}/export", + headers={"Authorization": f"Bearer {token}"}, + ) + ).json() + job = IslandJobV1.model_validate(data["job"]) + challenge = JoinChallenge.model_validate(data["challenge"]) + objects = TypeAdapter(dict[str, Hex64]).validate_python(data["objects"]) + if ( + job.run_id != run_id + or challenge.manifest_hash != run_id + or challenge.admission_id != admission_id + or challenge.nonce != data["nonce"] + or set(objects) != set(job.object_paths) + ): + raise ValueError("export job/challenge/object binding differs") + for name in ("start_state", "ef_in", "v0"): + if objects.get(name) != getattr(job, name + "_sha256"): + raise ValueError(f"{name} sha256 differs from job") + envelope = _checked(client.get(url + f"/join/{admission_id}/challenge")).json() + signed = envelope_v2.parse_envelope(envelope) + if ( + signed.type != "JoinChallenge" + or signed.run_id != run_id + or signed.signer != job.manifest.training.coord_pubkey + or signed.body != challenge.body() + or not envelope_v2.verify_envelope(envelope) + ): + raise ValueError("signed challenge differs from exported trial") + + parent = destination.parent + parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=".trial-export-", dir=parent) as temporary: + root = Path(temporary).resolve() + destinations = {root / "job.json", root / "challenge.json"} + paths: dict[str, Path] = {} + for name, relative in job.object_paths.items(): + target = (root / relative).resolve() + final = (out / relative).resolve() + if ( + target == root + or not target.is_relative_to(root) + or not final.is_relative_to(destination) + or any( + target == p or target in p.parents or p in target.parents + for p in destinations + ) + ): + raise ValueError(f"unsafe object path: {relative}") + destinations.add(target) + paths[name] = target + files: dict[str, str] = {} + for name, digest in objects.items(): + target = paths[name] + target.parent.mkdir(parents=True, exist_ok=True) + hasher = hashlib.sha256() + with client.stream("GET", url + "/objects/" + digest) as response: + _checked(response) + with target.open("xb") as stream: + for chunk in response.iter_bytes(): + hasher.update(chunk) + stream.write(chunk) + if hasher.hexdigest() != digest: + raise ValueError(f"{name} sha256 mismatch") + files[target.relative_to(root).as_posix()] = digest + for name, raw in ( + ("job.json", canonicalize(job.body())), + ("challenge.json", canonicalize(envelope)), + ): + (root / name).write_bytes(raw) + files[name] = hashlib.sha256(raw).hexdigest() + if destination.exists(): + if destination.is_symlink() or any(destination.iterdir()): + raise ValueError("--out is no longer empty") + published: list[Path] = [] + try: + for path in root.iterdir(): + target = path.rename(destination / path.name) + published.append(target) + except OSError: + for path in reversed(published): + path.rename(root / path.name) + raise + else: + root.rename(destination) + print( + json.dumps( + { + "out": str(out), + "run_id": run_id, + "admission_id": admission_id, + "nonce": challenge.nonce, + "deadline": job.deadline, + "files": files, + } + ) + ) + except (httpx.HTTPError, OSError, ValueError, KeyError, TypeError) as error: + message = str(error) + print(message.replace(token, "[redacted]") if token else message, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/hypertrain/datasets/__main__.py b/src/hypertrain/datasets/__main__.py index 94fbde0..efadaeb 100644 --- a/src/hypertrain/datasets/__main__.py +++ b/src/hypertrain/datasets/__main__.py @@ -1,4 +1,4 @@ -"""python -m hypertrain.datasets build|verify|publish|sources.""" +"""python -m hypertrain.datasets build|verify|pin|register|publish|sources.""" from __future__ import annotations @@ -9,7 +9,10 @@ from collections import Counter from pathlib import Path +import httpx + from hypertrain.datasets.build import build_mix +from hypertrain.datasets.pin import pin_objects, register from hypertrain.datasets.shards16 import ShardSet16Manifest, shard_name16, verify_shards16 @@ -33,6 +36,20 @@ def main(argv: list[str] | None = None) -> int: b.add_argument("--limit-docs", type=int) v = sub.add_parser("verify") v.add_argument("data_dir", type=Path) + pn = sub.add_parser("pin", help="write the contract-2 samples/proofs objects of a shard set") + pn.add_argument("--data-dir", type=Path, required=True) + pn.add_argument("--out-objects", type=Path, required=True) + pn.add_argument("--holdout-dir", type=Path, help="holdout shard set (adds holdout_commit)") + rg = sub.add_parser("register", help="link pinned objects as a v2 run's dataset inputs") + rg.add_argument("--api", required=True, help="challenge base URL") + rg.add_argument("--run-id", required=True) + rg.add_argument("--admin-token-file", type=Path, required=True) + rg.add_argument("--objects", type=Path, required=True, help="pin --out-objects dir") + rg.add_argument( + "--state-dir", type=Path, required=True, help="the service's CHALLENGE_STATE_DIR" + ) + rg.add_argument("--samples-hash", required=True) + rg.add_argument("--proofs-hash", required=True) pb = sub.add_parser("publish") pb.add_argument("data_dir", type=Path) pb.add_argument("--r2", type=Path, help="local target dir (content-addressed copy, tests)") @@ -59,6 +76,27 @@ def main(argv: list[str] | None = None) -> int: errs = verify_shards16(a.data_dir, m) print("\n".join(errs) or "OK") return 1 if errs else 0 + elif a.cmd in ("pin", "register"): + from hypertrain.data.store import LocalFSStore, StoreError + + try: + if a.cmd == "pin": + rec = pin_objects(a.data_dir, LocalFSStore(a.out_objects), a.holdout_dir) + print(json.dumps(rec, indent=1)) + return 0 + with httpx.Client(base_url=a.api, timeout=120) as client: + return register( + client, + a.run_id, + a.admin_token_file.read_text().strip(), + a.objects, + a.state_dir, + a.samples_hash, + a.proofs_hash, + ) + except (ValueError, StoreError, OSError, httpx.HTTPError) as e: + print(f"{a.cmd}: {e}", file=sys.stderr) + return 1 elif a.cmd == "publish-labels": from hypertrain.storage.publish import publish_dir from hypertrain.storage.r2 import R2Client, load_config diff --git a/src/hypertrain/datasets/pin.py b/src/hypertrain/datasets/pin.py new file mode 100644 index 0000000..14130a4 --- /dev/null +++ b/src/hypertrain/datasets/pin.py @@ -0,0 +1,123 @@ +"""Contract-2 dataset objects: `samples` (all rows in id order) and `proofs` (one hex path per +sample id), the pair `PUT /v2/runs/{run_id}/admin/dataset` links and `stage_trial_v2` slices. + +ponytail: register copies the objects into the service's local per-run store, because the +object route only accepts a subnet-registered, run-admitted hotkey signature; switch to the +signed upload route once the operator holds such a hotkey. +""" + +from __future__ import annotations + +import os +import re +import sys +from pathlib import Path +from typing import Any + +import httpx + +from hypertrain.data.store import LocalFSStore, Store +from hypertrain.datasets.shards16 import ( + ShardSet16Manifest, + U16ShardSamples, + shard_name16, + verify_shards16, +) +from hypertrain.protocol.hashing import MerkleTree +from hypertrain.protocol.jcs import canonicalize + + +class PinError(ValueError): + pass + + +def pin_objects(data_dir: Path, store: Store, holdout_dir: Path | None = None) -> dict[str, Any]: + m = ShardSet16Manifest.from_json((data_dir / "manifest.json").read_text()) + errs = verify_shards16(data_dir, m) + if errs: + raise PinError( + f"{data_dir}: shards differ from manifest merkle_root {m.merkle_root}: {errs}" + ) + tree = MerkleTree(U16ShardSamples(data_dir, m.n_shards, m.samples_per_shard, m.seq_len)) + if tree.root.hex() != m.merkle_root: + raise PinError(f"{data_dir}: sample tree root {tree.root.hex()} != merkle_root") + fragment: dict[str, Any] = { + "merkle_root": m.merkle_root, + "depth": m.depth, + "n_samples": m.n_samples, + "sample_format": m.sample_format, + "shard_uri_template": m.shard_uri_template, + "shard_sha256_root": m.shard_sha256_root, + "assign_unit": m.unit, + "unit_sha256_root": m.unit_sha256_root, + } + if holdout_dir is not None: + hold = ShardSet16Manifest.from_json((holdout_dir / "manifest.json").read_text()) + if hold.seq_len != m.seq_len: + raise PinError(f"holdout seq_len {hold.seq_len} != dataset seq_len {m.seq_len}") + if errs := verify_shards16(holdout_dir, hold): + raise PinError(f"{holdout_dir}: holdout shards differ from its manifest: {errs}") + fragment["holdout_commit"] = hold.merkle_root + # Shards are row-major u16 LE in id order, so their concatenation is the samples object. + # ponytail: each object is built whole in memory, one at a time (the service loads them whole + # too); stream into the store if a full-size set outgrows host RAM. + samples_hash = store.put( + b"".join((data_dir / shard_name16(i)).read_bytes() for i in range(m.n_shards)) + ) + proofs_hash = store.put( + canonicalize([[p.hex() for p in tree.proof(i)] for i in range(m.n_samples)]) + ) + return { + "samples_hash": samples_hash, + "proofs_hash": proofs_hash, + "n_samples": m.n_samples, + "merkle_root": m.merkle_root, + "unit_sha256_root": m.unit_sha256_root, + "shard_sha256_root": m.shard_sha256_root, + "dataset_fragment": fragment, + } + + +def register( + client: httpx.Client, + run_id: str, + token: str, + objects: Path, + state_dir: Path, + samples_hash: str, + proofs_hash: str, +) -> int: + """Copy both objects into state_dir/runs//objects, then link them over HTTP.""" + run = state_dir / "runs" / run_id + if not re.fullmatch(r"[0-9a-f]{64}", run_id) or not (run / "challenge.db").is_file(): + raise PinError( + f"{run}/challenge.db not found: --state-dir must be the service's state dir and " + "the run a hosted sub-run (the first run lives in the root store, refused here)" + ) + # A symlinked run, objects or prefix dir would pass the checks above and write the root store. + root = state_dir.resolve() + prefixes = [run / "objects" / h[:2] for h in (samples_hash, proofs_hash)] + for p in (run.resolve(), *(d.resolve() for d in (run / "objects", *prefixes))): + if root / "runs" not in p.parents or p == root / "objects": + raise PinError(f"{run}: resolves to {p}, outside {root}/runs or the root store") + # New prefix dirs take the caller's uid; a foreign one would block the service's own writes. + if (run / "objects").stat().st_uid != os.geteuid(): + raise PinError(f"{run}/objects belongs to another uid: run register as the service user") + if not objects.is_dir(): + raise PinError(f"{objects}: not a directory") + # httpx echoes an illegal header value in its error, so never let one reach the header. + if not re.fullmatch(r"[\x21-\x7e]+", token): + raise PinError("admin token file must hold one line of printable ASCII") + src, dst = LocalFSStore(objects), LocalFSStore(run / "objects") + for sha in (samples_hash, proofs_hash): + dst.put(src.get(sha)) # put skips an object the run already has + r = client.put( + f"/v2/runs/{run_id}/admin/dataset", + json={"samples_hash": samples_hash, "proofs_hash": proofs_hash}, + headers={"Authorization": f"Bearer {token}"}, + ) + if not r.is_success: + print(f"HTTP {r.status_code}: {r.text}", file=sys.stderr) + return 1 + print(r.text) + return 0 diff --git a/src/hypertrain/ledger/escrow_v2.py b/src/hypertrain/ledger/escrow_v2.py index 262e37a..7564350 100644 --- a/src/hypertrain/ledger/escrow_v2.py +++ b/src/hypertrain/ledger/escrow_v2.py @@ -970,6 +970,24 @@ def pay(self, op: EscrowOperation, *, signer: str) -> EscrowReceipt: self._move(op, "available", "paid") return self._event(op.operation_id, "PAY", op.model_dump_json(), op.owner) + def payable(self, now_beacon: int) -> list[sqlite3.Row]: + """Free units of live reward origins matured by now_beacon, oldest maturity first. + + Shadow (bootstrap) and genesis origins are collateral, never emission. Only units still + held by the origin's issued owner pay (its emptied reward_pending row marks it): units + transferred to another owner or a referee are not the earning hotkey's to emit. + """ + return self.db.execute( + "SELECT u.origin,u.owner,o.hotkey,u.units FROM escrow_units u " + "JOIN escrow_origins o ON o.origin=u.origin WHERE u.bucket='available' " + "AND u.units>0 AND o.hotkey IS NOT NULL AND o.mature_at<=? " + "AND o.origin NOT IN (SELECT origin FROM escrow_shadow_finalized) " + "AND EXISTS(SELECT 1 FROM escrow_units p WHERE p.origin=u.origin " + "AND p.owner=u.owner AND p.bucket='reward_pending') " + "ORDER BY o.mature_at,u.origin,u.owner", + (now_beacon,), + ).fetchall() + def balances(self, owner: str | None = None) -> Balances: query = "SELECT bucket,SUM(units) total FROM escrow_units" rows = self.db.execute( diff --git a/src/hypertrain/miner/cli.py b/src/hypertrain/miner/cli.py index f13fb3e..340bf04 100644 --- a/src/hypertrain/miner/cli.py +++ b/src/hypertrain/miner/cli.py @@ -30,6 +30,7 @@ shard_name16, verify_shards16, ) +from hypertrain.miner.coldkey_ops import cosign_join, escrow_lock, prepare_join from hypertrain.miner.core import Miner, MinerConfig, MinerError, NetworkMiner, canonical from hypertrain.miner.dispute_watch import add_watch_args, run_follow from hypertrain.miner.keys import create_key, list_keys, revoke_key @@ -358,11 +359,29 @@ def main(argv: list[str] | None = None) -> int: g.add_argument("--check", action="store_true") g.add_argument("--apply", action="store_true") s.add_argument("--manifest-url") + s = sub.add_parser( + "join-v2-cosign", help="operator machine: co-sign and submit a prepared join" + ) + s.add_argument("--request", type=Path, required=True) + s.add_argument("--coldkey", type=Path, required=True) + s.add_argument("--api", required=True) + s = sub.add_parser("escrow-lock", help="operator machine: lock admission collateral") + s.add_argument("--api", required=True) + s.add_argument("--run-id", required=True) + s.add_argument("--coldkey", type=Path, required=True) + s.add_argument("--hotkey-ss58", required=True) + s.add_argument("--units", type=int, required=True) + s.add_argument("--origin", action="append", required=True) + s.add_argument("--expiry", type=int) + s.add_argument("--dry-run", action="store_true") for name in ("join-v2", "status-v2", "probe-v2", "proof-v2", "run-v2", "watch"): s = sub.add_parser(name) s.add_argument("--config", type=Path) if name == "join-v2": - s.add_argument("--coldkey", type=Path, required=True) + g = s.add_mutually_exclusive_group(required=True) + g.add_argument("--coldkey", type=Path) + g.add_argument("--prepare", type=Path) + s.add_argument("--coldkey-ss58") s.add_argument("--request-id", required=True) s.add_argument("--expiry", type=int, required=True) if name in ("probe-v2", "run-v2", "watch"): @@ -380,6 +399,52 @@ def main(argv: list[str] | None = None) -> int: args = p.parse_args(argv) logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") try: + if args.command in ("join-v2-cosign", "escrow-lock") or ( + args.command == "join-v2" and args.prepare is not None + ): + try: + with httpx.Client(timeout=120, follow_redirects=False) as client: + if args.command == "join-v2": + if args.coldkey_ss58 is None: + raise MinerError("join-v2 --prepare requires --coldkey-ss58") + result = prepare_join( + client, + MinerConfig.load(args.config), + args.coldkey_ss58, + args.request_id, + args.expiry, + args.prepare, + ) + elif args.command == "join-v2-cosign": + result = cosign_join(client, args.api, args.request, args.coldkey) + else: + result = escrow_lock( + client, + args.api, + args.run_id, + args.coldkey, + args.hotkey_ss58, + args.units, + args.origin, + args.expiry, + args.dry_run, + ) + except httpx.HTTPStatusError as error: + print( + json.dumps( + {"status": error.response.status_code, "body": error.response.text}, + sort_keys=True, + ) + ) + return 1 + except (OSError, ValueError, KeyError, TypeError, httpx.InvalidURL) as error: + raise MinerError( + f"{args.command}: invalid input or response, or file unavailable" + ) from error + except httpx.RequestError as error: + raise MinerError(f"{args.command}: HTTP request failed") from error + print(json.dumps(result, sort_keys=True)) + return 0 local = _local_command(args) if local is not None: print(json.dumps(local, sort_keys=True)) diff --git a/src/hypertrain/miner/coldkey_ops.py b/src/hypertrain/miner/coldkey_ops.py new file mode 100644 index 0000000..016cdfb --- /dev/null +++ b/src/hypertrain/miner/coldkey_ops.py @@ -0,0 +1,173 @@ +"""Public join preparation and operator-side coldkey operations; no miner runtime needed.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import httpx +from pydantic import TypeAdapter + +from hypertrain.miner.core import MinerConfig, MinerError, canonical, load_keyfile +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.keys import decode_hotkey, verify +from hypertrain.protocol.messages import Hex64 +from hypertrain.protocol.messages_v2 import ( + U64, + EconomicsPolicyV2, + EscrowLock, + HardwareHint, + JoinRequest, + Positive, + RunManifestV2, +) + + +def required_units(policy: EconomicsPolicyV2, manifest: RunManifestV2, locked: int) -> int: + denominator = policy.s_lower_ppm * policy.beta_ppm * 1_000_000 + return max( + policy.S_min_units, + manifest.training.verify.s_min_units(policy.R_collectible_units), + (policy.G_max_units * 10**18 + denominator - 1) // denominator + if denominator + else policy.G_max_units + locked + 1, + ) + + +def _request(client: httpx.Client, api: str, method: str, path: str, **kw: Any) -> httpx.Response: + response = client.request(method, api.rstrip("/") + path, **kw) + response.raise_for_status() + return response + + +def _manifest(summary: dict[str, Any], run_id: str, owner: str | None = None) -> RunManifestV2: + env = envelope_v2.parse_envelope(summary["manifest_envelope"]) + if ( + env.type != "RunManifestV2" + or env.run_id != run_id + or owner is not None + and env.signer != owner + or not envelope_v2.verify_envelope(env.model_dump()) + ): + raise MinerError("v2 manifest authority/signature mismatch") + manifest = RunManifestV2.model_validate(env.body) + manifest.validate_run_id(run_id) + return manifest + + +def prepare_join( + client: httpx.Client, + cfg: MinerConfig, + coldkey_ss58: str, + request_id: str, + expiry: int, + out: Path, +) -> dict[str, Any]: + if cfg.run_id is None or cfg.owner_hotkey is None: + raise MinerError("v2 requires pinned run_id and owner_hotkey") + TypeAdapter(Hex64).validate_python(cfg.run_id) + decode_hotkey(coldkey_ss58) + summary = _request(client, cfg.api, "GET", f"/v2/runs/{cfg.run_id}").json() + manifest = _manifest(summary, cfg.run_id, cfg.owner_hotkey) + hotkey = load_keyfile(cfg.keyfile) + request = JoinRequest( + run_id=cfg.run_id, + request_id=request_id, + hotkey=hotkey.ss58, + coldkey=coldkey_ss58, + expires_beacon=expiry, + policy_hash=manifest.network.admission_policy_hash, + hardware_hint=HardwareHint( + device_name=cfg.device, + device_count=manifest.training.reference_spec.layout.n_gpus, + driver="advisory", + ), + hot_sig="0" * 128, + cold_sig="0" * 128, + ) + message = envelope_v2.join_signing_message(request) + body = {**request.body(), "hot_sig": hotkey.sign(message).hex()} + with out.open("xb") as stream: + stream.write(canonical(body)) + return {"prepared": str(out), "message_sha256": sha256_hex(message)} + + +def cosign_join(client: httpx.Client, api: str, request_path: Path, coldfile: Path) -> Any: + request = JoinRequest.model_validate(envelope_v2.load_json(request_path.read_bytes())) + message = envelope_v2.join_signing_message(request) + if not verify(decode_hotkey(request.hotkey), message, bytes.fromhex(request.hot_sig)): + raise MinerError("join hotkey signature does not verify") + coldkey = load_keyfile(coldfile) + if request.coldkey != coldkey.ss58: + raise MinerError("join coldkey differs from keyfile owner") + body = {**request.body(), "cold_sig": coldkey.sign(message).hex()} + response = _request( + client, api, "POST", f"/v2/runs/{request.run_id}/join", content=canonical(body) + ) + try: + return response.json() + except ValueError: + return response.text + + +def escrow_lock( + client: httpx.Client, + api: str, + run_id: str, + coldfile: Path, + hotkey_ss58: str, + units: int, + origin_ids: list[str], + expiry: int | None = None, + dry_run: bool = False, +) -> dict[str, Any]: + TypeAdapter(Hex64).validate_python(run_id) + decode_hotkey(hotkey_ss58) + TypeAdapter(Positive).validate_python(units) + origins = sorted(TypeAdapter(list[Hex64]).validate_python(origin_ids)) + if expiry is not None: + TypeAdapter(Positive).validate_python(expiry) + coldkey = load_keyfile(coldfile) + root = f"/v2/runs/{run_id}" + status_path = root + "/admission/" + hotkey_ss58 + status = _request(client, api, "GET", status_path).json() + record = status["record"] + if record["coldkey"] != coldkey.ss58: + raise MinerError("admission coldkey differs from keyfile owner") + admission_id = TypeAdapter(Hex64).validate_python(record["admission_id"]) + operation = EscrowLock( + operation_id=sha256_hex( + canonical(["ht-escrow-lock-cli", run_id, admission_id, coldkey.ss58, units, origins]) + ), + owner=coldkey.ss58, + units=units, + origin_ids=origins, + admission_id=admission_id, + dispute_id=None, + kind="LOCK_ADMISSION", + ) + if dry_run: + return operation.body() + summary = _request(client, api, "GET", root).json() + manifest = _manifest(summary, run_id) + policy_bytes = _request( + client, api, "GET", root + "/objects/" + manifest.network.economics_policy_hash + ).content + if sha256_hex(policy_bytes) != manifest.network.economics_policy_hash: + raise MinerError("economics policy hash mismatch") + policy = EconomicsPolicyV2.model_validate(envelope_v2.load_json(policy_bytes)) + if expiry is None: + expiry = TypeAdapter(U64).validate_python(summary["now_round"]) + 1000 + envelope = envelope_v2.seal(coldkey, "EscrowLock", run_id, operation, expiry) + result = _request( + client, api, "POST", root + "/escrow/lock", content=canonical(envelope) + ).json() + status = _request(client, api, "GET", status_path).json() + locked = TypeAdapter(U64).validate_python(status["funding"]["locked_units"]) + return { + "event_hash": result["event_hash"], + "locked_units": locked, + "required_units": required_units(policy, manifest, locked), + "eligible": status["eligible"], + } diff --git a/src/hypertrain/ops/__init__.py b/src/hypertrain/ops/__init__.py new file mode 100644 index 0000000..9282832 --- /dev/null +++ b/src/hypertrain/ops/__init__.py @@ -0,0 +1 @@ +"""Offline operator tools for validated run artifacts.""" diff --git a/src/hypertrain/ops/manifest_v2.py b/src/hypertrain/ops/manifest_v2.py new file mode 100644 index 0000000..7db2d17 --- /dev/null +++ b/src/hypertrain/ops/manifest_v2.py @@ -0,0 +1,329 @@ +"""Build owner-signed network-v2 manifests offline; emit tiny public example inputs.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, Never + +from pydantic import JsonValue, ValidationError + +from hypertrain.data.store import CorruptObjectError, LocalFSStore +from hypertrain.ledger.escrow_v2 import genesis_allocation_hash +from hypertrain.protocol.envelope_v2 import Intake, load_json, parse_envelope, seal +from hypertrain.protocol.example import example_manifest +from hypertrain.protocol.hashing import MerkleTree, sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import encode_hotkey +from hypertrain.protocol.messages import f32hex +from hypertrain.protocol.messages_v2 import ( + AdmissionPolicyV2, + AggregationPolicyV2, + ArtifactLimits, + AuditPolicyV2, + DisputePolicyV2, + EconomicsPolicyV2, + RunManifestV2, + TrainingManifestV2, + WireModel, +) +from hypertrain.protocol.relay_messages import RelayRegistryV1 + +POLICIES: dict[str, type[WireModel]] = { + "admission": AdmissionPolicyV2, + "economics": EconomicsPolicyV2, + "aggregation": AggregationPolicyV2, + "audit": AuditPolicyV2, + "dispute": DisputePolicyV2, +} + + +class _Parser(argparse.ArgumentParser): + def error(self, message: str) -> Never: + raise ValueError("invalid command arguments; use --help") + + +def _read(path: Path) -> dict[str, JsonValue]: + raw = path.read_bytes() + try: + return load_json(raw) + except (ValueError, RecursionError): + raise ValueError( + "invalid JSON input: expected a bounded object without duplicates or floats" + ) from None + + +def _check_out(out: Path) -> None: + if out.is_symlink() or (out.exists() and (not out.is_dir() or any(out.iterdir()))): + raise ValueError("--out must not exist or be empty") + + +def _write(out: Path, files: dict[str, bytes]) -> None: + _check_out(out) + out.mkdir(parents=True, exist_ok=True) + for name, raw in files.items(): + with (out / name).open("xb") as target: + target.write(raw) + + +def _build(args: argparse.Namespace) -> dict[str, JsonValue]: + if args.exp_round <= 0: + raise ValueError("--exp-round must be positive") + _check_out(args.out) + raw_policies = _read(args.policies) + if raw_policies.keys() != POLICIES.keys(): + raise ValueError( + "--policies requires exactly admission, economics, aggregation, audit, dispute" + ) + objects = {name: model.model_validate(raw_policies[name]) for name, model in POLICIES.items()} + objects["relay_registry"] = RelayRegistryV1.model_validate(_read(args.relays)) + training = _read(args.training) + supplied_hash = training.get("init_state_hash") + training.setdefault("init_state_hash", "0" * 64) + validated = TrainingManifestV2.model_validate(training) + audit = objects["audit"] + assert isinstance(audit, AuditPolicyV2) + audit.validate_training(validated) + if len(set(validated.auditors)) < 2: + raise ValueError("two distinct audit reference identities required") + if validated.inner.state_policy == "derived": + raise ValueError("v2 derived-state influence requires authenticated v0 integration") + economics = objects["economics"] + assert isinstance(economics, EconomicsPolicyV2) + if economics.round_reward_units != validated.budget.epochs_per_round * 1_000_000: + raise ValueError("MANIFEST_REWARD_BUDGET") + if economics.reward_authority != validated.coord_pubkey: + raise ValueError("PINNED_AUTHORITIES") + hashes = {name: model.digest() for name, model in objects.items()} + network = { + **{f"{name}_policy_hash": hashes[name] for name in POLICIES}, + "relay_registry_hash": hashes["relay_registry"], + "audit_mode": "anchored-full", + "full_anchor_version": 1, + "capabilities": ["island-replay", "all-level-disputes", "transport-receipts"], + } + draft = RunManifestV2.model_validate( + {"manifest_version": 2, "training": validated.body(), "network": network} + ) + from hypertrain.miner.core import KeyfileError, load_keyfile + from hypertrain.trainer.compress import state_hash + from hypertrain.trainer.config import TrainConfig + from hypertrain.trainer.model import init_params + + try: + owner = load_keyfile(args.owner_seed) + except KeyfileError as error: + raise ValueError(str(error)) from None + initial_hash = state_hash(init_params(TrainConfig.from_manifest_v2(draft).model)) + if supplied_hash is not None and supplied_hash != initial_hash: + raise ValueError("init_state_hash does not match computed initial state") + final_training = validated.body() + final_training["init_state_hash"] = initial_hash + manifest = RunManifestV2.model_validate( + {"manifest_version": 2, "training": final_training, "network": network} + ) + run_id = manifest.run_id() + envelope = canonicalize(seal(owner, "RunManifestV2", run_id, manifest, args.exp_round)) + parsed = parse_envelope(envelope) + RunManifestV2.model_validate(parsed.body).validate_run_id(parsed.run_id) + Intake(run_id, {"RunManifestV2": lambda signer: signer == owner.ss58}).accept(envelope, 1) + files = { + "manifest.json": canonicalize(manifest.body(), allow_float=False), + "manifest-envelope.json": envelope, + **{ + f"policy-{name}.json": canonicalize(objects[name].body(), allow_float=False) + for name in POLICIES + }, + "relay-registry.json": canonicalize(objects["relay_registry"].body(), allow_float=False), + } + index: dict[str, JsonValue] = { + "run_id": run_id, + "owner_ss58": owner.ss58, + "objects": dict(hashes), + "files": {name: sha256_hex(raw) for name, raw in files.items()}, + } + files["index.json"] = canonicalize(index, allow_float=False) + if args.state_dir is not None: + store = LocalFSStore(args.state_dir / "objects") + for model in objects.values(): + store.get(store.put(canonicalize(model.body(), allow_float=False))) + _write(args.out, files) + return index + + +def _example() -> dict[str, bytes]: + coord, auditor0, auditor1, referee, relay, cold, other = ( + encode_hotkey(bytes([i]) * 32) for i in range(1, 8) + ) + training: dict[str, Any] = example_manifest().body() + training["model"].update( + n_layers=1, + d_model=8, + n_heads=2, + n_kv_heads=2, + d_ff=12, + vocab=16, + seq_len=4, + n_experts=1, + top_k_experts=1, + param_count=824, + ) + training["inner"].update(H=1, J=1, micro_batch=1, grad_accum=1, state_policy="reset") + training["inner"]["lr_schedule"].update(warmup=0, stable=100, peak_lr=f32hex(0.001)) + rows = [b"".join(((i * 5 + j) % 16).to_bytes(4, "little") for j in range(5)) for i in range(16)] + training["dataset"].update(merkle_root=MerkleTree(rows).root.hex(), n_samples=16, depth=4) + training["reference_spec"]["driver_allowlist"] = ["cpu-test"] + training["coord_pubkey"] = coord + training["auditors"] = [auditor0, auditor1] + training["init_state_hash"] = "0" * 64 + training = TrainingManifestV2.model_validate(training).body() + del training["init_state_hash"] + admission = AdmissionPolicyV2( + q_base="0000803e", + E=4, + clean_finalizations=12, + work_screen_epoch_rounds=1000, + max_pending=32, + max_trial_replays=2, + max_trials_per_beacon=2, + join_per_beacon=2, + join_burst=4, + ip_prefix_per_beacon=32, + ip_prefix_burst=64, + challenge_rounds=100, + artifact_limits=ArtifactLimits( + max_object_bytes=1_000_000, max_body_bytes=65536, max_chunk_manifest_bytes=1048576 + ), + ) + economics = EconomicsPolicyV2( + ledger_mode="test", + G_max_units=100, + R_collectible_units=100, + S_min_units=1000, + beta_ppm=1_000_000, + gammaV_units=0, + s_lower_ppm=1_000_000, + q_floor=1_000_000, + genesis_allocation_hash=genesis_allocation_hash([(cold, 20000, 1), (other, 10000, 1)]), + reward_authority=coord, + round_reward_units=1_000_000, + max_total_issuance=20_000_000, + shadow_bootstrap_rounds=12, + ) + aggregation = AggregationPolicyV2( + arithmetic="flat-cclip-cap-v1", + order="utf8", + center="prev_outer_update", + trust_mode="uniform-verified", + weight_quantum=16777216, + miner_cap_units=4194304, + probation_cap_units=4194304, + owner_group_caps={}, + preclip_norm=f32hex(1), + cclip_tau=f32hex(1), + cclip_iters=2, + ) + audit = AuditPolicyV2( + lease_rounds=100, + max_attempts=2, + max_concurrent_per_auditor=1, + max_running_jobs=2, + max_queued_jobs=32, + max_anchor_age_rounds=1, + max_steps_per_attempt=2, + ) + dispute = DisputePolicyV2( + referees=[referee], + max_open=32, + max_transcript_entries=256, + max_entry_bytes=65536, + fanout=2, + max_referee_reassignments=1, + max_referee_cost_units=100, + ) + registry = RelayRegistryV1.model_validate( + { + "registry_version": 1, + "epoch": 0, + "previous_registry_hash": None, + "specs": [ + { + "id": "local", + "region": "local", + "https_url": "https://relay.test", + "pubkeys": [ + { + "key_id": "k1", + "pubkey": relay, + "valid_from_round": 0, + "valid_until_round": 10000, + } + ], + "max_object_bytes": 1000000, + "max_inflight_bytes": 1000000, + "codecs": ["ht-sparse-v1", "ht-dense-int8-v1"], + "mode": "transport", + } + ], + } + ) + policies = dict( + admission=admission, + economics=economics, + aggregation=aggregation, + audit=audit, + dispute=dispute, + ) + return { + "training.json": canonicalize(training, allow_float=False), + "policies.json": canonicalize( + {k: v.body() for k, v in policies.items()}, allow_float=False + ), + "relays.json": canonicalize(registry.body(), allow_float=False), + } + + +def main(argv: list[str] | None = None) -> int: + parser = _Parser(prog="python -m hypertrain.ops.manifest_v2") + sub = parser.add_subparsers(dest="cmd", required=True) + build = sub.add_parser("build", help="validate, initialize, sign, and write a v2 run") + for name in ("training", "policies", "relays", "owner-seed", "out"): + build.add_argument(f"--{name}", type=Path, required=True) + build.add_argument("--exp-round", type=int, required=True) + build.add_argument("--state-dir", type=Path) + example = sub.add_parser("example", help="write tiny CPU inputs with public placeholders") + example.add_argument("--out", type=Path, required=True) + try: + args = parser.parse_args(argv) + if args.cmd == "build": + result = _build(args) + else: + _check_out(args.out) + files = _example() + _write(args.out, files) + result = {"files": {name: sha256_hex(raw) for name, raw in files.items()}} + except ValidationError as error: + detail = error.errors(include_input=False, include_context=False, include_url=False)[0] + location = " ".join(".".join(map(str, detail["loc"])).splitlines()) or "input" + print(f"error: {location}: {detail['type']}", file=sys.stderr) + return 2 + except ValueError as error: + print("error: " + " ".join(str(error).splitlines()), file=sys.stderr) + return 2 + except CorruptObjectError: + print("error: object store contains corrupt content", file=sys.stderr) + return 2 + except OSError: + print("error: file access failed", file=sys.stderr) + return 2 + except Exception: + print("error: unexpected failure", file=sys.stderr) + return 1 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/hypertrain/protocol/relay_messages.py b/src/hypertrain/protocol/relay_messages.py index c5e6d69..0566bff 100644 --- a/src/hypertrain/protocol/relay_messages.py +++ b/src/hypertrain/protocol/relay_messages.py @@ -122,6 +122,9 @@ class UploadGrant(WireModel): retain_until: Positive nonce: Hex64 exp_drand: Positive + # Digest of the unreceipted grant this one supersedes at the same relay; omitted when + # None, so existing grant digests and signatures are unchanged. + replaces: Hex64 | None = Field(None, exclude_if=lambda v: v is None) @model_validator(mode="after") def _retention(self) -> UploadGrant: diff --git a/src/hypertrain/relay/core.py b/src/hypertrain/relay/core.py index 91bc42e..b43ab03 100644 --- a/src/hypertrain/relay/core.py +++ b/src/hypertrain/relay/core.py @@ -8,6 +8,7 @@ from __future__ import annotations import hashlib +import logging import secrets from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager @@ -18,7 +19,7 @@ import httpx from pydantic import BaseModel, ConfigDict, Field -from hypertrain.data.store import CorruptObjectError, ObjectNotFound +from hypertrain.data.store import CorruptObjectError, ObjectNotFound, StoreError from hypertrain.data.stream_store import StreamStore, receive, spool from hypertrain.protocol import envelope_v2, relay_envelope from hypertrain.protocol.envelope import SignatureError @@ -53,6 +54,10 @@ class RelayError(ValueError): """Authenticated contract, authorization, quota or lifecycle rejection.""" +class AssemblyMismatch(RelayError): + """Granted chunks are held but their concatenation is not the granted delta.""" + + @dataclass(frozen=True, slots=True) class Settlement: """Authenticated L0 journal facts, not caller-provided JSON or a wire replacement.""" @@ -250,6 +255,22 @@ def commit(state: Journal) -> None: return if self.draining: raise RelayError("relay draining; new grants disabled") + replaced = state.uploads.get(grant.replaces) if grant.replaces else None + if replaced is not None: + # Master-signed supersession of this miner's own unreceipted reservation; + # a replaced grant the relay never saw needs nothing removed. + old = UploadGrant.model_validate(replaced.grant.body) + if (old.w, old.hotkey, old.relay_id) != (grant.w, grant.hotkey, grant.relay_id): + raise RelayError("replacement names another miner/round reservation") + if replaced.receipt is not None: + raise RelayError("replaced upload already has a receipt") + if replaced.chunks: + # Signed chunk custody keeps its retention promise; never drop it. + raise RelayError("replaced upload already holds signed chunk custody") + assert grant.replaces is not None + del state.uploads[grant.replaces] + if state.nonces.get(old.nonce) == grant.replaces: + del state.nonces[old.nonce] key_id = self._key(now, state) if len(state.uploads) >= self.max_uploads: raise RelayError("retained upload quota exhausted") @@ -399,11 +420,20 @@ async def _complete(self, grant_hash: str) -> RelayEnvelope: if not has_full_custody(grant, record.manifest, opportunity, chunks): raise RelayError("completion requires exact timely full custody") with spool() as whole: + digest = hashlib.sha256() for chunk in record.manifest.chunks: with spool() as file: await self.backing.get(chunk.chunk_sha256, chunk.len, file) while data := file.read(64 << 10): + digest.update(data) whole.write(data) + if whole.tell() != grant.size or digest.hexdigest() != grant.delta_hash: + # Deterministic: the held chunks can never assemble, so never retry. + def mismatch(state: Journal) -> None: + self._record(state, grant_hash).forward_error = AssemblyMismatch.__name__ + + await self._change(mismatch) + raise AssemblyMismatch("granted chunks do not assemble to delta_hash") whole.seek(0) await self.backing.put(whole, grant.delta_hash, grant.size) now = await self.now() @@ -472,10 +502,15 @@ async def recover(self) -> None: if ( len(record.chunks) == len(record.manifest.chunks) and record.master_acceptance is None + # Deterministic failures: retrying the same held bytes cannot succeed. + and record.forward_error + not in (AssemblyMismatch.__name__, CorruptObjectError.__name__) ): try: await self.complete(grant_hash) - except RelayError as error: + except AssemblyMismatch: + continue # recorded by complete; custody of each chunk stays retrievable + except (RelayError, StoreError) as error: # A missed completion deadline must not prevent retained-byte retrieval. failure_name = type(error).__name__ @@ -490,6 +525,12 @@ def pending( except httpx.HTTPError: # Exact failure recorded by complete; custody remains retrievable. continue + except (OSError, ValueError) as error: + # Retriable (TimeoutError, ValidationError, SignatureError, I/O): record + # nothing terminal so a later recover() tries again; never log payloads. + logging.getLogger(__name__).warning( + "relay recover deferred: %s", type(error).__name__ + ) def _scope(self, record: UploadRecord, hashes: list[str]) -> list[Pin]: if ( diff --git a/src/hypertrain/trainer/attention.py b/src/hypertrain/trainer/attention.py index 85bb49d..7c5b1d8 100644 --- a/src/hypertrain/trainer/attention.py +++ b/src/hypertrain/trainer/attention.py @@ -13,10 +13,18 @@ The math path keeps the explicit K/V expansion: a broadcast (native-GQA) math path differs from it by 1 ulp in dK for some shapes, so it cannot be the bitwise oracle. Native GQA lives in the fused backends (enable_gqa / flash-attn MQA), which never materialise expanded K/V. + +sdpa-flash is the flash kernel only: with enable_gqa (the qualified path) or, when that is +rejected, on K/V expanded to q heads. Nothing else is served under this name (no mem-efficient +kernel, no math fallback): numerics outside the qualification record must stop the run, not +change silently. The path is probed once per (device, dtype, q heads per kv head, head dim), +cached and logged once; it is a pure function of hardware, torch build and model geometry, so +ranks and reruns on the same hardware take the same path. chosen_paths() exposes it for evidence. """ from __future__ import annotations +import logging import math import os from typing import Literal @@ -24,6 +32,9 @@ import torch import torch.nn.functional as F from torch import Tensor +from torch.nn.attention import SDPBackend, sdpa_kernel + +log = logging.getLogger(__name__) Backend = Literal["math", "sdpa-flash", "flash-attn"] BACKENDS: tuple[Backend, ...] = ("sdpa-flash", "flash-attn", "math") @@ -34,6 +45,11 @@ ((12, 0), "2.14.0+cu130"): "sdpa-flash", } +# sdpa-flash paths, in order; "+expanded-kv" repeats K/V heads and calls SDPA without enable_gqa. +PATHS = ("flash", "flash+expanded-kv") +_NO_KERNEL = ("No available kernel", "No viable backend") # CUDA / CPU dispatcher messages +_PATHS: dict[tuple[torch.device, torch.dtype, int, int], str] = {} + _MASKS: dict[tuple[int, torch.device], Tensor] = {} @@ -57,15 +73,62 @@ def select_backend(device: torch.device) -> Backend: return QUALIFIED_BACKENDS.get((cap, torch.__version__), "math") +def _sdpa(path: str, q: Tensor, k: Tensor, v: Tensor) -> Tensor: + if path.endswith("+expanded-kv"): + g = q.shape[1] // k.shape[1] + k, v = k.repeat_interleave(g, 1), v.repeat_interleave(g, 1) + with sdpa_kernel([SDPBackend.FLASH_ATTENTION]): + return F.scaled_dot_product_attention( + q, k, v, is_causal=True, enable_gqa=k.shape[1] != q.shape[1] + ) + + +@torch.compiler.assume_constant_result # HT_COMPILE=1 traces the cached choice, not the probe +def _path(device: torch.device, dtype: torch.dtype, group: int, hd: int) -> str: + """First of PATHS whose kernel accepts these inputs, probed once on tiny zero tensors. + + Raises RuntimeError when none does (e.g. fp32): force HT_ATTN_BACKEND=math for such models. + """ + key = (device, dtype, group, hd) + if key in _PATHS: + return _PATHS[key] + path = "" + # Grad enabled: training constraints apply, and no_grad replay must pick the same path. + # Identity hooks keep the probe's saved tensors out of an enclosing HT_ACT_CKPT checkpoint. + with torch.enable_grad(), torch.autograd.graph.saved_tensors_hooks(lambda x: x, lambda x: x): + q = torch.zeros(1, group, 2, hd, device=device, dtype=dtype, requires_grad=True) + kv = torch.zeros(1, 1, 2, hd, device=device, dtype=dtype, requires_grad=True) + for candidate in PATHS: + if group == 1 and candidate.endswith("+expanded-kv"): + continue + try: + _sdpa(candidate, q, kv, kv) + except RuntimeError as e: + if not str(e).startswith(_NO_KERNEL): + raise + continue + path = candidate + break + if not path: + raise RuntimeError( + f"sdpa-flash: no flash path accepts {device} {dtype} group {group} head dim {hd}" + ) + _PATHS[key] = path + log.warning("attention backend: %s (%s %s, group %d, hd %d)", path, device, dtype, group, hd) + return path + + +def chosen_paths() -> list[str]: + """Distinct sdpa-flash paths this process has used, for run evidence.""" + return sorted(set(_PATHS.values())) + + def attention(q: Tensor, k: Tensor, v: Tensor, backend: Backend = "math") -> Tensor: """Causal attention. q [b, h, t, hd]; k, v [b, kvh, t, hd] with h % kvh == 0.""" b, h, t, hd = q.shape kvh = k.shape[1] if backend == "sdpa-flash": - from torch.nn.attention import SDPBackend, sdpa_kernel - - with sdpa_kernel([SDPBackend.FLASH_ATTENTION]): - return F.scaled_dot_product_attention(q, k, v, is_causal=True, enable_gqa=kvh != h) + return _sdpa(_path(q.device, q.dtype, h // kvh, hd), q, k, v) if backend == "flash-attn": from flash_attn import flash_attn_func # type: ignore[import-not-found] diff --git a/tests/beacon/test_push.py b/tests/beacon/test_push.py new file mode 100644 index 0000000..da6f24c --- /dev/null +++ b/tests/beacon/test_push.py @@ -0,0 +1,312 @@ +import json +import os +import secrets +import signal +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import httpx +import pytest + +import hypertrain.beacon.__main__ as beacon_cli +import hypertrain.beacon.push as push +from hypertrain.beacon import DrandQuicknet +from hypertrain.beacon.push import plan + +VECTORS = json.loads((Path(__file__).parent / "vectors.json").read_text())["rounds"] +V0 = VECTORS[0] +# Round right after V0, fetched 2026-10-18 from api.drand.sh and drand.cloudflare.com (identical). +R1 = { + "round": 1000001, + "randomness": "9f45f439afd81e9846b3b4dc5e3e6051922c73c8459d18e9d507b52ddbd884ff", + "signature": ( + "a5bd91e5e2d8c0bf51bffdfad87eef34348fd9c0b2df2bee39db90bdef7e1399" + "b1a77bb2fe98b24d84c0936a306c4218" + ), +} +ADMIN_TOKEN = secrets.token_hex(16) +RELAY_ARGS = ["--relay", "https://a.test", "--relay", "https://b.test"] +REAL_CLIENT = httpx.Client + + +class Service: + """Fake challenge service: GET /v1/beacon/latest answers from `gets`, POSTs get `post`.""" + + def __init__( + self, gets: list[int | httpx.Response | Exception] | None = None, post: int = 200 + ) -> None: + self.gets = gets or [0] + self.post = post + self.seen: list[str] = [] + self.posts: list[tuple[dict[str, Any], str | None]] = [] + + def handler(self, req: httpx.Request) -> httpx.Response: + self.seen.append(f"{req.method} {req.url.path}") + if req.method == "GET": + assert req.url.path == "/v1/beacon/latest" + got = self.gets.pop(0) if len(self.gets) > 1 else self.gets[0] + if isinstance(got, Exception): + raise got + return ( + got if isinstance(got, httpx.Response) else httpx.Response(200, json={"round": got}) + ) + assert (req.method, req.url.path) == ("POST", "/v1/admin/beacon") + auth = req.headers.get("authorization") + self.posts.append((json.loads(req.content), auth)) + # Echo the bearer like a misconfigured proxy, straddling the 500-char log cut. + return httpx.Response(self.post, text="x" * 470 + f"echo {auth}") + + +def relay_client(per_host: dict[str, dict[str, Any] | int | str] | None = None) -> httpx.Client: + """Relays serving V0/R1 by path, or `per_host` bodies: dict = JSON, int = status, str = text.""" + by_path = {"latest": R1, str(V0["round"]): V0, str(R1["round"]): R1} + + def handler(req: httpx.Request) -> httpx.Response: + if per_host is None: + return httpx.Response(200, json=by_path[req.url.path.rsplit("/", 1)[1]]) + got = per_host[req.url.host] + if isinstance(got, int): + return httpx.Response(got) + return ( + httpx.Response(200, text=got) if isinstance(got, str) else httpx.Response(200, json=got) + ) + + return httpx.Client(transport=httpx.MockTransport(handler)) + + +@pytest.fixture +def wire(monkeypatch: pytest.MonkeyPatch) -> Callable[..., list[float]]: + def install(svc: Service, relays: httpx.Client | None = None) -> list[float]: + relays = relays or relay_client() + transport = httpx.MockTransport(svc.handler) + monkeypatch.setattr(beacon_cli, "DrandQuicknet", lambda r: DrandQuicknet(r, client=relays)) + monkeypatch.setattr(httpx, "Client", lambda **kw: REAL_CLIENT(transport=transport, **kw)) + slept: list[float] = [] + monkeypatch.setattr(push.time, "sleep", slept.append) + return slept + + return install + + +@pytest.fixture +def token_file(tmp_path: Path) -> Path: + path = tmp_path / "admin.token" + path.write_text(f" {ADMIN_TOKEN}\r\n") + return path + + +@pytest.fixture +def cli(token_file: Path, capsys: pytest.CaptureFixture[str]) -> Callable[..., Any]: + def run( + *extra: str, token: Path = token_file, api: str = "http://svc.test", relays: Any = None + ) -> tuple[object, str, str]: + argv = ["push", "--api", api, "--admin-token-file", str(token)] + try: + rc: object = beacon_cli.main( + [*argv, *(RELAY_ARGS if relays is None else relays), *extra] + ) + except SystemExit as e: + rc = e.code + out, err = capsys.readouterr() + # No 8-char slice of the token anywhere: a cut-then-redact bug leaks a prefix. + assert not any(ADMIN_TOKEN[i : i + 8] in out + err for i in range(len(ADMIN_TOKEN) - 7)) + return rc, out, err + + return run + + +def http(status: int) -> httpx.Response: + return httpx.Response(status, text="busy") + + +def payload(v: dict[str, Any]) -> dict[str, Any]: + return {"round": v["round"], "signature": v["signature"], "randomness": v["randomness"]} + + +def test_plan_pushes_after_known_bounded_by_catch_up() -> None: + assert plan(0, 5, 20) == [1, 2, 3, 4, 5] + assert plan(100, 105, 3) == [103, 104, 105] + assert plan(105, 105, 3) == [] + + +def test_tick_pushes_missing_rounds_in_order_with_bearer(wire: Any, cli: Any) -> None: + svc = Service(gets=[0]) + slept = wire(svc) + rc, out, _ = cli("--catch-up", "2", "--once") + assert (rc, slept) == (0, []) + assert svc.posts == [ + (payload(V0), f"Bearer {ADMIN_TOKEN}"), + (payload(R1), f"Bearer {ADMIN_TOKEN}"), + ] + assert [json.loads(line) for line in out.splitlines()] == [ + {"pushed": [V0["round"], R1["round"]], "latest": R1["round"], "known": 0} + ] + + +@pytest.mark.parametrize( + ("status", "rc"), [(409, 1), (400, 1), (404, 1), (302, 1), (401, 2), (403, 2)] +) +def test_rejected_push_exits_at_once_without_further_posts( + wire: Any, cli: Any, status: int, rc: int +) -> None: + svc = Service(gets=[0], post=status) + slept = wire(svc) + got, out, err = cli("--catch-up", "2", "--max-failures", "3") + assert (got, out, slept) == (rc, "", []) + assert [p["round"] for p, _ in svc.posts] == [V0["round"]] + assert f"round {V0['round']}: HTTP {status}: " in err and "echo Bearer ***" in err + + +@pytest.mark.parametrize("status", [408, 429, 503]) +def test_transient_push_status_is_retried(wire: Any, cli: Any, status: int) -> None: + svc = Service(gets=[0], post=status) + slept = wire(svc) + rc, out, err = cli("--catch-up", "2", "--max-failures", "2", "--interval", "0.25") + assert (rc, out, slept) == (1, "", [0.25]) + assert [p["round"] for p, _ in svc.posts] == [V0["round"], V0["round"]] + assert f"failure 2/2: ServiceError: round {V0['round']}: HTTP {status}" in err + + +def test_only_consecutive_failures_count(wire: Any, cli: Any) -> None: + down = httpx.ConnectError("refused") + svc = Service(gets=[http(503), down, R1["round"], http(502), down, http(500)]) + slept = wire(svc) + rc, out, err = cli("--interval", "0.25", "--max-failures", "3") + assert (rc, slept) == (1, [0.25] * 5) + assert svc.seen == ["GET /v1/beacon/latest"] * 6 + assert [json.loads(line) for line in out.splitlines()] == [ + {"pushed": [], "latest": R1["round"], "known": R1["round"]} + ] + assert "failure 3/3" in err + + +def test_once_failed_tick_exits_1(wire: Any, cli: Any) -> None: + slept = wire(Service(gets=[http(503)])) + rc, out, err = cli("--once") + assert (rc, out, slept) == (1, "", []) + assert "failure 1/10: ServiceError: GET /v1/beacon/latest: HTTP 503: busy" in err + + +@pytest.mark.parametrize( + ("relays", "error"), + [ + ({"a.test": V0, "b.test": VECTORS[1]}, "BeaconVerificationError: relays disagree"), + ({"a.test": V0, "b.test": 503}, "BeaconUnavailable"), + ({"a.test": V0, "b.test": {}}, "ValueError"), + ({"a.test": V0, "b.test": "[" * 100_000 + "]" * 100_000}, "RecursionError"), + ], + ids=["disagree", "down", "no-round", "nested"], +) +def test_relay_failure_pushes_nothing_and_counts_failures( + wire: Any, cli: Any, relays: dict[str, Any], error: str +) -> None: + svc = Service() + slept = wire(svc, relay_client(relays)) + rc, out, err = cli("--interval", "0.25", "--max-failures", "2") + assert (rc, out, slept, svc.seen) == (1, "", [0.25], []) + assert f"failure 1/2: {error}" in err and f"failure 2/2: {error}" in err + + +@pytest.mark.parametrize( + "content", [" \n", "Zq7Xw\nLeakMe9\n", "bGVhay1tZQ+/=="], ids=["empty", "newline", "base64"] +) +def test_unusable_token_file_exits_2_before_any_request( + wire: Any, cli: Any, tmp_path: Path, content: str +) -> None: + svc = Service() + wire(svc) + bad = tmp_path / "bad.token" + bad.write_text(content) + rc, out, err = cli("--once", token=bad) + assert (rc, out, svc.seen) == (2, "", []) + assert "token must match [A-Za-z0-9._~-]+" in err + assert not any(part in err for part in content.split()) + + +@pytest.mark.parametrize( + ("args", "kw"), + [ + ((), {"relays": ["--relay", "https://a.test"]}), + ((), {"relays": ["--relay", "https://a.test", "--relay", "https://a.test/"]}), + ((), {"relays": ["--relay", "https://a.test", "--relay", "b.test"]}), + ((), {"api": "ftp://svc.test"}), + ((), {"api": "http://[::1"}), + (("--interval", "0"), {}), + (("--interval", "nan"), {}), + (("--catch-up", "0"), {}), + ], + ids=["one-relay", "dup-relay", "relay-scheme", "api-scheme", "api-invalid", "interval-0"] + + ["interval-nan", "catch-up-0"], +) +def test_bad_usage_exits_2_before_any_request( + wire: Any, cli: Any, args: tuple[str, ...], kw: dict[str, Any] +) -> None: + svc = Service() + wire(svc) + rc, out, _ = cli(*args, "--once", **kw) + assert (rc, out, svc.seen) == (2, "", []) + + +def test_sigterm_exits_0_and_restores_handler(wire: Any, cli: Any) -> None: + class Killed(Exception): + pass + + def outer(*_: object) -> None: + raise Killed + + class Terminating(Service): + def handler(self, req: httpx.Request) -> httpx.Response: + os.kill(os.getpid(), signal.SIGTERM) + return super().handler(req) + + wire(Terminating()) + previous = signal.signal(signal.SIGTERM, outer) + try: + rc, _, _ = cli() + assert rc == 0 + assert signal.getsignal(signal.SIGTERM) is outer + finally: + signal.signal(signal.SIGTERM, previous) + + +def test_failure_message_redacts_relay_credentials(wire: Any, cli: Any) -> None: + relay = "https://u:PA'SSW@relay.example/x?apikey=SECRETKEY#SECFRAG" + wire(Service(), relay_client({"relay.example": 503, "b.test": V0})) + rc, out, err = cli("--once", relays=["--relay", relay, "--relay", "https://b.test"]) + assert (rc, out) == (1, "") + assert "failure 1/" in err and "***@relay.example/x" in err + assert "SSW" not in err and "SECRETKEY" not in err and "SECFRAG" not in err + + +PATH_SECRET = "sk_" + "A1b2C3d4E5f6G7h8I9j0K1l2" + + +def test_failure_message_redacts_relay_path_credential(wire: Any, cli: Any) -> None: + relay = f"https://relay.example/key/{PATH_SECRET}" + wire(Service(), relay_client({"relay.example": 503, "b.test": V0})) + rc, out, err = cli("--once", relays=["--relay", relay, "--relay", "https://b.test"]) + assert (rc, out) == (1, "") + assert "failure 1/" in err and "https://relay.example/key/***" in err + assert PATH_SECRET not in err and PATH_SECRET[:8] not in err + + +def test_live_check_redacts_relays_and_error(wire: Any, capsys: pytest.CaptureFixture[str]) -> None: + relay = f"https://u:PASSW@relay.example/{PATH_SECRET}?apikey=SECRETKEY" + wire(Service(), relay_client({"relay.example": 503, "b.test": 503})) + rc = beacon_cli.main(["check", "--live", "--relay", relay, "--relay", "https://b.test"]) + out = json.loads(capsys.readouterr().out) + assert (rc, out["live"]) == (3, "CENSORED(network)") + assert out["relays"] == ["https://***@relay.example/***", "https://b.test"] + assert "relay.example/***" in out["error"] and "503" in out["error"] + text = json.dumps(out) + assert PATH_SECRET not in text and "PASSW" not in text and "SECRETKEY" not in text + + +def test_redact_urls_keeps_short_segments_and_plain_text() -> None: + text = f"GET https://h.test/v1/{PATH_SECRET}/latest failed; see /{PATH_SECRET[:23]}" + assert push.redact_urls(text) == ( + f"GET https://h.test/v1/***/latest failed; see /{PATH_SECRET[:23]}" + ) + host = "https://a-relay-hostname-of-24-chars.test/x" + assert push.redact_urls(host) == host diff --git a/tests/challenge/test_indexed_transport_v2.py b/tests/challenge/test_indexed_transport_v2.py index b40d3cc..77750a5 100644 --- a/tests/challenge/test_indexed_transport_v2.py +++ b/tests/challenge/test_indexed_transport_v2.py @@ -5,6 +5,7 @@ import contextlib import hashlib import importlib +import sqlite3 import sys from pathlib import Path @@ -104,6 +105,8 @@ def test_island_job_maps_indexed_refs_to_whole_hashes(tmp_path: Path, monkeypatc } store = ChallengeStore.__new__(ChallengeStore) store.objects, store.state_dir = objects, tmp_path / "state" + store._db = sqlite3.connect(":memory:") # empty records_v2: first emission of the job + store._db.execute("CREATE TABLE records_v2(run_id, kind, id, data)") put: dict[str, object] = {} monkeypatch.setattr(store, "_tx", contextlib.nullcontext, raising=False) monkeypatch.setattr(store, "_run_v2", lambda *_: manifest, raising=False) diff --git a/tests/challenge/test_island_job_idempotent.py b/tests/challenge/test_island_job_idempotent.py new file mode 100644 index 0000000..c7cd6d6 --- /dev/null +++ b/tests/challenge/test_island_job_idempotent.py @@ -0,0 +1,147 @@ +"""A miner resuming after a crash gets the same staged island job back, never a 500.""" + +from __future__ import annotations + +import importlib.util +import os +import sys +import time +from pathlib import Path + +import pytest + +from hypertrain.auditor.replay import pack_state +from hypertrain.challenge.store import ChallengeError, _stage_once +from hypertrain.data.store import put_large +from hypertrain.protocol.messages_v2 import StartStateV2 +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.model import init_params + +spec = importlib.util.spec_from_file_location( + "island_job_idempotent_fixture", Path(__file__).parent / "test_service_network_v2.py" +) +assert spec is not None and spec.loader is not None +fixtures = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = fixtures +spec.loader.exec_module(fixtures) +network = fixtures.network +HOT = fixtures.HOT[0].ss58 + + +def staged_round(network, monkeypatch, shift: float = 0.0, global_step0: int = 0): + """Real store, objects, dataset and records table; only round/start authority is pinned.""" + store = network.store.for_run(network.manifest.run_id()) + theta = init_params(TrainConfig.from_manifest_v2(network.manifest).model) + theta = {name: value + shift for name, value in theta.items()} + start = StartStateV2( + run_id=network.manifest.run_id(), + w=0, + hotkey=HOT, + theta_hash="11" * 32, + state_object_sha256=put_large(pack_state(theta), store.objects.put), + opt_state_hash="22" * 32, + ef_object_sha256=put_large(pack_state(theta), store.objects.put), + ef_hash="33" * 32, + parent_anchor_hash="44" * 32, + global_step0=global_step0, + anchor_verdict_hash="55" * 32, + ) + real = type(store)._record_v2 + pinned = {"start": start.body(), "round": {"body": {"d_commit": 10, "roster": []}}} + monkeypatch.setattr( + store, + "_record_v2", + lambda run, kind, identity: pinned.get(kind) or real(store, run, kind, identity), + ) + monkeypatch.setattr(store, "_owner_v2", lambda *_: fixtures.COLD[0].ss58) + batch = network.manifest.training.batch_samples() + monkeypatch.setattr(store, "_assignment_v2", lambda *_: tuple(range(batch))) + return store, store.state_dir / "jobs-v2" / "0" / HOT + + +def test_island_job_reemission_is_idempotent(network, monkeypatch): + store, _ = staged_round(network, monkeypatch) + first = store.island_job_v2(network.manifest.run_id(), 0, HOT) + assert store.island_job_v2(network.manifest.run_id(), 0, HOT) == first + + +def test_island_job_reemission_refuses_altered_staged_input(network, monkeypatch): + store, directory = staged_round(network, monkeypatch) + store.island_job_v2(network.manifest.run_id(), 0, HOT) + altered = bytearray((directory / "samples").read_bytes()) + altered[0] ^= 1 + (directory / "samples").write_bytes(altered) + with pytest.raises(ChallengeError) as refused: + store.island_job_v2(network.manifest.run_id(), 0, HOT) + assert refused.value.status == 409 + assert (directory / "samples").read_bytes() == altered + + +def test_island_job_reemission_ignores_crash_leftover_temp_file(network, monkeypatch): + store, directory = staged_round(network, monkeypatch) + first = store.island_job_v2(network.manifest.run_id(), 0, HOT) + # SIGKILL mid-write of the next input: its final name absent, a torn temp left behind. + (directory / "ef_in").unlink() + (directory / ".ef_in.k3j9x2q1.part").write_bytes(b"torn") # the mkstemp temp name + assert store.island_job_v2(network.manifest.run_id(), 0, HOT) == first + assert (directory / "ef_in").read_bytes() == (directory / "start_state").read_bytes() + + +# theta: the staged input bytes differ; global_step0: same bytes, only the job record differs. +@pytest.mark.parametrize("change", [{"shift": 1.0}, {"global_step0": 7}], ids=["theta", "step"]) +def test_island_job_reemission_refuses_different_start_state(network, monkeypatch, change): + store, directory = staged_round(network, monkeypatch) + first = store.island_job_v2(network.manifest.run_id(), 0, HOT) + before = (directory / "start_state").read_bytes() + store, _ = staged_round(network, monkeypatch, **change) + with pytest.raises(ChallengeError) as refused: + store.island_job_v2(network.manifest.run_id(), 0, HOT) + assert refused.value.status == 409 + assert (directory / "start_state").read_bytes() == before + assert store._record_v2(network.manifest.run_id(), "island-job", "0:" + HOT) == first["job"] + + +def test_island_job_route_second_get_is_200_same_body(network, monkeypatch): + staged_round(network, monkeypatch) + first = network.client.get(network.url + "/rounds/0/job/" + HOT) + assert first.status_code == 200, first.text + second = network.client.get(network.url + "/rounds/0/job/" + HOT) + assert second.status_code == 200, second.text + assert second.json() == first.json() + + +def _part(path: Path, data: bytes, age: float = 0.0) -> Path: + path.write_bytes(data) + when = time.time() - age + os.utime(path, (when, when)) + return path + + +def test_stage_once_purges_stale_part_files_of_its_own_name(tmp_path): + stale = _part(tmp_path / ".start_state.abc123.part", b"torn", age=3600) + _stage_once(tmp_path / "start_state", b"x") + assert not stale.exists() + assert (tmp_path / "start_state").read_bytes() == b"x" + + +def test_stage_once_keeps_fresh_part_files_and_other_names(tmp_path): + stale = _part(tmp_path / ".start_state.abc123.part", b"torn", age=3600) + fresh = _part(tmp_path / ".start_state.def456.part", b"live") # a concurrent emission's temp + other = _part(tmp_path / ".ef_in.zzz.part", b"other", age=3600) + _stage_once(tmp_path / "start_state", b"x") + assert not stale.exists() + assert fresh.read_bytes() == b"live" + assert other.read_bytes() == b"other" + assert (tmp_path / "start_state").read_bytes() == b"x" + + +def test_stage_once_purge_does_not_follow_symlinks(tmp_path): + target = _part(tmp_path / "outside", b"keep", age=3600) + job = tmp_path / "job" + job.mkdir() + link = job / ".start_state.link.part" + link.symlink_to(target) + os.utime(link, (target.stat().st_mtime,) * 2, follow_symlinks=False) # stale link too + _stage_once(job / "start_state", b"x") + assert link.is_symlink() + assert target.read_bytes() == b"keep" diff --git a/tests/challenge/test_service_network_v2.py b/tests/challenge/test_service_network_v2.py index 1d1e3b6..5d5356c 100644 --- a/tests/challenge/test_service_network_v2.py +++ b/tests/challenge/test_service_network_v2.py @@ -3453,7 +3453,26 @@ async def drain(): anyio.run(backing_http.aclose) -@pytest.mark.parametrize("invalid", [None, "partial-receipt", "duplicate", "gap", "contradiction"]) +@pytest.mark.parametrize( + "invalid", + [ + None, + "partial-receipt", + "duplicate", + "gap", + "contradiction", + "chunk-only", + "forged", + "held-chunk-only", + "held-forged", + "dead-relay", + "unavailable-relay", + "garbage-relay", + "bad-signature-relay", + "timeout-relay", + "held-corrupt", + ], +) def test_complete_receipt_prefix_rescue_http(network, monkeypatch, tmp_path, invalid): """Exact two-chunk receipt/prefix regression; no live-round/graduation sweep.""" import json @@ -3461,7 +3480,7 @@ def test_complete_receipt_prefix_rescue_http(network, monkeypatch, tmp_path, inv import anyio import httpx - from hypertrain.data.store import LocalFSStore + from hypertrain.data.store import CorruptObjectError, LocalFSStore, StoreError from hypertrain.data.stream_store import StreamStore from hypertrain.protocol import envelope_v2, relay_envelope from hypertrain.protocol.relay_messages import ( @@ -3477,7 +3496,7 @@ def test_complete_receipt_prefix_rescue_http(network, monkeypatch, tmp_path, inv UploadGrant, ) from hypertrain.relay.app import create_app as relay_app - from hypertrain.relay.core import Relay + from hypertrain.relay.core import AssemblyMismatch, Relay spec = importlib.util.spec_from_file_location( "receipt_prefix_shadow", @@ -3500,11 +3519,27 @@ def test_complete_receipt_prefix_rescue_http(network, monkeypatch, tmp_path, inv ) assert response.status_code == 200, response.text payload = b"a" * (4 << 20) + b"b" + # forged: miner-chosen chunk hashes that do not concatenate to delta_hash; an + # unheld delta lets upload_grant_v2 sign such a manifest. + forged = invalid in {"forged", "held-forged"} + sent = b"x" * (4 << 20) + b"y" if forged else payload + chunk_only = invalid in { + "chunk-only", + "forged", + "held-chunk-only", + "held-forged", + "dead-relay", + "unavailable-relay", + "garbage-relay", + "bad-signature-relay", + "timeout-relay", + "held-corrupt", + } chunks = UploadChunkManifest( size=len(payload), chunks=[ - UploadChunk(index=0, off=0, len=4 << 20, chunk_sha256=sha256_hex(payload[:-1])), - UploadChunk(index=1, off=4 << 20, len=1, chunk_sha256=sha256_hex(payload[-1:])), + UploadChunk(index=0, off=0, len=4 << 20, chunk_sha256=sha256_hex(sent[:-1])), + UploadChunk(index=1, off=4 << 20, len=1, chunk_sha256=sha256_hex(sent[-1:])), ], ) net = RelayNetworkManifest( @@ -3612,14 +3647,20 @@ async def uploaded(): async def source(part): yield part - first = await relay.chunk(grant.digest(), 0, source(payload[:-1])) - await relay.chunk(grant.digest(), 1, source(payload[-1:])) - return opportunity, first, await relay.complete(grant.digest()) - - opportunity_env, prefix_env, receipt_env = anyio.run(uploaded) + first = await relay.chunk(grant.digest(), 0, source(sent[:-1])) + if forged: + # The relay signs both chunk acks, then cannot complete bytes missing delta_hash. + with pytest.raises(AssemblyMismatch): + await relay.chunk(grant.digest(), 1, source(sent[-1:])) + last = (await relay.state()).uploads[grant.digest()].chunks["1"] + return opportunity, first, last, last + last = await relay.chunk(grant.digest(), 1, source(sent[-1:])) + return opportunity, first, last, await relay.complete(grant.digest()) + + opportunity_env, prefix_env, last_env, receipt_env = anyio.run(uploaded) opportunity = AcceptedUploadAck.model_validate(opportunity_env.body) prefix = ChunkCustodyAck.model_validate(prefix_env.body) - receipt = RelayReceipt.model_validate(receipt_env.body) + receipt = None if forged else RelayReceipt.model_validate(receipt_env.body) if invalid == "partial-receipt": receipt = receipt.model_copy(update={"size": prefix.len, "delta_hash": prefix.chunk_sha256}) receipt_env = relay_envelope.parse_envelope( @@ -3639,6 +3680,18 @@ async def source(part): ] if invalid == "duplicate": artifacts.insert(2, prefix_env.model_dump(mode="json")) + # Chunk-only custody while the master never held the original: the grant could not + # check that the chunk hashes concatenate to delta_hash, so disabling waits for rescue. + acks = [prefix] + if chunk_only: + artifacts[2] = last_env.model_dump(mode="json") + acks.append(ChunkCustodyAck.model_validate(last_env.body)) + if invalid in {"held-chunk-only", "held-forged"}: + network.store.objects.put(payload) + if invalid == "held-corrupt": # bit rot in the master's copy of the original + corrupt = network.store.objects._path(grant.delta_hash) + corrupt.parent.mkdir(parents=True, exist_ok=True) + corrupt.write_bytes(b"rot") network.push(grant.exp_drand + 1) for auditor in AUDITORS: url = network.url + f"/relay-observers/{grant.digest()}/{auditor.ss58}" @@ -3656,9 +3709,11 @@ async def source(part): grant_hash=grant.digest(), upload_ack_hash=opportunity.digest(), chunk_manifest_hash=grant.chunk_manifest_hash, - chunk_ack_hashes=[prefix.digest()], - receipt_hash=receipt.digest(), - retention_hash=receipt.digest(), + chunk_ack_hashes=[a.digest() for a in acks], + receipt_hash=None if chunk_only else receipt.digest(), + retention_hash=sha256_hex(canonicalize(sorted(a.digest() for a in acks))) + if chunk_only + else receipt.digest(), request=None, response=None, observer_ids=[a.ss58 for a in AUDITORS], @@ -3689,6 +3744,40 @@ async def source(part): anyio.run(backing.aclose) return assert failure.status_code == 200, failure.text + + def disabled(): + return network.store._db.execute( + "SELECT COUNT(*) FROM records_v2 WHERE kind='relay-disabled'" + ).fetchone()[0] + + if invalid == "held-chunk-only": # The held original proves each chunk: disable now. + assert failure.json()["disabled"] and disabled() == 1 + anyio.run(backing.aclose) + return + if chunk_only: + assert not failure.json()["disabled"] and disabled() == 0 + if invalid in {"dead-relay", "unavailable-relay", "garbage-relay"}: + + async def down(sha, size, file): + if invalid == "dead-relay": + raise StoreError("backend unavailable") # relay app answers HTTP 503 + if invalid == "garbage-relay": # relay signs SERVED, then streams wrong bytes + file.write(b"z" * size) + file.seek(0) + return + raise CorruptObjectError("bit rot") # relay signs RetrievalResponse UNAVAILABLE + + relay.backing.get = down + if invalid in {"bad-signature-relay", "timeout-relay"}: + from hypertrain.protocol.envelope import SignatureError + from hypertrain.relay.client import RelayClient + + async def broken(*args, **kwargs): + if invalid == "bad-signature-relay": + raise SignatureError("retrieval response nonce/key/time binding invalid") + raise TimeoutError # what anyio.fail_after raises on a stalled stream + + monkeypatch.setattr(RelayClient, "retrieval", broken) app = relay_app(relay, "x" * 32) monkeypatch.setattr( httpx, @@ -3698,8 +3787,45 @@ async def source(part): rescued = network.client.post( network.url + f"/admin/relay-fallback/{evidence.digest()}", headers=admin() ) - if invalid is not None: + if invalid in { + "dead-relay", + "unavailable-relay", + "garbage-relay", + "bad-signature-relay", + "timeout-relay", + }: + # Two pinned auditors proved the failure and the relay cannot serve the rescue. + assert rescued.status_code == 200, rescued.text + assert rescued.json()["disabled"] and not rescued.json()["rescued"] + row = network.store._db.execute( + "SELECT id,data FROM records_v2 WHERE kind='relay-disabled'" + ).fetchall() + assert [(r[0], json.loads(r[1])) for r in row] == [ + ( + f"local:{grant.assignment_epoch}:k1", + {"key_id": "k1", "evidence_hash": evidence.digest()}, + ) + ] + with network.store._tx(): + network.store._db.execute( + "DELETE FROM records_v2 WHERE kind='relay-assignment' AND id=?", + (f"0:{HOT[0].ss58}",), + ) + network.store._put_record_v2(run_id, "round", "0", {"body": opened}) + network.store._put_record_v2(run_id, "assignment", f"0:{HOT[0].ss58}", {}) + fresh = network.client.get(network.url + f"/rounds/0/relay/{HOT[0].ss58}") + assert fresh.status_code == 503, fresh.text # the only relay is out of selection + assert "no approved healthy relay" in fresh.text + elif invalid in {"chunk-only", "held-corrupt"}: + # held-corrupt: the rotten copy counts as absent, so the rescue decides and disables. + assert rescued.status_code == 200, rescued.text + assert disabled() == 1 + if invalid == "chunk-only": + assert network.store.objects.get(grant.delta_hash) == payload + elif invalid is not None: assert rescued.status_code == 422, rescued.text + if forged: + assert disabled() == 0 else: assert rescued.status_code == 200, rescued.text assert rescued.json()["grant"] == signed_grant @@ -3732,7 +3858,9 @@ async def source(part): "unknown-commit", ], ) -def test_upload_grant_exact_subject_with_two_live_rounds(network, tmp_path, fault): +# held=False: miners commit the delta by ref only; the master first stores it at relay receipt. +@pytest.mark.parametrize("held", [True, False]) +def test_upload_grant_exact_subject_with_two_live_rounds(network, tmp_path, fault, held): import anyio import httpx @@ -3808,7 +3936,8 @@ def test_upload_grant_exact_subject_with_two_live_rounds(network, tmp_path, faul if w == 1 and fault == "wrong-run": committed_envelope = envelope_v2.seal(HOT[0], "CommitV2", "ff" * 32, commit, 10000) with network.store._tx(): - network.store.objects.put(payload) + if held: + network.store.objects.put(payload) network.store._put_record_v2(run_id, "relay-assignment", f"{w}:{HOT[0].ss58}", signed) network.store._put_record_v2(run_id, "commit", f"{w}:{HOT[0].ss58}", committed_envelope) # Exact accepted journal boundary, real signed subjects; no graduation/train. @@ -4092,3 +4221,228 @@ def wire(request): with pytest.raises(CapturedSubject): actor.run_round(1) assert json.loads(canonicalize(saved["commit"]))["body"]["w"] == 1 + + +def test_upload_grant_one_live_grant_when_original_unheld(network): + from hypertrain.protocol import relay_envelope + from hypertrain.protocol.messages_v2 import CommitV2 + from hypertrain.protocol.relay_messages import ( + RelayAssignment, + RelayNetworkManifest, + UploadChunk, + UploadChunkManifest, + ) + + assert network.join().status_code == 200 + run_id = network.manifest.run_id() + registry = network.store._registry_v2(network.manifest) + relay_net = RelayNetworkManifest( + run_id=run_id, + base_manifest_hash=network.manifest.training.training_hash(), + registry_hash=registry.digest(), + specs_hash=sha256_hex(canonicalize([s.body() for s in registry.specs])), + assignment_policy="master-observed-median3-v1", + ) + payload = b"a" * (4 << 20) + b"b" + assignment = RelayAssignment( + w=0, + hotkey=HOT[0].ss58, + primary_id="local", + fallback_ids=[], + assignment_epoch=0, + manifest_hash=relay_net.digest(), + exp_drand=552, + ) + commit = CommitV2.model_validate( + { + "w": 0, + "hotkey": HOT[0].ss58, + "leaf_scheme": "ht-leaf-v1", + "n_leaves": 2, + "leaves_root": "01" * 32, + "metrics_root": "02" * 32, + "final_theta_hash": "03" * 32, + "tokens": 8, + "ef_in_hash": "04" * 32, + "ef_out_hash": "05" * 32, + "delta_hash": sha256_hex(payload), + "delta_bytes": len(payload), + } + ) + with network.store._tx(): + network.store._put_record_v2( + run_id, + "relay-assignment", + f"0:{HOT[0].ss58}", + relay_envelope.seal(COORD, "RelayAssignment", run_id, assignment, 552), + ) + network.store._put_record_v2( + run_id, "commit", f"0:{HOT[0].ss58}", network.signed(HOT[0], "CommitV2", commit) + ) + network.push(385) + + def manifest(data): + return UploadChunkManifest( + size=len(data), + chunks=[ + UploadChunk(index=0, off=0, len=4 << 20, chunk_sha256=sha256_hex(data[:-1])), + UploadChunk(index=1, off=4 << 20, len=1, chunk_sha256=sha256_hex(data[-1:])), + ], + ) + + def request(chunks, expiry=583): + return network.client.post( + network.url + "/upload-grant", + json=relay_envelope.seal(HOT[0], "UploadChunkManifest", run_id, chunks, expiry), + ) + + def grants(): + return network.store._db.execute( + "SELECT COUNT(*) FROM records_v2 WHERE kind='grant'" + ).fetchone()[0] + + # An expired grant for another list (e.g. a past attempt) never blocks a new one. + stale = manifest(b"x" * (4 << 20) + b"y") + with network.store._tx(): + network.store._put_record_v2( + run_id, + "grant", + "ee" * 32, + { + "body": { + "w": 0, + "hotkey": HOT[0].ss58, + "relay_id": "local", + "assignment_epoch": 0, + "chunk_manifest_hash": stale.chunk_manifest_hash(), + "exp_drand": 385, + } + }, + ) + first = request(manifest(payload)) + assert first.status_code == 200, first.text + # Honest retry (fresh miner signature, same chunk list): same signed grant, no new row. + retry = request(manifest(payload), expiry=584) + assert retry.status_code == 200, retry.text + assert retry.json() == first.json() and grants() == 2 + forged = request(stale) + assert forged.status_code == 409, forged.text + assert "another chunk list" in forged.text and grants() == 2 + # Concurrent retries serialize on the store transaction: one grant for both. + from concurrent.futures import ThreadPoolExecutor + + raw = canonicalize( + relay_envelope.seal(HOT[0], "UploadChunkManifest", run_id, manifest(payload), 590) + ) + with ThreadPoolExecutor(2) as pool: + both = list(pool.map(lambda _: network.store.upload_grant_v2(run_id, raw), range(2))) + assert both == [first.json(), first.json()] and grants() == 2 + + +def test_upload_grant_replaced_when_original_held(network): + from hypertrain.protocol import relay_envelope + from hypertrain.protocol.messages_v2 import CommitV2 + from hypertrain.protocol.relay_messages import ( + RelayAssignment, + RelayNetworkManifest, + UploadChunk, + UploadChunkManifest, + ) + + assert network.join().status_code == 200 + run_id = network.manifest.run_id() + registry = network.store._registry_v2(network.manifest) + relay_net = RelayNetworkManifest( + run_id=run_id, + base_manifest_hash=network.manifest.training.training_hash(), + registry_hash=registry.digest(), + specs_hash=sha256_hex(canonicalize([s.body() for s in registry.specs])), + assignment_policy="master-observed-median3-v1", + ) + payload = b"a" * (4 << 20) + b"b" + assignment = RelayAssignment( + w=0, + hotkey=HOT[0].ss58, + primary_id="local", + fallback_ids=[], + assignment_epoch=0, + manifest_hash=relay_net.digest(), + exp_drand=552, + ) + commit = CommitV2.model_validate( + { + "w": 0, + "hotkey": HOT[0].ss58, + "leaf_scheme": "ht-leaf-v1", + "n_leaves": 2, + "leaves_root": "01" * 32, + "metrics_root": "02" * 32, + "final_theta_hash": "03" * 32, + "tokens": 8, + "ef_in_hash": "04" * 32, + "ef_out_hash": "05" * 32, + "delta_hash": sha256_hex(payload), + "delta_bytes": len(payload), + } + ) + with network.store._tx(): + network.store._put_record_v2( + run_id, + "relay-assignment", + f"0:{HOT[0].ss58}", + relay_envelope.seal(COORD, "RelayAssignment", run_id, assignment, 552), + ) + network.store._put_record_v2( + run_id, "commit", f"0:{HOT[0].ss58}", network.signed(HOT[0], "CommitV2", commit) + ) + network.push(385) + + def manifest(data): + return UploadChunkManifest( + size=len(data), + chunks=[ + UploadChunk(index=0, off=0, len=4 << 20, chunk_sha256=sha256_hex(data[:-1])), + UploadChunk(index=1, off=4 << 20, len=1, chunk_sha256=sha256_hex(data[-1:])), + ], + ) + + def request(chunks, expiry=583): + return network.client.post( + network.url + "/upload-grant", + json=relay_envelope.seal(HOT[0], "UploadChunkManifest", run_id, chunks, expiry), + ) + + def grants(): + return network.store._db.execute( + "SELECT COUNT(*) FROM records_v2 WHERE kind='grant'" + ).fetchone()[0] + + # A live grant for a miner-chosen list was signed while the original was unheld. + forged = manifest(b"x" * (4 << 20) + b"y") + with network.store._tx(): + network.store._put_record_v2( + run_id, + "grant", + "ef" * 32, + { + "body": { + "w": 0, + "hotkey": HOT[0].ss58, + "relay_id": "local", + "assignment_epoch": 0, + "chunk_manifest_hash": forged.chunk_manifest_hash(), + "exp_drand": 552, + } + }, + ) + assert request(manifest(payload)).status_code == 409 # original absent: one live grant + network.store.objects.put(payload) + # Original held: the honest list replaces the live forged grant instead of a 409. + honest = request(manifest(payload)) + assert honest.status_code == 200, honest.text + body = honest.json()["body"] + assert body["chunk_manifest_hash"] == manifest(payload).chunk_manifest_hash() + assert body["replaces"] == "ef" * 32 # the relay may hold the forged reservation + assert grants() == 2 + retry = request(manifest(payload), expiry=584) + assert retry.status_code == 200 and retry.json() == honest.json() and grants() == 2 diff --git a/tests/challenge/test_trial_export.py b/tests/challenge/test_trial_export.py new file mode 100644 index 0000000..a686c88 --- /dev/null +++ b/tests/challenge/test_trial_export.py @@ -0,0 +1,365 @@ +"""Admin trial export and the copyable probe-v2 input folder.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import httpx +import pytest + +from hypertrain.data.trial_assignment import trial_samples +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages_v2 import IslandJobV1, JoinChallenge + +_spec = importlib.util.spec_from_file_location( + "trial_export_service_fixture", Path(__file__).parent / "test_service_network_v2.py" +) +assert _spec is not None and _spec.loader is not None +service = importlib.util.module_from_spec(_spec) +sys.modules[_spec.name] = service +_spec.loader.exec_module(service) +network = service.network + + +@pytest.fixture +def trial(network): + response = network.join() + assert response.status_code == 200, response.text + identity = response.json()["admission_id"] + network.push(network.now + 1) + response = network.client.get(network.url + f"/join/{identity}/challenge") + assert response.status_code == 200, response.text + return network, identity, response.json() + + +def export(n, identity): + return n.client.post(n.url + f"/admin/trial/{identity}/export", headers=service.admin()) + + +def test_export_matches_prepare_reference_inputs(trial): + n, identity, envelope = trial + response = export(n, identity) + assert response.status_code == 200, response.text + body = response.json() + job = IslandJobV1.model_validate(body["job"]) + job.manifest.validate_run_id(job.run_id) + challenge = JoinChallenge.model_validate(envelope["body"]) + assert job.sample_ids == list( + trial_samples( + n.manifest, identity, challenge.nonce, n.store._beacon_v2(challenge.seed_beacon) + ) + ) + assert body["challenge"] == challenge.body() + assert body["nonce"] == challenge.nonce + row = n.store._db.execute( + "SELECT trial_epoch FROM admissions_v2 WHERE admission_id=?", (identity,) + ).fetchone() + assert job.w == row["trial_epoch"] + assert ( + job.deadline + == n.manifest.training.beacon.genesis_time + (challenge.deadline_beacon - 1) * 3 + ) + directory = n.store.state_dir / "trials-v2" / challenge.nonce + saved = IslandJobV1.model_validate_json((directory / "job.json").read_bytes()) + saved.manifest.validate_run_id(saved.run_id) + assert saved == job + assert (directory / "job.json").read_text() == job.model_dump_json() + assert set(body["objects"]) == {"start_state", "ef_in", "v0", "samples", "sample_proofs"} + for name, digest in body["objects"].items(): + raw = (directory / name).read_bytes() + assert sha256_hex(raw) == digest + downloaded = n.client.get(n.url + "/objects/" + digest) + assert downloaded.status_code == 200, downloaded.text + assert downloaded.content == raw + for name in ("start_state", "ef_in", "v0"): + assert body["objects"][name] == getattr(job, name + "_sha256") + + +def test_export_is_idempotent_without_challenge_lifecycle_calls(trial, monkeypatch): + n, identity, envelope = trial + _, admission, _ = n.store._services(n.manifest.run_id()) + + def forbidden(*args, **kwargs): + pytest.fail("export must read the stored challenge, not issue one") + + monkeypatch.setattr(admission, "challenge", forbidden) + before = [tuple(row) for row in n.store._db.execute("SELECT * FROM admission_trials")] + first = export(n, identity) + actual_write = Path.write_bytes + + def write(path, raw): + if path.name == "job.json": + pytest.fail("retry must not rewrite immutable job.json") + return actual_write(path, raw) + + monkeypatch.setattr(Path, "write_bytes", write) + second = export(n, identity) + assert first.status_code == second.status_code == 200 + assert first.json() == second.json() + assert first.json()["challenge"] == envelope["body"] + assert sorted(p.name for p in (n.store.state_dir / "trials-v2").iterdir()) == [ + envelope["body"]["nonce"] + ] + assert [tuple(row) for row in n.store._db.execute("SELECT * FROM admission_trials")] == before + + +@pytest.mark.parametrize("known", [True, False]) +def test_export_before_challenge(network, known): + identity = network.join().json()["admission_id"] if known else "ab" * 32 + response = export(network, identity) + assert response.status_code == 409, response.text + assert response.json() == {"detail": "NO_CHALLENGE"} + assert not (network.store.state_dir / "trials-v2").exists() + + +@pytest.mark.parametrize("outcome", ["EXPIRED", None]) +def test_export_requires_open_trial(trial, outcome): + n, identity, _ = trial + with n.store._tx() as db: + if outcome is None: + db.execute("DELETE FROM admission_trial_results") + else: + db.execute("UPDATE admission_trial_results SET outcome=?", (outcome,)) + response = export(n, identity) + assert response.status_code == 409, response.text + assert response.json() == {"detail": "REFERENCE_TRIAL_CLOSED"} + assert not (n.store.state_dir / "trials-v2").exists() + + +def test_export_requires_admin(trial): + n, identity, _ = trial + response = n.client.post(n.url + f"/admin/trial/{identity}/export") + assert response.status_code == 401, response.text + assert not (n.store.state_dir / "trials-v2").exists() + + +def test_export_refuses_changed_job(trial): + n, identity, envelope = trial + directory = n.store.state_dir / "trials-v2" / envelope["body"]["nonce"] + directory.mkdir(parents=True) + path = directory / "job.json" + path.write_bytes(b"different job") + response = export(n, identity) + assert response.status_code == 409, response.text + assert "immutable" in response.json()["detail"] + assert path.read_bytes() == b"different job" + + +def test_export_unavailable_seed(trial): + n, identity, envelope = trial + with n.store._tx() as db: + db.execute("DELETE FROM beacon WHERE round=?", (envelope["body"]["seed_beacon"],)) + response = export(n, identity) + assert response.status_code == 503, response.text + assert "beacon" in response.json()["detail"] + assert not (n.store.state_dir / "trials-v2").exists() + + +@pytest.fixture +def cli(trial, tmp_path, monkeypatch): + from hypertrain.challenge import trial_export + + n, identity, envelope = trial + requests = [] + fault = {} + + def handler(request): + requests.append(request) + if "/objects/" in request.url.path: + assert "authorization" not in request.headers + if fault.get("corrupt") == sum("/objects/" in r.url.path for r in requests): + return httpx.Response(200, content=b"corrupt object") + if fault.get("status") and request.method == "POST": + return httpx.Response( + fault["status"], + text="export unavailable " + request.headers["authorization"], + ) + response = n.client.request( + request.method, request.url.path, headers=request.headers, content=request.content + ) + if response.status_code == 200 and request.method == "POST" and "job" in fault: + body = response.json() + body["job"]["object_paths"]["samples"] = fault["job"] + return httpx.Response(200, json=body) + if fault.get("challenge") and request.url.path.endswith("/challenge"): + body = response.json() + if fault["challenge"] == "signature": + body["sig"] = "00" * 64 + else: + body["body"]["nonce"] = "cd" * 32 + return httpx.Response(200, json=body) + return httpx.Response(response.status_code, content=response.content) + + client_type = httpx.Client + monkeypatch.setattr( + trial_export.httpx, + "Client", + lambda **kwargs: client_type(transport=httpx.MockTransport(handler), **kwargs), + ) + out = tmp_path / "download" / "trial" + args = [ + "--api", + "https://coordinator.test", + "--run-id", + n.manifest.run_id(), + "--admission-id", + identity, + "--admin-token-file", + str(tmp_path / "admin"), + "--out", + str(out), + ] + return trial_export, args, out, requests, fault + + +@pytest.mark.parametrize("existing", [False, True]) +def test_cli_exports_copyable_folder(cli, trial, capsys, record_property, monkeypatch, existing): + module, args, out, requests, _ = cli + n, identity, envelope = trial + if existing: + out.mkdir(parents=True) + monkeypatch.chdir(out) + args[-1] = "." + inode = out.stat().st_ino + assert module.main(args) == 0 + if existing: + assert out.stat().st_ino == inode + assert Path.cwd() == out + output = capsys.readouterr() + assert output.err == "" + report = json.loads(output.out) + record_property("cli_json", output.out.strip()) + job = IslandJobV1.model_validate_json((out / "job.json").read_bytes()) + job.manifest.validate_run_id(job.run_id) + assert (out / "job.json").read_bytes() == canonicalize(job.body()) + assert json.loads((out / "challenge.json").read_bytes()) == envelope + expected = {"job.json", "challenge.json", *job.object_paths.values()} + assert {p.relative_to(out).as_posix() for p in out.rglob("*") if p.is_file()} == expected + assert report == { + "out": "." if existing else str(out), + "run_id": n.manifest.run_id(), + "admission_id": identity, + "nonce": envelope["body"]["nonce"], + "deadline": job.deadline, + "files": {name: sha256_hex((out / name).read_bytes()) for name in expected}, + } + staged = n.store.state_dir / "trials-v2" / report["nonce"] + for relative in job.object_paths.values(): + assert (out / relative).read_bytes() == (staged / relative).read_bytes() + assert len(requests) == 7 + assert requests[0].method == "POST" + assert requests[0].url.path == n.url + f"/admin/trial/{identity}/export" + assert requests[0].headers["authorization"] == service.admin()["Authorization"] + + +@pytest.mark.parametrize( + "fault,error", + [ + ({"corrupt": 5}, "sha256"), + ({"status": 409}, "409 export unavailable"), + ({"status": 302}, "302 export unavailable"), + ({"job": "../escape"}, "path"), + ({"job": "job.json"}, "path"), + ({"job": "ef_in"}, "path"), + ({"challenge": True}, "challenge"), + ({"challenge": "signature"}, "challenge"), + ], +) +def test_cli_failure_leaves_no_partial_folder(cli, capsys, fault, error): + module, args, out, _, faults = cli + faults.update(fault) + assert module.main(args) == 1 + output = capsys.readouterr() + assert output.out == "" + assert error in output.err.lower() + assert service.admin()["Authorization"].removeprefix("Bearer ") not in output.err + assert not out.exists() or list(out.iterdir()) == [] + assert not (out.parent / "escape").exists() + assert list(out.parent.glob(".trial-export-*")) == [] + + +def test_cli_refuses_resolved_path_escape(cli, tmp_path, capsys): + module, args, out, _, fault = cli + outside = tmp_path / "outside" + outside.mkdir() + (outside / "keep").write_bytes(b"existing data") + actual_resolve = Path.resolve + + def resolve(path, *args, **kwargs): + if path.name == "samples" and path.parent.name.startswith(".trial-export-"): + return outside / "escaped" + return actual_resolve(path, *args, **kwargs) + + fault["job"] = "samples" + with pytest.MonkeyPatch.context() as patch: + patch.setattr(Path, "resolve", resolve) + assert module.main(args) == 1 + assert "path" in capsys.readouterr().err + assert not out.exists() + assert sorted(p.name for p in outside.iterdir()) == ["keep"] + assert (outside / "keep").read_bytes() == b"existing data" + + +def test_cli_publication_failure_rolls_back(cli, monkeypatch, capsys): + module, args, out, _, _ = cli + out.mkdir(parents=True) + inode = out.stat().st_ino + actual_rename = Path.rename + published = 0 + + def rename(path, target): + nonlocal published + if path.parent.name.startswith(".trial-export-"): + published += 1 + if published == 2: + raise OSError("simulated publication failure") + return actual_rename(path, target) + + with monkeypatch.context() as patch: + patch.setattr(Path, "rename", rename) + assert module.main(args) == 1 + output = capsys.readouterr() + assert output.out == "" + assert "simulated publication failure" in output.err + assert list(out.iterdir()) == [] + assert out.stat().st_ino == inode + assert list(out.parent.glob(".trial-export-*")) == [] + assert module.main(args) == 0 + + +def test_cli_rejects_malformed_token_before_http(cli, monkeypatch, capsys): + module, args, out, requests, _ = cli + token = service.admin()["Authorization"].removeprefix("Bearer ") + "\nsecond-line" + actual_read = Path.read_text + + def read(path, *a, **kwargs): + return token if path == Path(args[7]) else actual_read(path, *a, **kwargs) + + def client(**kwargs): + raise httpx.LocalProtocolError(f"Illegal header value {('Bearer ' + token).encode()!r}") + + monkeypatch.setattr(Path, "read_text", read) + monkeypatch.setattr(module.httpx, "Client", client) + assert module.main(args) == 1 + output = capsys.readouterr() + assert output.out == "" + safe_error = output.err.strip() == "admin token file must contain one ASCII bearer token" + assert safe_error, "malformed token must be rejected before HTTP without exposing it" + assert requests == [] + assert not out.exists() + + +def test_cli_refuses_nonempty_output(cli, capsys): + module, args, out, requests, _ = cli + out.mkdir(parents=True) + existing = out / "keep" + existing.write_bytes(b"existing data") + assert module.main(args) == 2 + output = capsys.readouterr() + assert "empty" in output.err.lower() + assert existing.read_bytes() == b"existing data" + assert requests == [] diff --git a/tests/challenge/test_v2_weights_bridge.py b/tests/challenge/test_v2_weights_bridge.py new file mode 100644 index 0000000..a88ec26 --- /dev/null +++ b/tests/challenge/test_v2_weights_bridge.py @@ -0,0 +1,73 @@ +"""Does a matured contract-2 reward origin ever reach GET /internal/v1/get_weights?""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +spec = importlib.util.spec_from_file_location( + "v2_weights_bridge_e2e", Path(__file__).parents[1] / "e2e/test_service_network_v2_e2e.py" +) +assert spec is not None and spec.loader is not None +e2e = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = e2e +spec.loader.exec_module(e2e) +network = e2e.network + + +@pytest.mark.slow +def test_v2_matured_reward_reaches_get_weights(network, monkeypatch, tmp_path): + # Given: the shortest existing flow to a live round 0 that finalize_v2 settles. + e2e._graduate_four(network) + e2e._live_checkpoint(network) + store = network.store + run_id = network.manifest.run_id() + assert store._record_v2(run_id, "replayed-finality", "0")["shadow"] is False + mature_at = store._db.execute( + "SELECT MAX(mature_at) FROM escrow_origins WHERE reward_round=0 AND issued>0" + ).fetchone()[0] + # When: beacon rounds pass mature_at (push_beacon matures it), then Cortex asks for weights. + network.push(max(network.now, mature_at) + 1) + # Every round-0 origin, free and still held by its earning owner, is owed to its hotkey. + expected: dict[str, float] = {} + for hotkey, units in store._db.execute( + "SELECT o.hotkey,u.units FROM escrow_units u JOIN escrow_origins o ON o.origin=u.origin " + "WHERE o.reward_round=0 AND o.issued>0 AND u.bucket='available' AND u.units>0" + ): + expected[hotkey] = expected.get(hotkey, 0.0) + units + # Four graduated miners, each awarded an equal quarter of the round-0 budget. + assert sorted(expected.values()) == [250_000.0] * 4 + epoch_at = int(store.clock()) + epoch = (epoch_at - store.params.genesis_unix) // store.params.epoch_seconds + token = tmp_path / "internal" + token.write_text("integration-internal") + route = next( + r for r in network.client.app.routes if getattr(r, "path", "") == "/internal/v1/get_weights" + ) + code = route.endpoint.__code__ + config = route.endpoint.__closure__[code.co_freevars.index("config")].cell_contents + monkeypatch.setattr(config, "internal_token_file", token) + response = network.client.get( + f"/internal/v1/get_weights?epoch={epoch}&epoch_at={epoch_at}", + headers={ + "Authorization": "Bearer integration-internal", + "x-platform-challenge-slug": config.slug, + }, + ) + # Then: exactly the matured v2 units are paid as weight to their hotkeys, the rest burns. + assert response.status_code == 200, response.text + answer = response.json() + assert answer["weights"] == expected, response.text + assert answer["metadata"]["units_paid"] == sum(expected.values()) + assert answer["metadata"]["units_burned_this_epoch"] == 10**6 - sum(expected.values()) + # And: a replayed request for the same epoch returns the same bytes and pays nothing twice. + paid = store._db.execute("SELECT COUNT(*) FROM escrow_events WHERE kind='PAY'").fetchone()[0] + again = network.client.get(response.request.url, headers=response.request.headers) + assert again.content == response.content + assert ( + store._db.execute("SELECT COUNT(*) FROM escrow_events WHERE kind='PAY'").fetchone()[0] + == paid + ) diff --git a/tests/challenge/test_v2_weights_pay.py b/tests/challenge/test_v2_weights_pay.py new file mode 100644 index 0000000..350307e --- /dev/null +++ b/tests/challenge/test_v2_weights_pay.py @@ -0,0 +1,337 @@ +"""ChallengeStore.weights pays matured live v2 reward origins from EscrowV2, once per origin.""" + +from __future__ import annotations + +import importlib.util +import json +import sqlite3 +import sys +from dataclasses import replace +from pathlib import Path + +import pytest + +from hypertrain.challenge.store import ChallengeStore +from hypertrain.data.store import LocalFSStore +from hypertrain.ledger import FULL_SHARE_MASS, Params +from hypertrain.ledger.escrow_v2 import Settlement +from hypertrain.protocol.messages_v2 import EscrowLock, EscrowRelease, EscrowTransfer + +spec = importlib.util.spec_from_file_location( + "v2_weights_pay_escrow", Path(__file__).parents[1] / "ledger/test_escrow_v2.py" +) +assert spec is not None and spec.loader is not None +E = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = E +spec.loader.exec_module(E) +actual_reward = E.actual_reward + + +@pytest.fixture +def paying(tmp_path, actual_reward, monkeypatch): + """A store hosting the real signed live w=0 reward (1e6 units, HOT -> COLD), matured at 20.""" + s, evidence, reward = actual_reward + store = ChallengeStore( + tmp_path / "state", + # v1 genesis deliberately differs from the drand genesis the maturity round uses. + Params("hypertrain", s.manifest.training.beacon.genesis_time - 600, 300, 1, 1), + None, + None, + lambda payload: None, + LocalFSStore(tmp_path / "objects"), + ) + fraud: set[str] = set() + escrow = E.ledger( + s, + store._db, + replay=E.verified_shadow, + settlement=lambda ref: Settlement( + finality_hash=reward.finalize_hash, + closed_dispute_root=E.H, + release_beacon=20, + unresolved=False, + outcome="FRAUD" if ref in fraud else "MATCH", + ), + ) + escrow.mutex = store._lock + store._db.execute( + "INSERT INTO runs VALUES(?,?,?,'created',NULL)", + (s.manifest.run_id(), s.manifest.model_dump_json(), "{}"), + ) + monkeypatch.setattr(store, "_services", lambda run_id: (escrow, None, None)) + escrow.reward_finalize(reward, evidence) + yield store, escrow, reward, evidence, fraud + store.close() + + +def at(escrow, rnd: int) -> int: + """Chain time of drand round rnd for the hosted manifest.""" + beacon = escrow.manifest.training.beacon + return beacon.genesis_time + (rnd - 1) * beacon.period + + +def mature(escrow, reward) -> None: + escrow.mature(E.H, tuple(reward.origin_ids), 20) + + +def mass(answer: dict) -> int: + return int(sum(answer["weights"].values())) + answer["metadata"]["units_burned_this_epoch"] + + +def pays(escrow) -> int: + return escrow.db.execute("SELECT COUNT(*) FROM escrow_events WHERE kind='PAY'").fetchone()[0] + + +def test_matured_reward_paid_once_when_epoch_asked_twice(paying) -> None: + # Given: one matured live origin of 1e6 units. + store, escrow, reward, _, _ = paying + mature(escrow, reward) + # When: Cortex asks the same epoch twice, then the next epoch. + first = store.weights(0, at(escrow, 20), 1) + second = store.weights(0, at(escrow, 20), 2) + later = json.loads(store.weights(1, at(escrow, 21), 3)) + # Then: paid as weight to the origin hotkey exactly once; replay returns the same bytes. + answer = json.loads(first) + assert answer["weights"] == {E.HOT.ss58: 1_000_000.0} + assert answer["metadata"]["units_paid"] == 1_000_000 and mass(answer) == FULL_SHARE_MASS + assert second == first + assert later["weights"] == {} and mass(later) == FULL_SHARE_MASS + assert escrow.balances().paid == 1_000_000 and escrow.balances().available == 0 + assert pays(escrow) == 1 and escrow.balances().conserved() + + +def test_immature_round_not_paid_until_epoch_reaches_mature_at(paying) -> None: + # Given: the origin is available, mature_at=20. + store, escrow, reward, _, _ = paying + mature(escrow, reward) + # When: an epoch at round 19, then one at round 20. + early = json.loads(store.weights(0, at(escrow, 19), 1)) + held = escrow.balances().available + due = json.loads(store.weights(1, at(escrow, 20), 2)) + # Then: nothing before maturity, everything once the epoch's round reaches it. + assert early["weights"] == {} and early["metadata"]["units_burned_this_epoch"] == 10**6 + assert held == 1_000_000 + assert due["weights"] == {E.HOT.ss58: 1_000_000.0} and pays(escrow) == 1 + + +def test_dispute_locked_and_burned_units_never_paid(paying) -> None: + # Given: 400k locked for a contest, 300k admission-locked then slashed (burned), 300k free. + store, escrow, reward, _, fraud = paying + mature(escrow, reward) + origin, cold = reward.origin_ids[0], E.COLD.ss58 + contest = EscrowLock( + operation_id="22" * 32, + owner=cold, + units=400_000, + origin_ids=[origin], + admission_id=None, + dispute_id=E.H, + kind="LOCK_CONTEST", + ) + escrow.lock(contest, signer=cold) + admission = EscrowLock( + operation_id="33" * 32, + owner=cold, + units=300_000, + origin_ids=[origin], + admission_id=E.H, + dispute_id=None, + kind="LOCK_ADMISSION", + ) + escrow.lock(admission, signer=cold) + fraud.add(admission.operation_id) + slash = EscrowRelease( + operation_id="44" * 32, + owner=cold, + units=300_000, + origin_ids=[origin], + admission_id=E.H, + dispute_id=None, + lock_id=admission.operation_id, + finality_hash=reward.finalize_hash, + closed_dispute_root=E.H, + ) + escrow.slash(slash, authority=E.COORD.ss58, now_beacon=20) + # When + answer = json.loads(store.weights(0, at(escrow, 20), 1)) + # Then: only the free 300k is paid; the rest of the epoch mass burns. + assert answer["weights"] == {E.HOT.ss58: 300_000.0} + assert answer["metadata"]["units_burned_this_epoch"] == 700_000 + b = escrow.balances() + assert (b.paid, b.dispute_locked, b.burned, b.available) == (300_000, 400_000, 300_000, 0) + + +def test_v1_and_v2_share_one_full_share_mass(paying) -> None: + # Given: a v1 entitlement of 500k for "a" (b's FAULT slice burns) and 1e6 matured v2 units. + store, escrow, reward, _, _ = paying + mature(escrow, reward) + g = store.params.genesis_unix + for hotkey in ("a", "b"): + store.ledger.commit(0, hotkey, g + 10) + store.ledger.verdict(0, "a", "MATCH", 1000, 10**6, g + 20) + store.ledger.verdict(0, "b", "FAULT", 1000, 10**6, g + 20) + store.ledger.finalize(0, g + 30) + # When: two consecutive epochs once v1 vesting has passed. + first = json.loads(store.weights(3, at(escrow, 20), 1)) + second = json.loads(store.weights(4, at(escrow, 20) + 300, 2)) + # Then: v2 fills only the mass v1 left unpaid, and the remainder pays next epoch. + assert first["weights"] == {"a": 500_000.0, E.HOT.ss58: 500_000.0} + assert first["metadata"]["units_burned_this_epoch"] == 0 and mass(first) == FULL_SHARE_MASS + assert second["weights"] == {E.HOT.ss58: 500_000.0} and mass(second) == FULL_SHARE_MASS + assert escrow.balances().paid == 1_000_000 and escrow.balances().available == 0 + + +def test_pay_rolls_back_when_answer_not_persisted(paying) -> None: + # Given: persisting the epoch answer fails (a crash between pay and persistence). + store, escrow, reward, _, _ = paying + mature(escrow, reward) + store._db.execute( + "CREATE TRIGGER fail_persist BEFORE INSERT ON weights_runs " + "BEGIN SELECT RAISE(ABORT, 'crash'); END" + ) + with pytest.raises(sqlite3.IntegrityError): + store.weights(0, at(escrow, 20), 1) + # Then: nothing was paid. + assert escrow.balances().available == 1_000_000 and pays(escrow) == 0 + # When: the retry (the v1 journal already holds this epoch's answer) persists. + store._db.execute("DROP TRIGGER fail_persist") + retry = json.loads(store.weights(0, at(escrow, 20), 2)) + # Then: the epoch pays exactly once, on the retry. + assert retry["weights"] == {E.HOT.ss58: 1_000_000.0} + assert escrow.balances().paid == 1_000_000 and pays(escrow) == 1 + + +def test_held_units_never_paid(paying) -> None: + # Given: the epoch's round reached mature_at, but the origin has not matured (reward_pending). + store, escrow, reward, _, _ = paying + pending = json.loads(store.weights(0, at(escrow, 20), 1)) + # And then: it matures, 400k is admission-locked and 500k moves to another owner. + mature(escrow, reward) + origin, cold = reward.origin_ids[0], E.COLD.ss58 + lock = E.lock(origin, units=400_000) + escrow.lock(lock, signer=cold) + gift = EscrowTransfer( + operation_id="66" * 32, + owner=cold, + units=500_000, + origin_ids=[origin], + admission_id=None, + dispute_id=None, + recipient=E.OTHER.ss58, + ) + escrow.transfer(gift, signer=cold) + # When + held = json.loads(store.weights(1, at(escrow, 21), 2)) + # Then: only the 100k the earning owner still holds free pays to the origin hotkey. + assert pending["weights"] == {} + assert held["weights"] == {E.HOT.ss58: 100_000.0} and mass(held) == FULL_SHARE_MASS + b = escrow.balances() + assert (b.paid, b.admission_locked, b.available) == (100_000, 400_000, 500_000) + assert escrow.balances(E.OTHER.ss58).available == 500_000 + + +def test_shadow_bootstrap_reward_not_paid(paying) -> None: + # Given: a matured shadow (bootstrap) origin next to the matured live origin, same hotkey. + store, escrow, reward, evidence, _ = paying + shadow = E.sign_actual_shadow(reward, E.reserve_actual_shadow(escrow, evidence)) + escrow.reward_finalize(shadow, replace(evidence, shadow=True)) + escrow.mature("55" * 32, tuple(shadow.origin_ids), 20) + mature(escrow, reward) + # When + answer = json.loads(store.weights(0, at(escrow, 20), 1)) + # Then: only the live origin is paid; the shadow units stay available as collateral. + assert answer["weights"] == {E.HOT.ss58: 1_000_000.0} + held = dict( + escrow.db.execute( + "SELECT origin,bucket FROM escrow_units WHERE units>0 AND origin IN (?,?)", + (reward.origin_ids[0], shadow.origin_ids[0]), + ).fetchall() + ) + assert held == {reward.origin_ids[0]: "paid", shadow.origin_ids[0]: "available"} + + +def test_legacy_epoch_after_upgrade_gap_never_repaid(paying) -> None: + # Given: before the upgrade the v1 journal answered epochs 0 and 2, skipping epoch 1. + store, escrow, reward, _, _ = paying + mature(escrow, reward) + served = {e: store.ledger.get_weights(e, at(escrow, 20) + e * 300, 1) for e in (0, 2)} + # When: the first post-upgrade call asks for the gap epoch, then for the answered ones. + gap = json.loads(store.weights(1, at(escrow, 20) + 300, 2)) + again = {e: store.weights(e, at(escrow, 20) + e * 300, 3) for e in (2, 0)} + fresh = json.loads(store.weights(3, at(escrow, 20) + 900, 4)) + # Then: already-served epochs keep their bytes; the out-of-order gap pays nothing; + # v2 pays on the first epoch past the journal. + assert again == served + assert gap["metadata"]["reason"] == "out_of_order" and gap["weights"] == {} + assert fresh["weights"] == {E.HOT.ss58: 1_000_000.0} and pays(escrow) == 1 + + +def origin(escrow, origin_id: str, hotkey: str, units: int, mature_at: int) -> None: + """A matured live origin held by its issued owner, as reward_finalize + mature leave it.""" + escrow.db.execute( + "INSERT INTO escrow_origins VALUES(?,?,?,0,?)", (origin_id, units, mature_at, hotkey) + ) + for bucket, held in (("reward_pending", 0), ("available", units)): + escrow.db.execute( + "INSERT INTO escrow_units VALUES(?,?,?,'',?)", (origin_id, E.COLD.ss58, bucket, held) + ) + + +def test_oldest_maturity_paid_first_and_same_hotkey_accumulates(paying) -> None: + # Given: the fixture origin (HOT, matures 20, 1e6) and two older ones whose ids sort after it. + store, escrow, reward, _, _ = paying + mature(escrow, reward) + origin(escrow, "ff" * 32, E.HOT.ss58, 300_000, 10) + origin(escrow, "ee" * 32, "other", 500_000, 15) + # When: one epoch with the whole 1e6 mass unpaid by v1. + answer = json.loads(store.weights(0, at(escrow, 20), 1)) + # Then: 300k (round 10) + 500k (round 15) + 200k of the round-20 origin, HOT summed. + assert answer["weights"] == {E.HOT.ss58: 500_000.0, "other": 500_000.0} + assert answer["metadata"]["units_burned_this_epoch"] == 0 + left = escrow.db.execute( + "SELECT units FROM escrow_units WHERE origin=? AND bucket='available'", + (reward.origin_ids[0],), + ).fetchone()[0] + assert left == 800_000 and pays(escrow) == 3 + + +def test_hotkey_past_entry_cap_keeps_units_for_later(paying, monkeypatch) -> None: + # Given: a one-entry weight cap and a second hotkey matured between two HOT origins. + store, escrow, reward, _, _ = paying + mature(escrow, reward) + origin(escrow, "ff" * 32, E.HOT.ss58, 300_000, 10) + origin(escrow, "ee" * 32, "other", 500_000, 15) + monkeypatch.setattr("hypertrain.ledger.MAX_ENTRIES", 1) + # When + answer = json.loads(store.weights(0, at(escrow, 20), 1)) + # Then: the capped hotkey is skipped, HOT already listed still fills the mass. + assert answer["weights"] == {E.HOT.ss58: 1_000_000.0} + assert answer["metadata"]["units_burned_this_epoch"] == 0 + kept = escrow.db.execute( + "SELECT units FROM escrow_units WHERE origin=? AND bucket='available'", ("ee" * 32,) + ).fetchone()[0] + assert kept == 500_000 and pays(escrow) == 2 + + +def test_no_v2_pay_before_any_live_reward_finalized(paying) -> None: + # Given: matured units, but no live reward was ever finalized for the run. + store, escrow, reward, _, _ = paying + mature(escrow, reward) + escrow.db.execute("DELETE FROM escrow_finalized") + # When + answer = json.loads(store.weights(0, at(escrow, 20), 1)) + # Then: the epoch is the plain v1 answer. + assert answer["weights"] == {} and answer["metadata"]["units_burned_this_epoch"] == 10**6 + assert pays(escrow) == 0 + + +def test_out_of_order_epoch_not_completed_with_v2(paying) -> None: + # Given: epoch 1 already answered, then Cortex asks for the older epoch 0. + store, escrow, reward, _, _ = paying + store.weights(1, at(escrow, 19), 1) + mature(escrow, reward) + # When + late = json.loads(store.weights(0, at(escrow, 20), 2)) + # Then: v1 refuses it as out_of_order and v2 does not fill its burned mass. + assert late["metadata"]["reason"] == "out_of_order" and late["weights"] == {} + assert pays(escrow) == 0 and escrow.balances().available == 1_000_000 diff --git a/tests/conftest.py b/tests/conftest.py index 86efe03..1123b67 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,6 +44,59 @@ def build_corpus(root: Path, n_train: int = 2, n_holdout: int = 1) -> Path: return root +# `fast` marker: the CPU mining-path subset scripts/check-fast.sh runs in under 2 minutes +# (protocol, beacon, data, ledger, aggregator, trainer core, auditor, service contract/rounds, +# miner client). Chosen from --durations profiles on 2026-10-18; a file is listed, not a test, +# so new tests in these files are fast by default. FAST_SLOW_TESTS are the few multi-second +# scale/perf cases inside those files that add nothing to the mining path. +FAST_DIRS = ( + "tests/protocol", + "tests/beacon", + "tests/data", + "tests/datasets", + "tests/ledger", + "tests/aggregator", + "tests/ops", +) +FAST_FILES = ( + "tests/test_smoke.py", + "tests/trainer/test_trainer_core.py", + "tests/auditor/test_auditor.py", + "tests/challenge/test_contract.py", + "tests/challenge/test_rounds.py", + "tests/miner/test_miner.py", + "tests/miner/test_segments_e2e.py", + "tests/miner/test_selfcheck.py", + "tests/miner/test_server_keys.py", + "tests/miner/test_cli_v3.py", + "tests/miner/test_update.py", + "tests/miner/test_genesis_cpu_binding.py", + "tests/miner/test_hardware_class.py", + "tests/miner/test_client_network_v2.py", + "tests/miner/test_multigpu_cli.py", + "tests/miner/test_coldkey_ops.py", + "tests/challenge/test_trial_export.py", +) +FAST_SLOW_TESTS = ( + "tests/data/test_parts.py::test_oversize_input_never_publishes_index", + "tests/aggregator/test_weighted_v2.py::" + "test_scale_roster_when_unique_owner_caps_saturate_in_many_stages[1024]", + "tests/ledger/test_ledger.py::test_cap_65536_largest_ties_by_hotkey_rest_carried", + "tests/challenge/test_contract.py::test_get_weights_p99_under_2s_with_65536_hotkeys", +) + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + root = Path(str(config.rootpath)) + for item in items: + rel = Path(item.fspath).relative_to(root).as_posix() + nodeid = f"{rel}::{item.name}" + if ( + rel in FAST_FILES or any(rel.startswith(d + "/") for d in FAST_DIRS) + ) and nodeid not in FAST_SLOW_TESTS: + item.add_marker(pytest.mark.fast) + + @pytest.fixture(scope="session", autouse=True) def corpus_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: """Hermetic corpus: tests and their subprocesses read HYPERTRAIN_DATA_DIR, never /data. diff --git a/tests/datasets/test_pin.py b/tests/datasets/test_pin.py new file mode 100644 index 0000000..b0b8521 --- /dev/null +++ b/tests/datasets/test_pin.py @@ -0,0 +1,231 @@ +import json +import os +from collections.abc import Callable +from pathlib import Path + +import httpx +import numpy as np +import pytest + +from hypertrain.data.store import LocalFSStore +from hypertrain.datasets.__main__ import main +from hypertrain.datasets.shards16 import ShardSet16Manifest, shard_name16, write_shards16 +from hypertrain.protocol.hashing import MerkleTree +from hypertrain.protocol.messages import DatasetSpec + +from .test_datasets import _build, _docs + +RUN = "ab" * 32 +TOKEN = "admin-secret-token" + + +def _holdout(out: Path, seq_len: int) -> ShardSet16Manifest: + return write_shards16([np.arange(seq_len + 1) % 200 + i for i in range(4)], out, seq_len, 4) + + +def _flip(shard: Path) -> None: + raw = bytearray(shard.read_bytes()) + raw[5] ^= 0xFF + shard.write_bytes(raw) + + +def test_pin_writes_service_objects(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + m, _ = _build(tmp_path / "data", _docs()) + hold = _holdout(tmp_path / "hold", m.seq_len) + out = tmp_path / "objects" + argv = ["pin", "--data-dir", str(tmp_path / "data"), "--out-objects", str(out)] + assert main([*argv, "--holdout-dir", str(tmp_path / "hold")]) == 0 + got = json.loads(capsys.readouterr().out) + n, width = m.n_samples, (m.seq_len + 1) * 2 + assert m.n_shards > 1 and got["n_samples"] == n + store = LocalFSStore(out) + samples = store.get(got["samples_hash"]) + proofs = json.loads(store.get(got["proofs_hash"])) + assert len(samples) == n * width and len(proofs) == n + for i in (0, n // 2, n - 1): + assert MerkleTree.verify( + samples[i * width : (i + 1) * width], + i, + [bytes.fromhex(p) for p in proofs[i]], + bytes.fromhex(m.merkle_root), + n, + ) + assert got["dataset_fragment"] == { + "merkle_root": m.merkle_root, + "depth": m.depth, + "n_samples": n, + "sample_format": m.sample_format, + "shard_uri_template": m.shard_uri_template, + "shard_sha256_root": m.shard_sha256_root, + "assign_unit": m.unit, + "unit_sha256_root": m.unit_sha256_root, + "holdout_commit": hold.merkle_root, + } + DatasetSpec.model_validate(got["dataset_fragment"]) + + +@pytest.mark.parametrize( + ("case", "want"), + [ + ("data", "shard-00001.u16: sha256 mismatch"), + ("holdout", "holdout shards differ"), + ("holdout-seq", "holdout seq_len 8"), + ], +) +def test_pin_refuses_bad_shards( + tmp_path: Path, capsys: pytest.CaptureFixture[str], case: str, want: str +) -> None: + m, _ = _build(tmp_path / "data", _docs()) + _holdout(tmp_path / "hold", 8 if case == "holdout-seq" else m.seq_len) + if case == "data": + _flip(tmp_path / "data" / shard_name16(1)) + elif case == "holdout": + _flip(tmp_path / "hold" / shard_name16(0)) + out = tmp_path / "objects" + argv = ["pin", "--data-dir", str(tmp_path / "data"), "--out-objects", str(out)] + assert main([*argv, "--holdout-dir", str(tmp_path / "hold")]) == 1 + err = capsys.readouterr().err + assert want in err and (case != "data" or m.merkle_root in err) + assert not [p for p in out.rglob("*") if p.is_file()] + + +def _hosted(tmp_path: Path, token: str = TOKEN) -> tuple[list[str], Path, str, str]: + """`register` argv against a service state dir hosting RUN as a sub-store.""" + pinned = LocalFSStore(tmp_path / "pinned") + samples, proofs = pinned.put(b"\x01\x00" * 18), pinned.put(b'[["00"]]') + state = tmp_path / "state" + (state / "runs" / RUN / "objects").mkdir(parents=True) + (state / "challenge.db").touch() + (state / "runs" / RUN / "challenge.db").touch() + (tmp_path / "admin.token").write_text(token + "\n") + argv = ["register", "--api", "https://master.test", "--run-id", RUN] + argv += ["--admin-token-file", str(tmp_path / "admin.token"), "--objects", str(pinned.root)] + argv += ["--state-dir", str(state), "--samples-hash", samples, "--proofs-hash", proofs] + return argv, state, samples, proofs + + +def _serve( + monkeypatch: pytest.MonkeyPatch, handler: Callable[[httpx.Request], httpx.Response] +) -> None: + real = httpx.Client + monkeypatch.setattr( + httpx, "Client", lambda **kw: real(transport=httpx.MockTransport(handler), **kw) + ) + + +def test_register_copies_objects_then_links_dataset( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + argv, state, samples, proofs = _hosted(tmp_path) + target = LocalFSStore(state / "runs" / RUN / "objects") + seen: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + for h in (samples, proofs): # linked objects must already be in the run's store + assert target.get(h) == LocalFSStore(tmp_path / "pinned").get(h) + return httpx.Response(200, json={"status": "configured"}) + + _serve(monkeypatch, handler) + before = set(state.rglob("*")) + assert main(argv) == 0 + [req] = seen + assert (req.method, str(req.url)) == ("PUT", f"https://master.test/v2/runs/{RUN}/admin/dataset") + assert json.loads(req.content) == {"samples_hash": samples, "proofs_hash": proofs} + assert req.headers["authorization"] == f"Bearer {TOKEN}" + assert all(target.root in (p, *p.parents) for p in set(state.rglob("*")) - before) + out = capsys.readouterr() + assert "configured" in out.out and TOKEN not in out.out + out.err + + +@pytest.mark.parametrize( + ("status", "detail"), [(422, "expected dataset object hashes"), (503, "store unavailable")] +) +def test_register_reports_service_refusal( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, + status: int, + detail: str, +) -> None: + argv, *_ = _hosted(tmp_path) + _serve(monkeypatch, lambda _r: httpx.Response(status, json={"detail": detail})) + assert main(argv) == 1 + out = capsys.readouterr() + assert f"HTTP {status}" in out.err and detail in out.err + assert TOKEN not in out.out + out.err + + +@pytest.mark.parametrize( + ("case", "want"), + [ + ("unhosted", "challenge.db not found"), + ("traversal", "challenge.db not found"), + ("no-db", "challenge.db not found"), + ("foreign-uid", "another uid"), + ("no-objects", "not a directory"), + ("token", "printable ASCII"), + ], +) +def test_register_refuses_before_any_write( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, + case: str, + want: str, +) -> None: + argv, state, *_ = _hosted(tmp_path, TOKEN + "\nline2" if case == "token" else TOKEN) + i = argv.index + if case == "unhosted": + argv[i("--run-id") + 1] = "cd" * 32 + elif case == "traversal": + argv[i("--run-id") + 1] = ".." + elif case == "no-db": + (state / "runs" / RUN / "challenge.db").unlink() + elif case == "foreign-uid": + monkeypatch.setattr(os, "geteuid", lambda: os.stat(state).st_uid + 1) + elif case == "no-objects": + argv[i("--objects") + 1] = str(tmp_path / "missing") + + def handler(_r: httpx.Request) -> httpx.Response: + raise AssertionError("refused before any request") + + _serve(monkeypatch, handler) + before = set(tmp_path.rglob("*")) + assert main(argv) == 1 + assert set(tmp_path.rglob("*")) == before + out = capsys.readouterr() + assert want in out.err and TOKEN not in out.out + out.err + + +@pytest.mark.parametrize("link", ["run", "objects", "prefix"]) +def test_register_refuses_symlink_into_root_store( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, + link: str, +) -> None: + argv, state, *_ = _hosted(tmp_path) + (state / "objects").mkdir() + run = state / "runs" / RUN + if link == "run": + (run / "objects").rmdir() + (run / "challenge.db").unlink() + run.rmdir() + run.symlink_to(state) + elif link == "objects": + (run / "objects").rmdir() + (run / "objects").symlink_to(state / "objects") + else: + for h in argv[argv.index("--samples-hash") + 1], argv[argv.index("--proofs-hash") + 1]: + (state / "objects" / h[:2]).mkdir(exist_ok=True) + if not (run / "objects" / h[:2]).exists(): + (run / "objects" / h[:2]).symlink_to(state / "objects" / h[:2]) + + def handler(_r: httpx.Request) -> httpx.Response: + raise AssertionError("refused before any request") + + _serve(monkeypatch, handler) + assert main(argv) == 1 + assert not [p for p in (state / "objects").rglob("*") if p.is_file()] + assert "root store" in capsys.readouterr().err diff --git a/tests/miner/test_coldkey_ops.py b/tests/miner/test_coldkey_ops.py new file mode 100644 index 0000000..8432c72 --- /dev/null +++ b/tests/miner/test_coldkey_ops.py @@ -0,0 +1,537 @@ +"""Operator-side coldkey signing; real CLI, protocol models and sr25519 signatures.""" + +from __future__ import annotations + +import json +import os +from dataclasses import asdict +from types import SimpleNamespace + +import httpx +import pytest + +from hypertrain.challenge.admission import AdmissionStatus +from hypertrain.challenge.admission_store import AdmissionRecord +from hypertrain.challenge.trust_v2 import FundedStatus +from hypertrain.ledger.escrow_v2 import genesis_allocation_hash +from hypertrain.miner import cli +from hypertrain.miner.admission import sign_join +from hypertrain.protocol.envelope_v2 import ( + join_signing_message, + parse_envelope, + seal, + verify_envelope, + verify_join, +) +from hypertrain.protocol.example import example_manifest +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair, verify +from hypertrain.protocol.messages_v2 import ( + AdmissionPolicyV2, + ArtifactLimits, + EconomicsPolicyV2, + EscrowLock, + HardwareHint, + JoinRequest, + RunManifestV2, +) + +API = "https://service.test" +HOT_SEED = bytes(range(32, 64)) +COLD_SEED = bytes(range(64, 96)) +REQUEST_ID = "11" * 32 +ADMISSION_ID = "44" * 32 +EVENT_HASH = "55" * 32 +ORIGINS = ["33" * 32, "22" * 32] +NOW = 37 +EXPIRY = 5000 +UNITS = 1250 + + +@pytest.fixture +def operator(tmp_path, monkeypatch): + for name in list(os.environ): + if name.startswith("HYPERTRAIN_MINER_"): + monkeypatch.delenv(name) + hot, cold = Keypair(HOT_SEED), Keypair(COLD_SEED) + coord, other = Keypair(bytes(range(32))), Keypair(bytes(range(96, 128))) + hotfile, coldfile = tmp_path / "hot.key", tmp_path / "cold.key" + for path, seed in ((hotfile, HOT_SEED), (coldfile, COLD_SEED)): + path.write_text(seed.hex()) + path.chmod(0o600) + admission = AdmissionPolicyV2( + q_base="0000803e", + E=4, + clean_finalizations=12, + work_screen_epoch_rounds=1000, + max_pending=32, + max_trial_replays=2, + max_trials_per_beacon=2, + join_per_beacon=2, + join_burst=4, + ip_prefix_per_beacon=32, + ip_prefix_burst=64, + challenge_rounds=100, + artifact_limits=ArtifactLimits( + max_object_bytes=1_000_000, max_body_bytes=65536, max_chunk_manifest_bytes=1048576 + ), + ) + policy = EconomicsPolicyV2( + ledger_mode="test", + G_max_units=100, + R_collectible_units=100, + S_min_units=1000, + beta_ppm=1_000_000, + gammaV_units=0, + s_lower_ppm=1_000_000, + q_floor=1_000_000, + genesis_allocation_hash=genesis_allocation_hash( + [(cold.ss58, 20000, 1), (other.ss58, 10000, 1)] + ), + reward_authority=coord.ss58, + round_reward_units=1_000_000, + max_total_issuance=20_000_000, + shadow_bootstrap_rounds=12, + ) + training = example_manifest().body() + training["model"]["param_count"] = 4_329_216 # v2 shape count; no tensors needed. + training["reference_spec"]["layout"].update(n_gpus=2, dp_size=2) + manifest = RunManifestV2.model_validate( + { + "manifest_version": 2, + "training": training, + "network": { + "admission_policy_hash": admission.digest(), + "economics_policy_hash": policy.digest(), + "aggregation_policy_hash": REQUEST_ID, + "dispute_policy_hash": REQUEST_ID, + "audit_policy_hash": REQUEST_ID, + "relay_registry_hash": REQUEST_ID, + "audit_mode": "anchored-full", + "full_anchor_version": 1, + "capabilities": ["island-replay", "all-level-disputes", "transport-receipts"], + }, + } + ) + config = tmp_path / "miner.toml" + config.write_text( + "".join( + f"{name} = {json.dumps(str(value))}\n" + for name, value in { + "api": API, + "keyfile": hotfile, + "workdir": tmp_path / "gpu-work", + "state_source": "unused", + "image_digest": manifest.training.reference_spec.image_digest, + "run_id": manifest.run_id(), + "owner_hotkey": coord.ss58, + "device": "cuda", + }.items() + ) + ) + return SimpleNamespace( + hot=hot, + cold=cold, + other=other, + coord=coord, + hotfile=hotfile, + coldfile=coldfile, + config=config, + manifest=manifest, + policy=policy, + run_id=manifest.run_id(), + ) + + +@pytest.fixture +def invoke(capsys, caplog): + def call(argv, *, parser_error=False): + try: + try: + code = cli.main(argv) + except SystemExit as error: + if not parser_error: + raise + code = error.code + finally: + captured = capsys.readouterr() + if any(COLD_SEED.hex() in text for text in (captured.out, captured.err, caplog.text)): + pytest.fail("coldkey seed exposed in command output", pytrace=False) + return code, captured.out, captured.err + caplog.text + + return call + + +def _record(op, owner=None): + return AdmissionRecord( + admission_id=ADMISSION_ID, + hotkey=op.hot.ss58, + coldkey=owner or op.cold.ss58, + state="PROBATION", + clean_count=4, + canary_blocks=(1,), + strikes=0, + pending_dispute=False, + reason="FULL_ROUND_MATCH", + policy_hash=op.manifest.network.admission_policy_hash, + screen_until=EXPIRY, + challenge_json=None, + ) + + +def _service(op, monkeypatch, *, post_status=200, owner=None, post_content=None): + requests = [] + root = f"/v2/runs/{op.run_id}" + manifest_envelope = seal(op.coord, "RunManifestV2", op.run_id, op.manifest, EXPIRY) + + def route(request): + requests.append(request) + assert request.url.host == "service.test" + if request.method == "GET" and request.url.path == root: + return httpx.Response( + 200, + json={ + "run_id": op.run_id, + "manifest_envelope": manifest_envelope, + "now_round": NOW, + }, + ) + if request.method == "GET" and request.url.path == root + "/objects/" + op.policy.digest(): + return httpx.Response(200, content=canonicalize(op.policy.body())) + if request.method == "GET" and request.url.path == root + "/admission/" + op.hot.ss58: + locked = 1475 if any(r.method == "POST" for r in requests) else 125 + funding = FundedStatus( + op.run_id, op.hot.ss58, ADMISSION_ID, "test", locked, 0, EVENT_HASH, True + ) + return httpx.Response( + 200, + json=asdict( + AdmissionStatus( + _record(op, owner), + funding, + 250_000, + 1_000_000, + locked >= 1000, + locked < 1000, + ) + ), + ) + if request.method == "POST" and request.url.path in (root + "/join", root + "/escrow/lock"): + if not 200 <= post_status < 300: + return httpx.Response(post_status, text="blocked-by-service") + if post_content is not None: + return httpx.Response(post_status, content=post_content) + body = ( + asdict(_record(op)) + if request.url.path.endswith("/join") + else {"event_hash": EVENT_HASH} + ) + return httpx.Response(post_status, json=body) + pytest.fail(f"unexpected HTTP request: {request.method} {request.url.path}") + + real_client = httpx.Client + transport = httpx.MockTransport(route) + monkeypatch.setattr( + httpx, "Client", lambda *args, **kwargs: real_client(*args, transport=transport, **kwargs) + ) + return requests + + +def _single_join(op): + return sign_join( + op.hot, + op.cold, + run_id=op.run_id, + request_id=REQUEST_ID, + expires_beacon=EXPIRY, + policy_hash=op.manifest.network.admission_policy_hash, + hardware_hint=HardwareHint(device_name="cuda", device_count=2, driver="advisory"), + ) + + +def _prepared(op, directory): + body = {**_single_join(op).body(), "cold_sig": "0" * 128} + path = directory / "join.json" + path.write_bytes(canonicalize(body)) + return path + + +def _prepare_args(op, out): + return [ + "join-v2", + "--config", + str(op.config), + "--request-id", + REQUEST_ID, + "--expiry", + str(EXPIRY), + "--coldkey-ss58", + op.cold.ss58, + "--prepare", + str(out), + ] + + +def _cosign_args(op, request): + return [ + "join-v2-cosign", + "--request", + str(request), + "--coldkey", + str(op.coldfile), + "--api", + API, + ] + + +def _lock_args(op, origins=ORIGINS): + return [ + "escrow-lock", + "--api", + API, + "--run-id", + op.run_id, + "--coldkey", + str(op.coldfile), + "--hotkey-ss58", + op.hot.ss58, + "--units", + str(UNITS), + *[arg for origin in origins for arg in ("--origin", origin)], + ] + + +def test_two_step_join_equals_single_step(operator, tmp_path, monkeypatch, invoke): + op = operator + requests = _service(op, monkeypatch, post_status=201) + artifact = tmp_path / "prepared.json" + code, out, _ = invoke(_prepare_args(op, artifact)) + assert code == 0 + prepared = JoinRequest.model_validate_json(artifact.read_bytes()) + message = join_signing_message(prepared) + assert json.loads(out) == {"prepared": str(artifact), "message_sha256": sha256_hex(message)} + assert prepared.cold_sig == "0" * 128 + assert verify(op.hot.public, message, bytes.fromhex(prepared.hot_sig)) + assert not any(r.method == "POST" for r in requests) + + # The operator has neither miner.toml nor the GPU host's hotkey seed. + op.config.unlink() + op.hotfile.unlink() + code, out, _ = invoke(_cosign_args(op, artifact)) + assert code == 0 and json.loads(out)["admission_id"] == ADMISSION_ID + posts = [r for r in requests if r.method == "POST"] + assert len(posts) == 1 and posts[0].url.path == f"/v2/runs/{op.run_id}/join" + submitted = JoinRequest.model_validate_json(posts[0].content) + single = _single_join(op) + assert canonicalize(submitted.model_dump(exclude={"hot_sig", "cold_sig"})) == canonicalize( + single.model_dump(exclude={"hot_sig", "cold_sig"}) + ) + assert submitted.hot_sig == prepared.hot_sig + assert verify_join(single, op.run_id, NOW) + assert verify_join(submitted, op.run_id, NOW) + + +@pytest.mark.parametrize("fault", ["hot_sig", "coldkey"]) +def test_cosign_rejects_bad_hot_signature_or_wrong_coldkey( + operator, tmp_path, monkeypatch, invoke, fault +): + op = operator + requests = _service(op, monkeypatch) + artifact = _prepared(op, tmp_path) + body = json.loads(artifact.read_bytes()) + if fault == "hot_sig": + body["hot_sig"] = "0" * 128 + else: + body["coldkey"] = op.other.ss58 + unsigned = JoinRequest.model_validate(body) + body["hot_sig"] = op.hot.sign(join_signing_message(unsigned)).hex() + artifact.write_bytes(canonicalize(body)) + code, _, _ = invoke(_cosign_args(op, artifact)) + assert code == 2 + assert not any(r.method == "POST" for r in requests) + + +def test_prepare_refuses_existing_output(operator, tmp_path, monkeypatch, invoke): + requests = _service(operator, monkeypatch) + artifact = tmp_path / "prepared.json" + artifact.write_bytes(b"existing operator data") + code, _, _ = invoke(_prepare_args(operator, artifact)) + assert code == 2 + assert artifact.read_bytes() == b"existing operator data" + assert not any(r.method == "POST" for r in requests) + + +def test_prepare_and_coldkey_are_mutually_exclusive(operator, tmp_path, monkeypatch, capsys): + requests = _service(operator, monkeypatch) + with pytest.raises(SystemExit) as error: + cli.main( + _prepare_args(operator, tmp_path / "join.json") + ["--coldkey", str(operator.coldfile)] + ) + captured = capsys.readouterr() + if COLD_SEED.hex() in captured.out + captured.err: + pytest.fail("coldkey seed exposed in command output", pytrace=False) + assert error.value.code == 2 and "not allowed with argument" in captured.err + assert not any(r.method == "POST" for r in requests) + assert not (tmp_path / "join.json").exists() + + +def test_escrow_lock_signs_owner_and_reuses_operation_id_on_reordered_retry( + operator, monkeypatch, invoke +): + op = operator + requests = _service(op, monkeypatch) + for origins, expiry in ((ORIGINS, None), (list(reversed(ORIGINS)), EXPIRY)): + args = _lock_args(op, origins) + if expiry is not None: + args += ["--expiry", str(expiry)] + start = len(requests) + code, out, _ = invoke(args) + assert code == 0 + assert json.loads(out) == { + "event_hash": EVENT_HASH, + "locked_units": 1475, + "required_units": 1000, + "eligible": True, + } + call = requests[start:] + index = next(i for i, r in enumerate(call) if r.method == "POST") + assert any( + r.method == "GET" and r.url.path.endswith("/admission/" + op.hot.ss58) + for r in call[index + 1 :] + ) + posts = [r for r in requests if r.method == "POST"] + assert len(posts) == 2 + assert canonicalize(parse_envelope(posts[0].content).body) == canonicalize( + parse_envelope(posts[1].content).body + ) + expected_id = sha256_hex( + canonicalize( + ["ht-escrow-lock-cli", op.run_id, ADMISSION_ID, op.cold.ss58, UNITS, sorted(ORIGINS)] + ) + ) + for request, expiry in zip(posts, (NOW + 1000, EXPIRY), strict=True): + assert request.url.path == f"/v2/runs/{op.run_id}/escrow/lock" + assert verify_envelope(request.content) + envelope = parse_envelope(request.content) + assert (envelope.type, envelope.run_id, envelope.signer, envelope.exp_drand) == ( + "EscrowLock", + op.run_id, + op.cold.ss58, + expiry, + ) + lock = EscrowLock.model_validate(envelope.body) + assert lock.operation_id == expected_id + assert lock.owner == op.cold.ss58 and lock.units == UNITS + assert sorted(lock.origin_ids) == sorted(ORIGINS) + assert (lock.kind, lock.admission_id, lock.dispute_id) == ( + "LOCK_ADMISSION", + ADMISSION_ID, + None, + ) + + +def test_escrow_lock_refuses_another_admission_owner(operator, monkeypatch, invoke): + requests = _service(operator, monkeypatch, owner=operator.other.ss58) + code, _, _ = invoke(_lock_args(operator)) + assert code == 2 + assert not any(r.method == "POST" for r in requests) + + +def test_escrow_dry_run_emits_only_unsigned_operation(operator, monkeypatch, invoke): + op = operator + requests = _service(op, monkeypatch) + code, out, _ = invoke(_lock_args(op) + ["--dry-run"]) + assert code == 0 + lock = EscrowLock.model_validate_json(out) + assert (lock.owner, lock.units, lock.admission_id, lock.dispute_id, lock.kind) == ( + op.cold.ss58, + UNITS, + ADMISSION_ID, + None, + "LOCK_ADMISSION", + ) + assert sorted(lock.origin_ids) == sorted(ORIGINS) + assert not any(r.method == "POST" for r in requests) + + +@pytest.mark.parametrize( + ("changes", "locked", "expected"), + [ + ({}, 0, 1000), # max(1000, 10 * 100, ceil(100 / 1)) + ({"S_min_units": 2000}, 0, 2000), # Policy floor wins. + ({"R_collectible_units": 150}, 0, 1500), # Training floor: 10 * 150. + ({"G_max_units": 1001, "s_lower_ppm": 333333}, 0, 3004), # ceil(1001 / .333333) + ({"G_max_units": 1000, "s_lower_ppm": 400000, "beta_ppm": 1500000}, 0, 1667), + # Zero denominator: G + already locked + 1 = 100 + 5000 + 1, never fundable. + ({"s_lower_ppm": 0}, 5000, 5101), + ({}, 5000, 1000), # Total requirement, not additional units after existing locks. + ], +) +def test_required_units_matches_admission_arithmetic(operator, changes, locked, expected): + from hypertrain.miner.coldkey_ops import required_units + + policy = EconomicsPolicyV2.model_validate({**operator.policy.body(), **changes}) + assert operator.manifest.training.verify.s_min_reward_multiple == 10 + assert required_units(policy, operator.manifest, locked) == expected + + +@pytest.mark.parametrize("command", ["join", "escrow"]) +@pytest.mark.parametrize("status", [202, 302, 422]) +def test_http_status_maps_to_exit_code_and_preserves_error_body( + operator, tmp_path, monkeypatch, invoke, command, status +): + op = operator + requests = _service(op, monkeypatch, post_status=status) + args = _cosign_args(op, _prepared(op, tmp_path)) if command == "join" else _lock_args(op) + code, out, diagnostic = invoke(args) + assert code == (0 if 200 <= status < 300 else 1) + assert len([r for r in requests if r.method == "POST"]) == 1 + if not 200 <= status < 300: + assert str(status) in out + diagnostic + assert "blocked-by-service" in out + diagnostic + + +@pytest.mark.parametrize(("status", "body"), [(204, b""), (200, b"accepted"), (200, b"[]")]) +def test_cosign_accepts_all_success_response_bodies( + operator, tmp_path, monkeypatch, invoke, status, body +): + requests = _service(operator, monkeypatch, post_status=status, post_content=body) + code, out, _ = invoke(_cosign_args(operator, _prepared(operator, tmp_path))) + assert code == 0 + assert json.loads(out) == ([] if body == b"[]" else body.decode()) + assert len([r for r in requests if r.method == "POST"]) == 1 + + +def test_invalid_api_returns_miner_error(operator, tmp_path, monkeypatch, invoke): + requests = _service(operator, monkeypatch) + args = _cosign_args(operator, _prepared(operator, tmp_path)) + args[args.index("--api") + 1] = "https://service.test:invalid" + code, _, diagnostic = invoke(args) + assert code == 2 and "MinerError" in diagnostic + assert requests == [] + + +@pytest.mark.parametrize("command", ["prepare", "cosign", "escrow"]) +def test_invalid_input_returns_miner_error_without_raw_validation_input( + operator, tmp_path, monkeypatch, invoke, command +): + op = operator + requests = _service(op, monkeypatch) + marker = "private-validation-input" + if command == "prepare": + args = _prepare_args(op, tmp_path / "join.json") + args[args.index("--request-id") + 1] = marker + elif command == "cosign": + artifact = _prepared(op, tmp_path) + body = json.loads(artifact.read_bytes()) + body["hardware_hint"]["device_count"] = marker + artifact.write_bytes(canonicalize(body)) + args = _cosign_args(op, artifact) + else: + args = _lock_args(op, [marker]) + code, out, diagnostic = invoke(args) + assert code == 2 + assert "MinerError" in diagnostic + assert marker not in out + diagnostic + assert not any(r.method == "POST" for r in requests) diff --git a/tests/ops/__init__.py b/tests/ops/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/ops/test_manifest_v2.py b/tests/ops/test_manifest_v2.py new file mode 100644 index 0000000..69245ba --- /dev/null +++ b/tests/ops/test_manifest_v2.py @@ -0,0 +1,279 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import httpx +import pytest +from fastapi.testclient import TestClient +from tests.challenge.conftest import ADMIN, Master, beacon_payload, bearer, verify_fixture + +from hypertrain.challenge.app import Config, create_app +from hypertrain.ops.manifest_v2 import main +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages_v2 import RunManifestV2 +from hypertrain.trainer.compress import state_hash +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.model import init_params + + +@pytest.fixture +def build_args(tmp_path: Path) -> list[str]: + inputs = tmp_path / "inputs" + assert main(["example", "--out", str(inputs)]) == 0 + owner_seed = tmp_path / "owner.seed" + owner_seed.write_bytes(b"\x77" * 32) + owner_seed.chmod(0o600) + return [ + "build", + "--training", + str(inputs / "training.json"), + "--policies", + str(inputs / "policies.json"), + "--relays", + str(inputs / "relays.json"), + "--owner-seed", + str(owner_seed), + "--exp-round", + "1000000", + "--state-dir", + str(tmp_path / "state"), + "--out", + str(tmp_path / "output"), + ] + + +def test_build_then_service_accepts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + build_args: list[str], + capsys: pytest.CaptureFixture[str], +) -> None: + output = tmp_path / "output" + state = tmp_path / "state" + training_path = tmp_path / "inputs" / "training.json" + training = json.loads(training_path.read_text()) + coord_seed = bytes(range(32)) + training["coord_pubkey"] = Keypair(coord_seed).ss58 + training_path.write_text(json.dumps(training)) + policies_path = tmp_path / "inputs" / "policies.json" + policies = json.loads(policies_path.read_text()) + policies["economics"]["reward_authority"] = training["coord_pubkey"] + policies_path.write_text(json.dumps(policies)) + assert main(build_args) == 0 + index = json.loads((output / "index.json").read_text()) + manifest = RunManifestV2.model_validate_json((output / "manifest.json").read_bytes()) + captured = capsys.readouterr() + envelope = json.loads((output / "manifest-envelope.json").read_text()) + assert captured.err == "" + assert envelope["sig"] not in captured.out + assert (b"\x77" * 32).hex() not in captured.out + assert (b"\x77" * 32).decode() not in captured.out + assert manifest.run_id() == index["run_id"] + assert manifest.training.init_state_hash == state_hash( + init_params(TrainConfig.from_manifest_v2(manifest).model) + ) + assert index["owner_ss58"] == Keypair(b"\x77" * 32).ss58 + assert set(index["objects"]) == { + "admission", + "economics", + "aggregation", + "audit", + "dispute", + "relay_registry", + } + expected_files = { + "manifest.json", + "manifest-envelope.json", + "policy-admission.json", + "policy-economics.json", + "policy-aggregation.json", + "policy-audit.json", + "policy-dispute.json", + "relay-registry.json", + } + assert set(index["files"]) == expected_files + assert {p.name for p in output.iterdir()} == expected_files | {"index.json"} + for name, digest in index["files"].items(): + assert sha256_hex((output / name).read_bytes()) == digest + for name, digest in index["objects"].items(): + path = state / "objects" / digest[:2] / digest + assert sha256_hex(path.read_bytes()) == digest + field = "relay_registry_hash" if name == "relay_registry" else f"{name}_policy_hash" + assert getattr(manifest.network, field) == digest + admin_file = tmp_path / "admin.token" + admin_file.write_text(ADMIN) + coord_file = tmp_path / "coord.key" + coord_file.write_bytes(coord_seed) + for name, value in { + "CHALLENGE_STATE_DIR": str(state), + "CHALLENGE_ADMIN_TOKEN_FILE": str(admin_file), + "HYPERTRAIN_COORD_KEY_FILE": str(coord_file), + "HYPERTRAIN_OWNER_HOTKEY": index["owner_ss58"], + }.items(): + monkeypatch.setenv(name, value) + config = Config.from_env() + assert config.owner_hotkey == index["owner_ss58"] + app = create_app( + config, + clock=lambda: float(manifest.training.beacon.genesis_time), + transport=httpx.MockTransport(Master().handler), + verify_beacon=verify_fixture, + ) + with TestClient(app) as client: + pushed = client.post("/v1/admin/beacon", json=beacon_payload(1), headers=bearer(ADMIN)) + assert pushed.status_code == 200, pushed.text + response = client.post( + "/v2/admin/runs", + content=(output / "manifest-envelope.json").read_bytes(), + headers={**bearer(ADMIN), "content-type": "application/json"}, + ) + assert response.status_code == 201, response.text + assert response.json()["run_id"] == index["run_id"] + + +@pytest.mark.parametrize( + "fault,message", + [ + ("audit", "audit policy budget must equal 2H"), + ("auditors", "two distinct audit reference identities required"), + ("derived", "v2 derived-state influence requires authenticated v0 integration"), + ("permissions", "keyfile must be mode 0600 and owned by uid"), + ("output", "--out must not exist or be empty"), + ("init_hash", "init_state_hash does not match computed initial state"), + ("policy_schema", "q_base"), + ("registry_schema", "registry_version"), + ("multiline_field", "extra_forbidden"), + ("expiry", "--exp-round must be positive"), + ("authority", "PINNED_AUTHORITIES"), + ("reward_budget", "MANIFEST_REWARD_BUDGET"), + ], +) +def test_build_refuses_invalid_inputs( + tmp_path: Path, + build_args: list[str], + capsys: pytest.CaptureFixture[str], + fault: str, + message: str, +) -> None: + inputs = tmp_path / "inputs" + training_path = inputs / "training.json" + training = json.loads(training_path.read_text()) + if fault in {"audit", "policy_schema", "authority", "reward_budget", "multiline_field"}: + path = inputs / "policies.json" + policies = json.loads(path.read_text()) + if fault == "audit": + policies["audit"]["max_steps_per_attempt"] = 3 + elif fault == "authority": + policies["economics"]["reward_authority"] = training["auditors"][0] + elif fault == "reward_budget": + policies["economics"]["round_reward_units"] = 999999 + elif fault == "multiline_field": + policies["admission"]["unexpected\nfield"] = True + else: + policies["admission"]["q_base"] = "00000000" + path.write_text(json.dumps(policies)) + elif fault == "registry_schema": + path = inputs / "relays.json" + registry = json.loads(path.read_text()) + registry["registry_version"] = 2 + path.write_text(json.dumps(registry)) + elif fault == "expiry": + build_args[build_args.index("--exp-round") + 1] = "0" + elif fault == "auditors": + training["auditors"] = training["auditors"][:1] + elif fault == "derived": + training["inner"]["state_policy"] = "derived" + elif fault == "permissions": + (tmp_path / "owner.seed").chmod(0o644) + elif fault == "output": + output = tmp_path / "output" + output.mkdir() + (output / "keep").write_text("untouched") + elif fault == "init_hash": + training["init_state_hash"] = "ff" * 32 + training_path.write_text(json.dumps(training)) + capsys.readouterr() + assert main(build_args) == 2 + captured = capsys.readouterr() + assert captured.out == "" + assert len(captured.err.splitlines()) == 1 + assert message in captured.err + assert not (tmp_path / "state" / "objects").exists() + assert not (tmp_path / "output" / "manifest-envelope.json").exists() + if fault == "output": + assert (tmp_path / "output" / "keep").read_text() == "untouched" + + +def test_build_without_store_accepts_hex_seed_and_matching_init_hash( + tmp_path: Path, build_args: list[str] +) -> None: + assert main(build_args) == 0 + first = json.loads((tmp_path / "output" / "manifest.json").read_text()) + training = tmp_path / "inputs" / "training.json" + training.write_text(json.dumps(first["training"])) + (tmp_path / "owner.seed").write_text((b"\x77" * 32).hex()) + output = tmp_path / "second-output" + output.mkdir() + no_store = build_args[:] + index = no_store.index("--state-dir") + del no_store[index : index + 2] + no_store[no_store.index("--out") + 1] = str(output) + assert main(no_store) == 0 + second = json.loads((output / "index.json").read_text()) + assert second["run_id"] == RunManifestV2.model_validate(first).run_id() + assert json.loads((output / "manifest.json").read_text()) == first + + +def test_example_and_help_do_not_import_torch(tmp_path: Path) -> None: + code = """ +import sys +class NoTorch: + def find_spec(self, fullname, path=None, target=None): + if fullname == 'torch' or fullname.startswith('torch.'): + raise RuntimeError('torch must remain optional') +sys.meta_path.insert(0, NoTorch()) +from hypertrain.ops.manifest_v2 import main +raise SystemExit(main(sys.argv[1:])) +""" + example = subprocess.run( + [sys.executable, "-c", code, "example", "--out", str(tmp_path / "inputs")], + capture_output=True, + text=True, + check=False, + ) + assert example.returncode == 0, example.stderr + training = json.loads((tmp_path / "inputs" / "training.json").read_text()) + assert "init_state_hash" not in training + assert len(set(training["auditors"])) == 2 + help_result = subprocess.run( + [sys.executable, "-c", code, "--help"], + capture_output=True, + text=True, + check=False, + ) + assert help_result.returncode == 0, help_result.stderr + assert "build" in help_result.stdout + assert "example" in help_result.stdout + + +def test_build_refuses_corrupt_existing_object( + tmp_path: Path, build_args: list[str], capsys: pytest.CaptureFixture[str] +) -> None: + from hypertrain.protocol.messages_v2 import AdmissionPolicyV2 + + policies = json.loads((tmp_path / "inputs" / "policies.json").read_text()) + digest = AdmissionPolicyV2.model_validate(policies["admission"]).digest() + path = tmp_path / "state" / "objects" / digest[:2] / digest + path.parent.mkdir(parents=True) + path.write_bytes(b"corrupt") + capsys.readouterr() + assert main(build_args) == 2 + captured = capsys.readouterr() + assert captured.out == "" + assert captured.err == "error: object store contains corrupt content\n" + assert path.read_bytes() == b"corrupt" + assert not (tmp_path / "output" / "manifest-envelope.json").exists() diff --git a/tests/relay/test_lifecycle_v2.py b/tests/relay/test_lifecycle_v2.py index 2dcb54c..de722b1 100644 --- a/tests/relay/test_lifecycle_v2.py +++ b/tests/relay/test_lifecycle_v2.py @@ -10,6 +10,7 @@ import pytest from test_transport_v2 import MASTER, MINER, NEXT, H, Rig +from hypertrain.data.store import CorruptObjectError, ObjectNotFound from hypertrain.protocol.jcs import canonicalize from hypertrain.protocol.relay_messages import ( AcceptedUploadAck, @@ -22,10 +23,11 @@ RetentionExtension, RetrievalRequest, RetrievalResponse, + UploadGrant, ) from hypertrain.relay.app import create_app from hypertrain.relay.client import receipt_verified -from hypertrain.relay.core import RelayError, Settlement +from hypertrain.relay.core import AssemblyMismatch, RelayError, Settlement @pytest.mark.parametrize("scope", ["chunk", "receipt", "failed-delete"]) @@ -674,3 +676,247 @@ async def read() -> None: assert not (await rig.relay.state()).uploads[grant.digest()].retrievals anyio.run(scenario) + + +def test_recover_when_journal_holds_unassemblable_upload(tmp_path: Path) -> None: + # Given: a granted manifest whose chunks are valid but concatenate to another hash. + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + forged = b"x" * (4 << 20) + b"y" + grant, manifest, _, assignment, opened = rig.upload(forged, nonce="91" * 32, w=1) + bad = UploadGrant.model_validate({**grant.body(), "delta_hash": H}) + await rig.relay.accept(rig.seal(bad), manifest, assignment, opened) + await rig.put(bad, forged[: 4 << 20], 0) + with pytest.raises(RelayError): + await rig.put(bad, forged[4 << 20 :], 1) + honest, plain, _, assignment, opened = rig.upload(b"abc", nonce="92" * 32, w=2) + await rig.relay.accept(rig.seal(honest), plain, assignment, opened) + await rig.put(honest, b"abc") + + def interrupted(state) -> None: + state.uploads[honest.digest()].receipt = None + + await rig.relay._change(interrupted) + # When: a restarted relay recovers the whole journal. + relay = rig.new_relay() + await relay.recover() + completed: list[str] = [] + complete = relay.complete + + async def counted(grant_hash: str): + completed.append(grant_hash) + return await complete(grant_hash) + + relay.complete = counted # type: ignore[method-assign] + await relay.recover() + # Then: the poisoned entry is failed once, the honest one still gets its receipt. + state = await relay.state() + assert state.uploads[bad.digest()].receipt is None + assert state.uploads[bad.digest()].forward_error == "AssemblyMismatch" + assert not rig.store.store._path(H).exists() + assert state.uploads[honest.digest()].receipt is not None + assert bad.digest() not in completed + + anyio.run(scenario) + + +async def interrupted_pair(rig: Rig) -> tuple[UploadGrant, UploadGrant]: + # Two fully chunked uploads whose receipts were lost: recover() must complete both. + grants = [] + for data, nonce, w in ((b"abc", "93" * 32, 1), (b"def", "94" * 32, 2)): + grant, manifest, raw, assignment, opened = rig.upload(data, nonce=nonce, w=w) + await rig.relay.accept(raw, manifest, assignment, opened) + await rig.put(grant, data) + grants.append(grant) + + def interrupted(state) -> None: + for grant in grants: + state.uploads[grant.digest()].receipt = None + + await rig.relay._change(interrupted) + return grants[0], grants[1] + + +def failing_get(rig: Rig, sha: str, error: Exception) -> None: + original = rig.store.get + + async def get(key, size, file): + if key == sha: + raise error + await original(key, size, file) + + rig.store.get = get # type: ignore[method-assign] + + +def counting(relay) -> list[str]: + completed: list[str] = [] + complete = relay.complete + + async def counted(grant_hash: str): + completed.append(grant_hash) + return await complete(grant_hash) + + relay.complete = counted + return completed + + +def test_recover_continues_after_object_not_found(tmp_path: Path) -> None: + # Given: the first journal entry lost its chunk in the backing store. + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + lost, honest = await interrupted_pair(rig) + failing_get(rig, hashlib.sha256(b"abc").hexdigest(), ObjectNotFound("gone")) + # When: a restarted relay recovers the whole journal. + relay = rig.new_relay() + await relay.recover() + # Then: the honest entry gets its receipt and the lost one is recorded. + state = await relay.state() + assert state.uploads[honest.digest()].receipt is not None + assert state.uploads[lost.digest()].receipt is None + assert state.uploads[lost.digest()].forward_error == "ObjectNotFound" + + anyio.run(scenario) + + +def test_recover_does_not_replay_corrupt_object_entry(tmp_path: Path) -> None: + # Given: the first journal entry's held chunk is rotten. + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + rotten, honest = await interrupted_pair(rig) + failing_get(rig, hashlib.sha256(b"abc").hexdigest(), CorruptObjectError("bit rot")) + relay = rig.new_relay() + await relay.recover() + completed = counting(relay) + # When: recovery runs again. + await relay.recover() + # Then: the honest entry recovered once and the rotten one is never retried. + state = await relay.state() + assert state.uploads[honest.digest()].receipt is not None + assert state.uploads[rotten.digest()].forward_error == "CorruptObjectError" + assert completed.count(rotten.digest()) == 0 + + anyio.run(scenario) + + +def test_recover_retries_after_transient_error( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + # Given: the first journal entry's backing read times out once. + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + slow, honest = await interrupted_pair(rig) + original = rig.store.get + failing_get(rig, hashlib.sha256(b"abc").hexdigest(), TimeoutError()) + relay = rig.new_relay() + await relay.recover() + state = await relay.state() + assert state.uploads[honest.digest()].receipt is not None + assert state.uploads[slow.digest()].forward_error is None + assert "TimeoutError" in caplog.text + # When: the backing recovers and recovery runs again. + rig.store.get = original # type: ignore[method-assign] + await relay.recover() + # Then: the transient entry is retried and completes. + assert (await relay.state()).uploads[slow.digest()].receipt is not None + + anyio.run(scenario) + + +def replacement(rig: Rig, data: bytes, replaces: str, *, hotkey: str = MINER.ss58): + # Master-signed grant for the honest chunk list that supersedes `replaces`. + grant, manifest, _, assignment, opened = rig.upload(data, nonce="61" * 32) + grant = UploadGrant.model_validate({**grant.body(), "hotkey": hotkey, "replaces": replaces}) + assigned = RelayAssignment.model_validate({**assignment.body, "hotkey": hotkey}) + return grant, manifest, rig.seal(grant), rig.seal(assigned), opened + + +async def miner_chosen(rig: Rig, data: bytes) -> UploadGrant: + # Accepted while the master did not hold the original: the list need not match the delta. + grant, _, _, assignment, opened = rig.upload(data) + manifest = rig.upload(b"x" * len(data))[1] + grant = UploadGrant.model_validate( + {**grant.body(), "chunk_manifest_hash": manifest.chunk_manifest_hash()} + ) + await rig.relay.accept(rig.seal(grant), manifest, assignment, opened) + return grant + + +def test_relay_accepts_replacement_after_old_accept(tmp_path: Path) -> None: + # Given: the relay reserved the round for a miner-chosen list, then the master + # (now holding the original) signs a replacement for the honest list. + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + old = await miner_chosen(rig, b"abc") + grant, manifest, raw, assignment, opened = replacement(rig, b"abc", old.digest()) + # When: the miner presents the replacement and uploads the original bytes. + await rig.relay.accept(raw, manifest, assignment, opened) + await rig.put(grant, b"abc") + # Then: the upload completes with a receipt and the old reservation is gone. + state = await rig.relay.state() + assert state.uploads[grant.digest()].receipt is not None + assert old.digest() not in state.uploads + assert old.nonce not in state.nonces + + anyio.run(scenario) + + +def test_relay_refuses_replacement_after_receipt(tmp_path: Path) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + old, _, _ = await rig.accept(b"abc") + await rig.put(old, b"abc") + _, manifest, raw, assignment, opened = replacement(rig, b"abc", old.digest()) + with pytest.raises(RelayError): + await rig.relay.accept(raw, manifest, assignment, opened) + assert (await rig.relay.state()).uploads[old.digest()].receipt is not None + + anyio.run(scenario) + + +def test_relay_refuses_replacement_after_chunk_custody(tmp_path: Path) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + old = await miner_chosen(rig, b"abc") + with pytest.raises(AssemblyMismatch): # the chunk is held under signed custody + await rig.put(old, b"xxx") + _, manifest, raw, assignment, opened = replacement(rig, b"abc", old.digest()) + with pytest.raises(RelayError, match="custody"): + await rig.relay.accept(raw, manifest, assignment, opened) + assert (await rig.relay.state()).uploads[old.digest()].chunks + + anyio.run(scenario) + + +def test_relay_refuses_replacement_with_wrong_replaces(tmp_path: Path) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + old = await miner_chosen(rig, b"abc") + _, manifest, raw, assignment, opened = replacement(rig, b"abc", "00" * 32) + with pytest.raises(RelayError): + await rig.relay.accept(raw, manifest, assignment, opened) + assert old.digest() in (await rig.relay.state()).uploads + + anyio.run(scenario) + + +def test_relay_refuses_replacement_for_other_miner(tmp_path: Path) -> None: + async def scenario() -> None: + async with httpx.AsyncClient() as client: + rig = Rig(tmp_path, client) + old = await miner_chosen(rig, b"abc") + _, manifest, raw, assignment, opened = replacement( + rig, b"abc", old.digest(), hotkey=NEXT.ss58 + ) + with pytest.raises(RelayError): + await rig.relay.accept(raw, manifest, assignment, opened) + assert old.digest() in (await rig.relay.state()).uploads + + anyio.run(scenario) diff --git a/tests/test_challenge_profile_example.py b/tests/test_challenge_profile_example.py new file mode 100644 index 0000000..8dbddfd --- /dev/null +++ b/tests/test_challenge_profile_example.py @@ -0,0 +1,14 @@ +"""deploy/challenge-profile-row.example.toml: a full profile an operator can copy as is.""" + +import tomllib +from pathlib import Path + +EXAMPLE = Path(__file__).resolve().parent.parent / "deploy/challenge-profile-row.example.toml" + + +def test_activation_example_shares_sum_to_10000_and_hypertrain_positive() -> None: + profile = tomllib.loads(EXAMPLE.read_text()) + shares = {c["id"]: c["emission_share_bps"] for c in profile["challenges"]} + assert sum(shares.values()) == 10000 + assert shares["hypertrain"] > 0 + assert profile["version"] >= 3 # a third challenge needs version >= 3 diff --git a/tests/test_check_proof.py b/tests/test_check_proof.py new file mode 100644 index 0000000..cf91257 --- /dev/null +++ b/tests/test_check_proof.py @@ -0,0 +1,75 @@ +"""check-proof.sh: a golden replay xfail fails the gate unless the diagnostic variable is set; +an unexpected skip still fails it.""" + +import os +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + +# Keeps one test per suite (speed) and turns every test under XFAIL_PREFIX into an xfail with +# reason XFAIL_WHY (default: the _expect_golden reason, as a leaf root outside GOLDEN gives); +# FORCE_SKIP also skips the codec v2 test. +PLUGIN = """ +import os + +import pytest + +GOLDEN_WHY = %r + +def pytest_collection_modifyitems(config, items): + config.hook.pytest_deselected(items=items[1:]) + del items[1:] + +def pytest_runtest_setup(item): + prefix = os.environ.get("XFAIL_PREFIX", "tests/trainer/test_golden_replay.py") + if item.nodeid.startswith(prefix): + pytest.xfail(os.environ.get("XFAIL_WHY", GOLDEN_WHY)) + if os.environ.get("FORCE_SKIP") and item.nodeid.startswith("tests/trainer/test_codec_v2.py"): + pytest.skip("forced skip") +""" +# Fixed marker the gate greps for: the start of test_golden_replay.XFAIL_REASON. +GOLDEN_WHY = "golden leaf root differs on this CPU class; forced" + + +def _gate(tmp_path: Path, **env: str) -> subprocess.CompletedProcess[str]: + (tmp_path / "force_golden_xfail.py").write_text(PLUGIN % GOLDEN_WHY) + return subprocess.run( + [str(ROOT / "scripts/check-proof.sh"), "-p", "force_golden_xfail"], + capture_output=True, + text=True, + check=False, + env={**os.environ, "PYTHONPATH": str(tmp_path), **env}, + timeout=600, + ) + + +def test_proof_gate_fails_on_golden_xfail(tmp_path: Path) -> None: + strict = _gate(tmp_path) + assert strict.returncode == 1, strict.stdout + strict.stderr + assert "golden replay xfail is not a proof" in strict.stderr + assert "check-proof: FAIL" in strict.stdout + allowed = _gate(tmp_path, CHECK_PROOF_ALLOW_GOLDEN_XFAIL="1") + assert allowed.returncode == 0, allowed.stdout + allowed.stderr + assert "check-proof: PASS" in allowed.stdout + # Reporting xfails must not drop the skip report (a second -r replaces the first). + skipped = _gate(tmp_path, CHECK_PROOF_ALLOW_GOLDEN_XFAIL="1", FORCE_SKIP="1") + assert skipped.returncode == 1, skipped.stdout + skipped.stderr + assert "unexpected skip in tests/trainer/test_codec_v2.py" in skipped.stderr + + +def test_proof_gate_fails_on_any_golden_consumer_xfail(tmp_path: Path) -> None: + # Matched on the reason, not the file: a consumer outside test_golden_replay.py is caught. + marker = GOLDEN_WHY.split(";")[0] + assert marker in (ROOT / "tests/trainer/test_golden_replay.py").read_text() + consumer = "tests/trainer/test_act_ckpt.py" + strict = _gate(tmp_path, XFAIL_PREFIX=consumer) + assert strict.returncode == 1, strict.stdout + strict.stderr + assert f"XFAIL {consumer}::" in strict.stderr + assert "check-proof: FAIL" in strict.stdout + allowed = _gate(tmp_path, XFAIL_PREFIX=consumer, CHECK_PROOF_ALLOW_GOLDEN_XFAIL="1") + assert allowed.returncode == 0, allowed.stdout + allowed.stderr + # Another xfail reason is not a golden mismatch and must not trip the golden rule. + other = _gate(tmp_path, XFAIL_PREFIX=consumer, XFAIL_WHY="known unrelated bug") + assert other.returncode == 0, other.stdout + other.stderr + assert "check-proof: PASS" in other.stdout diff --git a/tests/test_mine_smoke.py b/tests/test_mine_smoke.py new file mode 100644 index 0000000..e84efd5 --- /dev/null +++ b/tests/test_mine_smoke.py @@ -0,0 +1,76 @@ +import importlib.util +import json +import os +import stat +import tempfile +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("mine_smoke", ROOT / "scripts" / "mine_smoke.py") +assert spec is not None and spec.loader is not None +mine_smoke = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mine_smoke) + + +def test_default_evidence_is_a_fresh_private_file_per_run( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + def no_round(**kw: object) -> None: + raise RuntimeError("no round in this test") + + monkeypatch.delenv("MINE_SMOKE_EVIDENCE", raising=False) + monkeypatch.setattr(tempfile, "tempdir", str(tmp_path)) + monkeypatch.setattr(mine_smoke, "manifest", no_round) + planted = tmp_path / "mine-smoke-evidence.json" + planted.symlink_to(tmp_path / "victim") + assert mine_smoke.main() == 1 + assert mine_smoke.main() == 1 + capsys.readouterr() + files = sorted(p for p in tmp_path.rglob("*.json") if p != planted) + assert len(files) == 2 and files[0] != files[1] + for f in files: + assert stat.S_IMODE(f.stat().st_mode) == 0o600 + assert "no round in this test" in f.read_text() + assert not (tmp_path / "victim").exists() + + +def test_imposed_evidence_path_is_private_and_kept_until_the_write( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + target = tmp_path / "ev.json" + target.write_text("previous run") + target.chmod(0o644) + monkeypatch.setenv("MINE_SMOKE_EVIDENCE", str(target)) + out, fd = mine_smoke.evidence_file() + try: + assert out == target and stat.S_IMODE(target.stat().st_mode) == 0o600 + assert target.read_text() == "previous run" # not truncated before the run + finally: + os.close(fd) + + +def test_mine_smoke_stop_failure_still_writes_evidence( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + class World: + def __init__(self, *args: object) -> None: + pass + + def miner(self, k: object) -> None: + raise RuntimeError("no round in this test") + + def stop(self) -> None: + raise TimeoutError("process wait timed out") + + target = tmp_path / "ev.json" + monkeypatch.setenv("MINE_SMOKE_EVIDENCE", str(target)) + monkeypatch.setattr(mine_smoke, "manifest", lambda **kw: None) + monkeypatch.setattr(mine_smoke, "World", World) + assert mine_smoke.main() == 1 + capsys.readouterr() + ev = json.loads(target.read_text()) + assert ev["result"] == "FAIL" + assert ev["cleanup_failed"] == "TimeoutError: process wait timed out" + assert "no round in this test" in ev["failed"] diff --git a/tests/test_mine_smoke_v2.py b/tests/test_mine_smoke_v2.py new file mode 100644 index 0000000..41befea --- /dev/null +++ b/tests/test_mine_smoke_v2.py @@ -0,0 +1,162 @@ +import importlib.util +import json +import os +import signal +import stat +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location( + "mine_smoke_v2", ROOT / "scripts" / "mine_smoke_v2.py" +) +assert spec is not None and spec.loader is not None +mine_smoke_v2 = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mine_smoke_v2) +pytestmark = pytest.mark.fast # listed in scripts/check-fast.sh + + +def ok(name: str) -> dict[str, object]: + return {"name": name, "mode": "subprocess", "rc": 0, "ms": 1} + + +ALL = [ok(n) for n in mine_smoke_v2.REQUIRED] + + +def test_imposed_evidence_path_refuses_a_symlink( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + victim = tmp_path / "victim" + link = tmp_path / "ev.json" + link.symlink_to(victim) + monkeypatch.setenv("MINE_SMOKE_V2_EVIDENCE", str(link)) + with pytest.raises(OSError): + mine_smoke_v2.evidence_file() + assert not victim.exists() + + +def test_imposed_evidence_path_is_private_and_kept_until_the_write( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + target = tmp_path / "ev.json" + target.write_text("previous run") + target.chmod(0o644) + monkeypatch.setenv("MINE_SMOKE_V2_EVIDENCE", str(target)) + out, fd = mine_smoke_v2.evidence_file() + try: + assert out == target and stat.S_IMODE(target.stat().st_mode) == 0o600 + assert target.read_text() == "previous run" + finally: + os.close(fd) + + +def test_default_evidence_is_a_fresh_private_file(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("MINE_SMOKE_V2_EVIDENCE", raising=False) + paths = [] + for _ in range(2): + out, fd = mine_smoke_v2.evidence_file() + os.close(fd) + paths.append(out) + assert stat.S_IMODE(out.stat().st_mode) == 0o600 + try: + assert paths[0] != paths[1] + finally: + for p in paths: + p.unlink() + + +def test_all_required_stages_green_is_pass() -> None: + assert mine_smoke_v2.verdict([*ALL, {**ok("graduate-peers"), "mode": "in-process"}]) is None + + +@pytest.mark.parametrize( + "broken", + [{"rc": 2}, {"rc": None}, {"skipped": True}], + ids=["failed", "never-finished", "skipped"], +) +def test_one_bad_stage_fails_the_run_and_is_named(broken: dict[str, object]) -> None: + stages = [*ALL[:1], {**ok("probe-v2"), **broken}, *ALL[1:]] + assert "probe-v2" in mine_smoke_v2.verdict(stages) + + +def test_a_missing_required_stage_fails_the_run() -> None: + assert "run-v2" in mine_smoke_v2.verdict(ALL[:-1]) + + +def test_stage_records_rc_and_raises_on_failure(monkeypatch: pytest.MonkeyPatch) -> None: + class Done: + returncode, stdout, stderr = 3, "", "boom" + + monkeypatch.setattr(mine_smoke_v2.subprocess, "run", lambda *a, **k: Done) + stages: list[dict[str, object]] = [] + with pytest.raises(RuntimeError, match="probe-v2"): + mine_smoke_v2.stage(stages, "probe-v2", ["true"]) + assert stages[0]["rc"] == 3 and stages[0]["mode"] == "subprocess" and "ms" in stages[0] + assert "probe-v2" in mine_smoke_v2.verdict(stages) + + +def test_main_fails_on_a_post_stage_check_even_when_every_stage_is_green( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + def chain(tmp: Path, ev: dict, stack: object) -> None: + ev["stages"].extend(ALL) + raise RuntimeError("run-v2: delta recorded: False") + + evidence = tmp_path / "ev.json" + monkeypatch.setenv("MINE_SMOKE_V2_EVIDENCE", str(evidence)) + monkeypatch.setattr(mine_smoke_v2, "chain", chain) + assert mine_smoke_v2.main() == 1 + capsys.readouterr() + ev = json.loads(evidence.read_text()) + assert ev["result"] == "FAIL" and "delta recorded" in ev["failed"] + + +def test_unknown_break_stage_is_rejected_before_any_stage( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + def chain(tmp: Path, ev: dict, stack: object) -> None: + raise AssertionError("must not run") + + monkeypatch.setenv("MINE_SMOKE_V2_BREAK", "escrow-lock") + monkeypatch.setattr(mine_smoke_v2, "chain", chain) + assert mine_smoke_v2.main() == 2 + assert "'escrow-lock' is not a breakable stage: join-v2" in capsys.readouterr().err + + +def test_sigterm_still_writes_fail_evidence( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + def chain(tmp: Path, ev: dict, stack: object) -> None: + ev["stages"].append(ok("join-v2")) + os.kill(os.getpid(), signal.SIGTERM) # what `timeout` sends at the budget + time.sleep(5) + + evidence = tmp_path / "ev.json" + evidence.write_text('{"result": "PASS"}') # a previous run's verdict must not survive + monkeypatch.setenv("MINE_SMOKE_V2_EVIDENCE", str(evidence)) + monkeypatch.setattr(mine_smoke_v2, "chain", chain) + previous = signal.getsignal(signal.SIGTERM) + try: + assert mine_smoke_v2.main() == 1 + finally: + signal.signal(signal.SIGTERM, previous) + capsys.readouterr() + ev = json.loads(evidence.read_text()) + assert ev["result"] == "FAIL" and "budget exceeded" in ev["failed"] + assert [s["name"] for s in ev["stages"]] == ["join-v2"] + + +def test_reap_kills_a_detached_process_naming_the_run_dir(tmp_path: Path) -> None: + marker = str(tmp_path / "attempt") + proc = subprocess.Popen( # noqa: S603 - fixed argv + [sys.executable, "-c", "import time; time.sleep(60)", marker], start_new_session=True + ) + try: + assert proc.pid in mine_smoke_v2.reap(str(tmp_path)) + assert proc.wait(timeout=10) == -signal.SIGKILL + finally: + proc.kill() diff --git a/tests/test_publish_exclusions.py b/tests/test_publish_exclusions.py index fa832cc..30c4573 100644 --- a/tests/test_publish_exclusions.py +++ b/tests/test_publish_exclusions.py @@ -393,7 +393,7 @@ def test_actual_publisher_selector_preserves_public_excludes_private(tmp_path: P "tests/challenge/test_authority_snapshot_v2.py", "docs/schemas/example-run-manifest.json", "docs/operator.md", - "deploy/trust-root-v3-row.example.toml", + "deploy/challenge-profile-row.example.toml", "deploy/k8s/local/fixtures.yaml", "scripts/network_authority_snapshot.py", "experiments/gpu_network_v2/profile.json", diff --git a/tests/test_vast_e2e.py b/tests/test_vast_e2e.py index cb675e7..f324e31 100644 --- a/tests/test_vast_e2e.py +++ b/tests/test_vast_e2e.py @@ -4,6 +4,10 @@ import importlib.util import json +import os +import shlex +import signal +import subprocess import sys import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -22,6 +26,7 @@ KEY = "fake-secret-key-0123456789" IID = 777 +ARGS = ["--key-file", "k", "--max-usd", "1", "--max-minutes", "1"] # parse() only class Fake: @@ -34,6 +39,7 @@ def __init__(self, dph: float = 0.6, credit: float = 50.0) -> None: self.throttle: dict[str, int] = {} self.asks: list[str] = [] self.gone: set[str] = set() + self.queries: list[dict] = [] def offers(self) -> list[dict]: base = dict( @@ -46,6 +52,8 @@ def offers(self) -> list[dict]: inet_down=1200, storage_cost=0.2, machine_id=1, + cuda_max_good=13.2, + driver_version="580.178.04", ) return [ base | {"id": 11, "dph_total": self.dph + 0.3, "machine_id": 2}, @@ -64,8 +72,14 @@ def handle(self, method: str, url: str, body: dict | None) -> tuple[int, dict]: return 200, {"id": 1, "credit": self.credit} if key == "GET /api/v0/bundles/": q = json.loads(parse_qs(urlsplit(url).query)["q"][0]) - assert q["num_gpus"] == {"eq": 2} and q["gpu_name"] == {"eq": "RTX 5090"} - return 200, {"offers": self.offers()} + assert q["num_gpus"] == {"eq": vast.NUM_GPUS} and q["gpu_name"] == {"eq": "RTX 5090"} + self.queries.append(q) + if "driver_version" in q: # real API answer, 2026-10-19 + return 400, {"error": "driver_vers in None: query values can't be None"} + # Real API: sorted by price, server-side dph_total floor, at most `limit` offers. + floor = q.get("dph_total", {}).get("gte", 0) + rows = sorted(self.offers(), key=lambda o: o["dph_total"]) + return 200, {"offers": [o for o in rows if o["dph_total"] >= floor][: q["limit"]]} if method == "PUT" and path.startswith("/api/v0/asks/"): if path in self.gone: return 400, {"success": False, "error": "no_such_ask"} @@ -280,6 +294,58 @@ def ssh_probe(self, host, port): assert json.loads((tmp_path / "ev/harness-result.json").read_text())["absent"] is True +def test_every_host_stuck_still_ends_with_parsed_absence(fake, tmp_path): + # Real run (instance 56659539): ssh never accepted the key, boot_timeout, DELETE 200, then + # the run ended with no cleanup_done and no listing after the boot-time absence check. + class NeverReady(Scripted): + def ssh_probe(self, host, port): + self.probe_error = "root@127.0.0.1: Permission denied (publickey)." + return False + + rc, journal, _ = run( + fake, tmp_path, "--boot-timeout", "0.3", "--boot-retries", "0", harness=NeverReady + ) + ev = events(journal) + assert rc == 1 and fake.asks == ["/api/v0/asks/12/"] + assert ev.index("boot_timeout") < ev.index("cleanup_start") < ev.index("cleanup_done") + listed = [i for i, j in enumerate(journal) if j.get("path", "").startswith("/api/v1/")] + assert listed[-1] > ev.index("cleanup_start") # absence re-proved at the end + done = journal[ev.index("cleanup_done")] + assert done["absent"] is True and done["deleted"] is True + + +def test_every_host_stuck_alerts_when_it_reappears(fake, tmp_path): + class NeverReady(Scripted): + def ssh_probe(self, host, port): + return False + + def cleanup(self, reason): + fake.deleted, fake.linger = True, 100 # the deleted host is listed again + super().cleanup(reason) + + rc, _, _ = run( + fake, tmp_path, "--boot-timeout", "0.3", "--boot-retries", "0", harness=NeverReady + ) + assert rc == 3 + assert json.loads((tmp_path / "ev/harness-result.json").read_text())["absent"] is False + + +def test_excluded_offer_is_never_rented(fake, tmp_path, capsys): + rc, _, _ = run(fake, tmp_path, "--dry-run", "--exclude-offer", "12") + assert rc == 0 and json.loads(capsys.readouterr().out)["chosen_offer"]["id"] == 11 + + +def test_excluded_offer_honoured_when_profile_overrides_eligible(fake, tmp_path, capsys): + # scripts/genuine_cuda_tests.py and scripts/vast_parity.py override eligible() without the + # exclude check: --exclude-offer was a silent no-op there. + class OwnFilter(Scripted): + def eligible(self, o): + return vast.eligible(o) + + rc, _, _ = run(fake, tmp_path, "--dry-run", "--exclude-offer", "12", harness=OwnFilter) + assert rc == 0 and json.loads(capsys.readouterr().out)["chosen_offer"]["id"] == 11 + + def test_vanished_offer_falls_back_to_next(fake, tmp_path): fake.gone = {"/api/v0/asks/12/"} rc, journal, _ = run(fake, tmp_path) @@ -306,6 +372,29 @@ def ssh_probe(self, host, port): assert [j["event"] for j in journal].count("ssh_probe_failed") == 1 +@pytest.mark.parametrize("during", ["remote", "rescue"]) +def test_stop_signals_never_skip_rescue_or_delete(fake, tmp_path, during): + class Stopped(Scripted): + def run_remote(self): + if during == "remote": # service manager stop/restart mid-run + os.kill(os.getpid(), signal.SIGTERM) + assert self.cancel.is_set() + return 143 + return 0 + + def rescue(self): + os.kill(os.getpid(), signal.SIGTERM) # a stop landing while cleanup runs + os.kill(os.getpid(), signal.SIGHUP) # and the other stop signal right after + return super().rescue() + + before = signal.getsignal(signal.SIGTERM), signal.getsignal(signal.SIGHUP) + rc, journal, _ = run(fake, tmp_path, harness=Stopped) + assert (signal.getsignal(signal.SIGTERM), signal.getsignal(signal.SIGHUP)) == before + result = json.loads((tmp_path / "ev/harness-result.json").read_text()) + assert rc == 1 and fake.deleted and result["absent"] is True and not result["remote_ok"] + assert (tmp_path / "ev/out/result.json").exists() and "cleanup_done" in events(journal) + + def test_absence_never_confirmed_alerts(fake, tmp_path): fake.linger = 100 rc, _, _ = run(fake, tmp_path) @@ -315,9 +404,9 @@ def test_absence_never_confirmed_alerts(fake, tmp_path): @pytest.fixture def mfu_globals(): - saved = vast.IMAGE, vast.NUM_GPUS, vast.MIN_PCIE_BW + saved = vast.IMAGE, vast.NUM_GPUS, vast.MIN_PCIE_BW, vast.ALLOWED_DRIVERS yield - vast.IMAGE, vast.NUM_GPUS, vast.MIN_PCIE_BW = saved + vast.IMAGE, vast.NUM_GPUS, vast.MIN_PCIE_BW, vast.ALLOWED_DRIVERS = saved def test_mfu_profile_picks_fast_pcie_host_and_passes_remote_env(fake, tmp_path, mfu_globals): @@ -353,9 +442,357 @@ def test_mfu_eligibility_floors(mfu_globals): assert not vast.eligible({**o, "cuda_max_good": 13.0}) assert not vast.eligible({k: v for k, v in o.items() if k != "pcie_bw"}) vast.MIN_PCIE_BW = None - assert vast.eligible({k: v for k, v in o.items() if k not in ("pcie_bw", "cuda_max_good")}) + assert vast.eligible({k: v for k, v in o.items() if k != "pcie_bw"}) + + +def test_cuda_floor_enforced_without_pcie(mfu_globals): + vast.MIN_PCIE_BW = None + o = Fake().offers()[1] + assert vast.eligible(o | {"cuda_max_good": 13.2}) + assert not vast.eligible(o | {"cuda_max_good": 13.0}) + assert not vast.eligible({k: v for k, v in o.items() if k != "cuda_max_good"}) + + +def test_num_gpus_applies_to_default_profile(fake, tmp_path, capsys, mfu_globals): + offers = fake.offers + fake.offers = lambda: [*offers(), offers()[1] | {"id": 14, "num_gpus": 8, "dph_total": 2.0}] + rc, _, _ = run(fake, tmp_path, "--num-gpus", "8", "--dry-run", usd="10") + assert rc == 0 and fake.queries[-1]["num_gpus"] == {"eq": 8} + chosen = json.loads(capsys.readouterr().out)["chosen_offer"] + assert chosen["id"] == 14 and chosen["num_gpus"] == 8 + + +def test_miner_flow_refuses_num_gpus_other_than_one(capsys): + argv = [*ARGS, "--profile", "miner-flow"] + assert vast.parse(argv).num_gpus == 1 + with pytest.raises(SystemExit) as e: + vast.parse([*argv, "--num-gpus", "2"]) + assert e.value.code == 2 and "--profile miner-flow" in capsys.readouterr().err + + +def test_default_evidence_dir_is_repo_relative(fake, tmp_path, monkeypatch, mfu_globals): + monkeypatch.setattr(vast, "ROOT", tmp_path) + (tmp_path / "key").write_text(KEY + "\n") + argv = ["--key-file", str(tmp_path / "key"), "--max-usd", "6", "--max-minutes", "90"] + assert vast.parse(argv).evidence_dir == tmp_path / ".evidence/vast" + argv += ["--api-base", fake.url, "--min-gap", "0", "--dry-run"] + assert vast.main(argv, harness=Scripted, sleep=lambda s: None) == 0 + assert (tmp_path / ".evidence/vast/journal.jsonl").is_file() + + +def test_driver_filter_in_query_and_eligible(fake, tmp_path, mfu_globals): + base = fake.offers()[1] + shape = [("615.71.09", 0.1), ("580.178.04", 0.4), ("570.211.01", 0.3), (None, 0.2)] + offers = [ + base | {"id": 20 + i, "dph_total": d, "driver_version": v} for i, (v, d) in enumerate(shape) + ] + del offers[3]["driver_version"] + fake.offers = lambda: offers + + def dry(*extra): + rc, journal, _ = run(fake, tmp_path, "--dry-run", *extra) + chosen = [j for j in journal if j["event"] == "offer_chosen"] + return rc, [(j["eligible"], j["quote"]["id"]) for j in chosen[-1:]] + + # The allow-list is applied client-side only: the server answers 400 to a driver filter. + assert dry() == (0, [(2, 22)]) + assert dry("--driver", "580.178.04") == (0, [(1, 21)]) + assert dry("--any-driver") == (0, [(4, 20)]) + assert not any("driver_version" in q for q in fake.queries) + with pytest.raises(SystemExit): + vast.parse([*ARGS, "--driver", "999.0"]) + + +def test_refusal_counts_driver_rejections(fake, tmp_path, mfu_globals): + offers = fake.offers + fake.offers = lambda: [o | {"driver_version": "615.71.09"} for o in offers()] + rc, journal, _ = run(fake, tmp_path, "--dry-run") + reason = next(j["reason"] for j in journal if j["event"] == "refused") + assert rc == 2 + assert reason == "no eligible 2x RTX 5090 offer among 3 (2 rejected by driver allow-list)" + + +def test_qualified_offer_beyond_first_page_is_found(fake, tmp_path, capsys, mfu_globals): + base = fake.offers()[1] | {"driver_version": "615.71.09"} + cheap = [base | {"id": 100 + i, "dph_total": 0.1 + i / 1000} for i in range(70)] + good = base | {"id": 99, "dph_total": 0.5, "driver_version": "580.178.04"} + fake.offers = lambda: [*cheap, good] + rc, journal, _ = run(fake, tmp_path, "--dry-run") + assert rc == 0 and json.loads(capsys.readouterr().out)["chosen_offer"]["id"] == 99 + assert [q.get("dph_total") for q in fake.queries] == [None, {"gte": cheap[63]["dph_total"]}] + + +def test_refusal_says_search_was_capped(fake, tmp_path, mfu_globals, monkeypatch): + monkeypatch.setattr(vast, "OFFER_PAGES", 2) + base = fake.offers()[1] | {"driver_version": "615.71.09"} + fake.offers = lambda: [base | {"id": 100 + i, "dph_total": 0.1 + i / 100} for i in range(200)] + rc, journal, _ = run(fake, tmp_path, "--dry-run") + reason = next(j["reason"] for j in journal if j["event"] == "refused") + assert rc == 2 and len(fake.queries) == 2 + assert reason.endswith("search stopped after 2 pages of 64: cheaper-first market not exhausted") + + +def test_refusal_says_search_stalled_on_one_price(fake, tmp_path, mfu_globals): + base = fake.offers()[1] | {"driver_version": "615.71.09"} + flat = [base | {"id": 100 + i, "dph_total": 0.2} for i in range(64)] + fake.offers = lambda: [*flat, base | {"id": 99, "dph_total": 0.5}] + rc, journal, _ = run(fake, tmp_path, "--dry-run") + reason = next(j["reason"] for j in journal if j["event"] == "refused") + assert rc == 2 and len(fake.queries) == 2 + assert reason.endswith("search stalled: 64 offers at 0.2 USD/h fill a page") + + +def test_gpu8_profile_dry_run_selects_eight_gpu_qualified_offer( + fake, tmp_path, capsys, mfu_globals +): + base = fake.offers()[1] | {"num_gpus": 8, "cuda_max_good": 13.2} + unqualified = [ + ("615.71.09", 11.75), + ("575.64.03", 2.9), + ("590.44.01", 3.4), + ("550.163.01", 3.8), + ("560.35.03", 4.0), + ("565.77", 4.1), + ("600.12", 6.2), + ] + offers = [ + base | {"id": 30 + i, "machine_id": 30 + i, "driver_version": v, "dph_total": d} + for i, (v, d) in enumerate(unqualified) + ] + offers += [ + base | {"id": 40, "driver_version": "580.178.04", "dph_total": 4.28, "pcie_bw": 26.4}, + base | {"id": 41, "driver_version": "595.84", "dph_total": 5.66, "pcie_bw": 27.0}, + ] + fake.offers = lambda: offers + rc, _, _ = run(fake, tmp_path, "--profile", "gpu8", "--dry-run", minutes="30", usd="10") + chosen = json.loads(capsys.readouterr().out)["chosen_offer"] + assert rc == 0 and not any(m == "PUT" for m, _ in fake.calls) + assert fake.queries[-1]["num_gpus"] == {"eq": 8} + assert chosen["num_gpus"] == 8 and chosen["dph_total"] == 4.28 + assert chosen["driver_version"] in vast.QUALIFIED_DRIVERS + with pytest.raises(SystemExit): + vast.parse([*ARGS, "--profile", "gpu8", "--num-gpus", "2"]) + + +def test_gpu_e2e_remote_gets_host_gpu_count(tmp_path): + class Capture(vast.Harness): + def sh(self, command, **kw): + self.command = command + return 0 + + a = vast.parse( + [ + *ARGS, + "--evidence-dir", + str(tmp_path), + "--profile", + "gpu8", + "--remote-env", + "HT_E2E_GPUS=2", + ] + ) + h = Capture(a, vast.Api("http://127.0.0.1:9", KEY, tmp_path / "j.jsonl")) + assert h.run_remote() == 0 + words = shlex.split(h.command) + assert words[words.index("bash") - 1] == "HT_E2E_GPUS=8" # env(1): the last value wins + remote = (vast.ROOT / "scripts/vast_e2e_remote.sh").read_text() + assert 'N = int(os.environ.get("HT_E2E_GPUS", "2"))' in remote + + +@pytest.mark.parametrize("ranks", [2, 8]) +def test_gpu_e2e_remote_runs_on_cpu_and_keeps_rank_logs(tmp_path, ranks): + pytest.importorskip("torch") + bindir = Path(sys.executable).parent + if not (bindir / "hypertrain-miner").exists(): + pytest.skip("hypertrain-miner not installed next to this interpreter") + env = { + **os.environ, + "PATH": f"{bindir}:{os.environ['PATH']}", + "HT_E2E_BACKEND": "cpu", + "HT_E2E_GPUS": str(ranks), + "HT_E2E_PYTHON": sys.executable, + "HT_E2E_SRC": str(vast.ROOT), + "HT_E2E_OUT": str(tmp_path / "out"), + "HT_E2E_WORK": str(tmp_path / "work"), + } + script = vast.ROOT / "scripts/vast_e2e_remote.sh" + p = subprocess.run(["bash", str(script)], env=env, capture_output=True, text=True, timeout=900) + log = tmp_path / "out/network_e2e.log" + assert p.returncode == 0, log.read_text()[-2000:] if log.exists() else p.stderr + e2e = json.loads((tmp_path / "out/e2e.json").read_text()) + assert [r["auditor_match"] for r in e2e["rounds"]] == [True, True] and e2e["repeat_match"] + assert len(e2e["rounds"][0]["miner"]["files"]) == ranks * 4 # every rank's artifacts + # Every torchrun worker's stderr lands in the rescued out/ (the 8x5090 failure left none). + logs = sorted((tmp_path / "out/torchelastic").rglob("stderr.log")) + assert {p.parent.name for p in logs} == {str(r) for r in range(ranks)} + + +def test_gpu_e2e_remote_forces_math_attention_for_the_fixture_model(tmp_path): + # Real 2x RTX 5090 (instance 56659094): every rank died in sdpa-flash with "No available + # kernel"; the fixture model is fp32, head_dim 4. Probe the env the remote gives its python. + probe = tmp_path / "bin" + probe.mkdir() + env_dump = tmp_path / "env.json" + (probe / "python").write_text( + f"#!/bin/sh\nenv | grep -E '^(HT_ATTN_BACKEND|PET_)' > {env_dump}\nexit 1\n" + ) + (probe / "python").chmod(0o755) + env = { + **os.environ, + "HT_E2E_BACKEND": "cpu", + "HT_E2E_PYTHON": str(probe / "python"), + "HT_E2E_SRC": str(tmp_path), + "HT_E2E_OUT": str(tmp_path / "out"), + } + env.pop("HT_ATTN_BACKEND", None) + script = vast.ROOT / "scripts/vast_e2e_remote.sh" + subprocess.run(["bash", str(script)], env=env, capture_output=True, timeout=60, check=False) + seen = dict(line.split("=", 1) for line in env_dump.read_text().split()) + assert seen["HT_ATTN_BACKEND"] == "math" and seen["PET_TEE"] == "3" + + +def test_miner_cycle_profile_uses_new_remote_script(fake, tmp_path, capsys, mfu_globals): + a = vast.parse([*ARGS, "--profile", "miner-cycle"]) + assert a.remote_script == "vast_miner_cycle_remote.sh" and a.num_gpus == 1 + assert (vast.ROOT / "scripts" / a.remote_script).is_file() + rc, _, _ = run(fake, tmp_path, "--profile", "miner-cycle", "--dry-run") + assert rc == 0 and vast.NUM_GPUS == 1 and vast.IMAGE == vast.MINER_FLOW_IMAGE + assert json.loads(capsys.readouterr().out)["chosen_offer"]["id"] == 13 + + +def test_proofs_dry_run_prints_plan_with_unqualified_driver(fake, tmp_path, capsys, mfu_globals): + offers = fake.offers + fake.offers = lambda: [o | {"driver_version": "610.57.04"} for o in offers()] + rc, _, _ = run(fake, tmp_path, "--profile", "proofs", "--dry-run", minutes="45", usd="8") + chosen, plan = (json.loads(x) for x in capsys.readouterr().out.splitlines()) + assert rc == 0 and not any(m == "PUT" for m, _ in fake.calls) + assert ( + chosen["chosen_offer"]["id"] == 12 and chosen["chosen_offer"]["driver_qualified"] is False + ) + assert plan["driver_qualified"] is False and plan["caps"]["drivers"] == "any" + assert [s.split(":")[0] for s in plan["steps"]] == [n for n, _ in vast.PROOF_PLAN] + assert plan["tests"] == list(vast.PROOF_TESTS) and "@sha256:" in plan["image"] + assert plan["caps"]["boot_retries"] == 0 and plan["caps"]["num_gpus"] == 2 + assert "HT_E2E_PROOFS=***" in plan["remote_env"] and "HT_E2E_GPUS=***" in plan["remote_env"] + assert plan["worst_case_usd"] == chosen["chosen_offer"]["worst_case_usd"] + + +def test_proofs_eight_gpu_only_on_explicit_flag_and_qualified_driver( + fake, tmp_path, capsys, mfu_globals +): + base = fake.offers()[1] | {"num_gpus": 8} + fake.offers = lambda: [ + base | {"id": 50, "dph_total": 3.0, "driver_version": "610.57.04"}, + base | {"id": 51, "dph_total": 4.0, "driver_version": "580.178.04"}, + ] + with pytest.raises(SystemExit): + vast.parse([*ARGS, "--profile", "proofs", "--num-gpus", "8"]) + with pytest.raises(SystemExit): + vast.parse([*ARGS, "--profile", "proofs", "--eight-gpu-qualified", "--any-driver"]) + rc, _, _ = run( + fake, + tmp_path, + "--profile", + "proofs", + "--eight-gpu-qualified", + "--dry-run", + minutes="45", + usd="8", + ) + chosen, plan = (json.loads(x) for x in capsys.readouterr().out.splitlines()) + assert rc == 0 and fake.queries[-1]["num_gpus"] == {"eq": 8} + assert chosen["chosen_offer"]["id"] == 51 and plan["driver_qualified"] is True + + +def test_proofs_run_records_driver_qualified_false(fake, tmp_path, mfu_globals): + offers = fake.offers + fake.offers = lambda: [o | {"driver_version": "610.57.04"} for o in offers()] + rc, _, _ = run(fake, tmp_path, "--profile", "proofs", minutes="45", usd="8") + result = json.loads((tmp_path / "ev/harness-result.json").read_text()) + # ran fine and cleaned up, but the exit code does not claim a qualified-driver pass + assert rc == 4 and result["remote_ok"] is True + assert result["driver_qualified"] is False and result["absent"] is True + + +def test_driver_qualified_follows_the_fallback_host(fake, tmp_path, mfu_globals): + offers = fake.offers + fake.offers = lambda: [ + o | {"driver_version": "580.178.04" if o["id"] == 12 else "610.57.04"} for o in offers() + ] + + class StuckFirst(Scripted): + def ssh_probe(self, host, port): + return len(fake.asks) > 1 # qualified cheapest host never answers ssh + + rc, _, _ = run( + fake, + tmp_path, + "--profile", + "proofs", + "--boot-timeout", + "0.3", + "--boot-retries", + "1", + harness=StuckFirst, + minutes="45", + usd="8", + ) + result = json.loads((tmp_path / "ev/harness-result.json").read_text()) + assert rc == 4 and fake.asks == ["/api/v0/asks/12/", "/api/v0/asks/11/"] + assert result["driver_qualified"] is False + + +def test_proof_steps_and_tests_exist(): + remote = (vast.ROOT / "scripts/vast_e2e_remote.sh").read_text() + assert all(f"step {name} " in remote for name, _ in vast.PROOF_PLAN) + for node in vast.PROOF_TESTS: + path, name = node.split("::") + assert f"def {name}(" in (vast.ROOT / path).read_text(), node def test_remote_env_must_be_key_value(tmp_path): with pytest.raises(SystemExit): vast.parse(["--key-file", "k", "--max-usd", "1", "--max-minutes", "1", "--remote-env", "X"]) + + +def test_dry_run_only_reads_and_masks_remote_env_values(fake, tmp_path, capsys, mfu_globals): + rc, _, _ = run( + fake, tmp_path, "--profile", "proofs", "--dry-run", "--remote-env", "TOK=hunter2", + minutes="45", usd="8", + ) # fmt: skip + out = capsys.readouterr().out + assert rc == 0 and "hunter2" not in out and "TOK=***" in out + assert fake.calls and {m for m, _ in fake.calls} == {"GET"} + + +def test_pagination_keeps_an_existing_price_ceiling(fake, tmp_path, monkeypatch): + base = vast.offer_query + monkeypatch.setattr( + vast, "offer_query", lambda: {**base(), "limit": 1, "dph_total": {"lte": 0.7}} + ) + run(fake, tmp_path, "--dry-run") # refuses (page of one stalls); only the queries matter + assert len(fake.queries) > 1 and fake.queries[1]["dph_total"]["gte"] == 0.01 + assert all(q["dph_total"].get("lte") == 0.7 for q in fake.queries) + + +def test_journal_argv_never_holds_a_remote_env_value(): + masked = vast._mask_remote_env( + ["--remote-env", "TOK=hunter2", "--remote-env=B=s3cret", "--dry-run"], + ["TOK=hunter2", "B=s3cret"], + ) + assert "hunter2" not in masked and "s3cret" not in masked + assert "TOK=***" in masked and "B=***" in masked and "--dry-run" in masked + + +@pytest.mark.parametrize( + "spelling", [["--remote", "TOK=secret"], ["--remote-e", "TOK=secret"], ["--remote=TOK=secret"]] +) +def test_vast_remote_env_masked_for_abbreviated_flags(fake, tmp_path, spelling): + # An abbreviation must never reach the journal unmasked: it is refused before any write. + with pytest.raises(SystemExit): + run(fake, tmp_path, "--dry-run", *spelling) + journal = tmp_path / "ev/journal.jsonl" + assert not journal.exists() or "secret" not in journal.read_text() + # Masking keys on the parsed values, so any spelling that did parse is hidden too. + masked = vast._mask_remote_env(["--dry-run", *spelling], ["TOK=secret"]) + assert "secret" not in masked and "TOK=***" in masked diff --git a/tests/trainer/test_act_ckpt.py b/tests/trainer/test_act_ckpt.py index efc81dc..6b560b7 100644 --- a/tests/trainer/test_act_ckpt.py +++ b/tests/trainer/test_act_ckpt.py @@ -7,7 +7,7 @@ import hypertrain.trainer # noqa: F401 (determinism before torch) # isort: split -from test_golden_replay import GOLDEN +from test_golden_replay import _expect_golden from trainer_fixtures import assignment, make_cfg, sample from hypertrain.trainer.loop import train_round @@ -16,8 +16,28 @@ @pytest.mark.parametrize("moe", [False, True]) @pytest.mark.parametrize("dt", ["fp32", "bf16"]) -def test_act_ckpt_bitwise(monkeypatch: pytest.MonkeyPatch, moe: bool, dt: str) -> None: +def test_act_ckpt_bitwise( + monkeypatch: pytest.MonkeyPatch, moe: bool, dt: str, request: pytest.FixtureRequest +) -> None: monkeypatch.setenv("HT_ACT_CKPT", "1") cfg = make_cfg(moe=moe, opt="adamw", dtype=dt) res = train_round(cfg, init_params(cfg.model), assignment(cfg), sample(cfg)) - assert res.leaves_root in GOLDEN[f"loop-{'moe' if moe else 'dense'}-adamw-{dt}"] + _expect_golden(f"loop-{'moe' if moe else 'dense'}-adamw-{dt}", res.leaves_root, request) + + +@pytest.mark.parametrize("moe", [False, True]) +@pytest.mark.parametrize("dt", ["fp32", "bf16"]) +def test_act_ckpt_matches_plain_in_one_process( + monkeypatch: pytest.MonkeyPatch, moe: bool, dt: str +) -> None: + # Hardware-independent: GOLDEN misses on an unknown runner CPU only xfail above. + cfg = make_cfg(moe=moe, opt="adamw", dtype=dt) + roots = [] + for ckpt in (False, True): + if ckpt: + monkeypatch.setenv("HT_ACT_CKPT", "1") + else: + monkeypatch.delenv("HT_ACT_CKPT", raising=False) + res = train_round(cfg, init_params(cfg.model), assignment(cfg), sample(cfg)) + roots.append(res.leaves_root) + assert roots[0] == roots[1] diff --git a/tests/trainer/test_attention.py b/tests/trainer/test_attention.py index 08def6a..8caaa11 100644 --- a/tests/trainer/test_attention.py +++ b/tests/trainer/test_attention.py @@ -4,6 +4,8 @@ import math import os +from collections.abc import Callable +from typing import Any import pytest @@ -12,6 +14,7 @@ # isort: split import torch +from hypertrain.trainer import attention as A from hypertrain.trainer.attention import Backend, attention, select_backend @@ -74,3 +77,74 @@ def test_backward_bitwise() -> None: from scripts.attention_spike import spike assert spike(torch.device("cuda", int(os.environ["HT_GENUINE_CUDA_INDEX"])))["chosen"] + + +def _sdpa_raising(when: Callable[[bool], bool], calls: list[bool]) -> Callable[..., torch.Tensor]: + """F.scaled_dot_product_attention raising the RTX 5090 error when when(enable_gqa).""" + real = A.F.scaled_dot_product_attention + + def sdpa(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, **kw: Any) -> torch.Tensor: + calls.append(kw["enable_gqa"]) + if when(kw["enable_gqa"]): + raise RuntimeError("No available kernel. Aborting execution.") + return real(q, k, v, **kw) + + return sdpa + + +def test_fallback_probed_once_when_native_kernel_missing( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + calls: list[bool] = [] + monkeypatch.setattr(A, "_PATHS", {}) + monkeypatch.setattr(A.F, "scaled_dot_product_attention", _sdpa_raising(lambda gqa: gqa, calls)) + g = torch.Generator().manual_seed(0) + q, k, v = (torch.randn(2, n, 33, 16, generator=g) for n in (8, 2, 2)) + with caplog.at_level("WARNING", logger=A.__name__): + first = attention(q, k, v, "sdpa-flash") + probed = list(calls) + second = attention(q, k, v, "sdpa-flash") + assert probed[0] and len(probed) > 1 # native enable_gqa tried first, then expanded K/V + assert calls[len(probed) :] == [False] # no re-probe + assert [r.getMessage().split()[2] for r in caplog.records] == ["flash+expanded-kv"] + assert A.chosen_paths() == ["flash+expanded-kv"] + assert torch.equal(first, second) + assert (first - attention(q, k, v)).abs().max() < 1e-5 + + +def test_no_flash_path_stops_instead_of_serving_another_kernel( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Only flash is qualified: no mem-efficient and no math fallback under the sdpa-flash name.""" + calls: list[bool] = [] + monkeypatch.setattr(A, "_PATHS", {}) + monkeypatch.setattr(A.F, "scaled_dot_product_attention", _sdpa_raising(lambda gqa: True, calls)) + q, k, v = (torch.randn(2, n, 33, 16) for n in (8, 2, 2)) + with pytest.raises(RuntimeError, match="no flash path"): + attention(q, k, v, "sdpa-flash") + assert A.PATHS == ("flash", "flash+expanded-kv") + assert A._PATHS == {} # a refusal is not cached as a choice + + +@pytest.mark.parametrize("dt", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize("h,kvh", [(8, 2), (8, 1)]) +def test_expanded_kv_close_to_native_gqa(dt: torch.dtype, h: int, kvh: int) -> None: + g = torch.Generator().manual_seed(h * 7 + kvh) + q, k, v = (torch.randn(2, n, 33, 16, generator=g).to(dt) for n in (h, kvh, kvh)) + native = A._sdpa("flash", q, k, v) + expanded = A._sdpa("flash+expanded-kv", q, k, v) + tol = 2e-2 if dt == torch.bfloat16 else 1e-5 + assert (native.float() - expanded.float()).abs().max() < tol + + +def test_layout_rejected_after_probe_stops(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(A, "_PATHS", {}) + g = torch.Generator().manual_seed(0) + q, k = (torch.randn(2, n, 33, 16, generator=g) for n in (8, 2)) + v = torch.randn(2, 2, 16, 33, generator=g).transpose(-1, -2) # stride(-1) != 1: no fused kernel + assert A._path(q.device, q.dtype, 4, 16) == "flash" # the contiguous probe accepts flash + with pytest.raises(RuntimeError, match="No available kernel"): + attention(q, k, v, "sdpa-flash") # no silent math: the run stops + assert torch.equal( + attention(q, k, v.contiguous(), "sdpa-flash"), A._sdpa("flash", q, k, v.contiguous()) + ) diff --git a/tests/trainer/test_attention_qualified.py b/tests/trainer/test_attention_qualified.py index a909225..d34909d 100644 --- a/tests/trainer/test_attention_qualified.py +++ b/tests/trainer/test_attention_qualified.py @@ -69,3 +69,5 @@ def test_qualified_backend_repeats_bitwise() -> None: h.update(x.contiguous().view(torch.uint8).cpu().numpy().tobytes()) digests.add(h.hexdigest()) assert len(digests) == 1 + if backend == "sdpa-flash": # the qualification record covers flash with enable_gqa only + assert A.chosen_paths() == ["flash"] diff --git a/tests/trainer/test_engine_flags.py b/tests/trainer/test_engine_flags.py index 75acd6a..52f8978 100644 --- a/tests/trainer/test_engine_flags.py +++ b/tests/trainer/test_engine_flags.py @@ -10,7 +10,7 @@ # isort: split import torch -from test_golden_replay import GOLDEN +from test_golden_replay import _expect_golden from trainer_fixtures import assignment, make_cfg, sample from hypertrain.trainer.codec_v2 import decode_v2 @@ -41,11 +41,13 @@ def test_codec_v2_roundtrip_same_ef(codec: str) -> None: assert rep.result == "MATCH" -def test_fp8_off_by_default(monkeypatch: pytest.MonkeyPatch) -> None: +def test_fp8_off_by_default( + monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest +) -> None: monkeypatch.delenv("HT_FP8", raising=False) cfg = make_cfg(dtype="bf16") assert not fp8_enabled(cfg.model) - assert _run(cfg).leaves_root in GOLDEN["loop-dense-adamw-bf16"] + _expect_golden("loop-dense-adamw-bf16", _run(cfg).leaves_root, request) def test_fp8_flag_repeatable(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/trainer/test_golden_replay.py b/tests/trainer/test_golden_replay.py index 4b943ae..aa1e3b6 100644 --- a/tests/trainer/test_golden_replay.py +++ b/tests/trainer/test_golden_replay.py @@ -2,6 +2,8 @@ from __future__ import annotations +import platform +import subprocess import sys from pathlib import Path @@ -85,21 +87,55 @@ }, } +XFAIL_REASON = ( + "golden leaf root differs on this CPU class; unexplained hardware dependence, " + "see livrables/forge/01-inconnus.md (golden replay)" +) + + +def _cpu() -> str: + try: + out = subprocess.run(["lscpu"], capture_output=True, text=True, check=False).stdout + except OSError: # no lscpu (non-Linux) + out = "" + model = [ln.split(":", 1)[1].strip() for ln in out.splitlines() if ln.startswith("Model name:")] + return model[0] if model else platform.processor() + + +def _expect_golden(key: str, root: str, request: pytest.FixtureRequest) -> None: + # A root outside GOLDEN is a reported xfail, not a red job: the cause is not yet known. + if root not in GOLDEN[key]: + pytest.xfail(f"{XFAIL_REASON}; test={request.node.nodeid} root={root} cpu={_cpu()}") + @pytest.mark.parametrize("moe", [False, True]) @pytest.mark.parametrize("opt", ["adamw", "muon"]) @pytest.mark.parametrize("dt", ["fp32", "bf16"]) -def test_loop_leaves_root_unchanged(moe: bool, opt: str, dt: str) -> None: +def test_loop_leaves_root_unchanged( + moe: bool, opt: str, dt: str, request: pytest.FixtureRequest +) -> None: cfg = make_cfg(moe=moe, opt=opt, dtype=dt) res = train_round(cfg, init_params(cfg.model), assignment(cfg), sample(cfg)) - assert res.leaves_root in GOLDEN[f"loop-{'moe' if moe else 'dense'}-{opt}-{dt}"] + _expect_golden(f"loop-{'moe' if moe else 'dense'}-{opt}-{dt}", res.leaves_root, request) @pytest.mark.parametrize("name", sorted(lf.LAYOUTS)) -def test_island_leaves_root_unchanged(name: str) -> None: +def test_island_leaves_root_unchanged(name: str, request: pytest.FixtureRequest) -> None: cfg, lay, a, get = lf.setup(name) th = init_params(cfg.model) roots = { r.leaves_root for r in emulate(lay.n_gpus, lambda c: train_island(cfg, lay, c, th, a, get)) } - assert len(roots) == 1 and roots <= GOLDEN["island-" + name] + assert len(roots) == 1 + _expect_golden("island-" + name, roots.pop(), request) + + +@pytest.mark.parametrize("moe", [False, True], ids=["dense", "moe"]) +def test_leaves_root_is_deterministic_in_process(moe: bool) -> None: + # Hardware-independent guard: holds on any CPU class, unlike GOLDEN. + cfg = make_cfg(moe=moe) + a, b = ( + train_round(cfg, init_params(cfg.model), assignment(cfg), sample(cfg)).leaves_root + for _ in range(2) + ) + assert a == b