diff --git a/.github/workflows/heavy.yml b/.github/workflows/heavy.yml index b0c8290..b05869a 100644 --- a/.github/workflows/heavy.yml +++ b/.github/workflows/heavy.yml @@ -141,4 +141,23 @@ jobs: enable-cache: true cache-dependency-glob: "uv.lock" - run: uv sync --locked --all-extras --group dev - - run: bash scripts/check.sh --ignore=tests/sim --ignore=tests/e2e + # The runner image ships its own kind; the od relay tests and relay_kind_smoke.py + # require the kind_version pinned in deploy/k8s/versions.json. + - name: Install pinned kind + env: + KIND_SHA256_V0_31_0: eb244cbafcc157dff60cf68693c14c9a75c4e6e6fedaf9cd71c58117cb93e3fa + run: | + set -euo pipefail + v="$(python3 -c 'import json; print(json.load(open("deploy/k8s/versions.json"))["kind_version"])')" + if [ "$v" != v0.31.0 ]; then + echo "kind_version $v: update KIND_SHA256 in heavy.yml" >&2 + exit 1 + fi + mkdir -p "$RUNNER_TEMP/kind-bin" + curl -fsSLo "$RUNNER_TEMP/kind-bin/kind" \ + "https://github.com/kubernetes-sigs/kind/releases/download/$v/kind-linux-amd64" + echo "$KIND_SHA256_V0_31_0 $RUNNER_TEMP/kind-bin/kind" | sha256sum -c - + chmod 755 "$RUNNER_TEMP/kind-bin/kind" + "$RUNNER_TEMP/kind-bin/kind" version | grep -q "^kind $v " + echo "$RUNNER_TEMP/kind-bin" >> "$GITHUB_PATH" + - run: bash scripts/check.sh -rs --ignore=tests/sim --ignore=tests/e2e diff --git a/scripts/publish-allowlist.txt b/scripts/publish-allowlist.txt index e0b0a01..3c04e93 100644 --- a/scripts/publish-allowlist.txt +++ b/scripts/publish-allowlist.txt @@ -175,6 +175,7 @@ src/hypertrain/challenge/store.py:5bc023ca9af1ff8eedaaaa32719611ded1b93836fa6d87 # test fixture value (synthetic, not a credential) tests/auditor/aud_fixtures.py:0d949c3ecc7d9c4044e4279b45bb3f003b569a42beb0ebd5c8fcda1b7b9fb6a5 # L42 +tests/beacon/test_push.py:c6a44b08a042c2fefeb9f4d0dc13aaaa27d6eafba8731c15ad3fd27163c7f336 # L282 tests/challenge/test_admission_v2.py:d6b19e1e09232f110810ee8bd506a4497a0cf7b0f0f6ce9f3d21eecb968c96a8 # L56 tests/challenge/test_contract.py:d8a58e1216b37763ba783aa441e27e1a6d1d4eeb7bebe4cb14d995d1c2072f2b # L37 tests/challenge/test_contract.py:dca6991abfb1e7eb0cf06ef5c6a6389c99d303a012b49e910c44694638086e6f # L77 @@ -185,6 +186,7 @@ tests/challenge/test_service_network_v2.py:39c103316286f9ef21526344b85dca1c86a93 tests/challenge/test_service_network_v2.py:6db3469c33b8debcaeb7597fb238d8a434cce7f3be4fd6c61247d8d5c49c4c03 # L2529 tests/challenge/test_service_network_v2.py:355ba43d75f8d5a922f885c3166b107ca10c387429177317b2a15951abb4f039 # L2536 tests/challenge/test_service_network_v2.py:3d09f1936ce966319dc2d46b6af880c97593e7fe3c35b1b88bd942e907cfa782 # L2551 +tests/datasets/test_pin.py:2e51842f098911cc7c612015b272581ea20449ba26d4d3e04c34429a1f930ebc # L19 tests/e2e/test_service_network_v2_e2e.py:3d09f1936ce966319dc2d46b6af880c97593e7fe3c35b1b88bd942e907cfa782 # L1394 tests/e2e/test_service_network_v2_e2e.py:a98684130926595a80aa7463d4c6632b80945404481c901f15b3dcbe4d98a3d0 # L1441 tests/gpu_ops/conftest.py:57fc89f6e67b8940a8bfc8afe90becae2443a421a57798ec03d1afe3fd8060f7 # L19 diff --git a/tests/challenge/test_capacity_outer_v2.py b/tests/challenge/test_capacity_outer_v2.py index 4e7a604..7d0dd4e 100644 --- a/tests/challenge/test_capacity_outer_v2.py +++ b/tests/challenge/test_capacity_outer_v2.py @@ -193,6 +193,7 @@ def test_request_metadata_rejects_without_tensor_allocation(tiny, tmp_path, monk OuterRequest.parse(data) +@pytest.mark.usefixtures("systemd_manager") def test_actual_bounded_outer_child_matches_original_signed_replay(tiny, tmp_path, monkeypatch): from hypertrain.aggregator.core import OuterState from hypertrain.aggregator.tape_v2 import TapeV2, make_tape, replay_tape @@ -672,6 +673,7 @@ def test_genesis_authority_rejects_before_kernel(genesis_tiny, monkeypatch, atta ).fetchone() +@pytest.mark.usefixtures("systemd_manager") def test_actual_bounded_genesis_preparation_and_duplicate_debit( genesis_tiny, tmp_path, monkeypatch ): diff --git a/tests/challenge/test_service_capacity_v2.py b/tests/challenge/test_service_capacity_v2.py index f3e2a18..ebd7a1a 100644 --- a/tests/challenge/test_service_capacity_v2.py +++ b/tests/challenge/test_service_capacity_v2.py @@ -716,6 +716,7 @@ def prior_bytes(path): assert store._record_v2(run_id, "service-admission", "run") == prior +@pytest.mark.usefixtures("systemd_manager") def test_real_capacity_launch_failure_keeps_debit_and_prevents_restart_rerun( tiny, tmp_path, monkeypatch ): diff --git a/tests/conftest.py b/tests/conftest.py index 1123b67..2ccb965 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,10 @@ import json import os +import subprocess +import uuid from pathlib import Path +from typing import Literal import numpy as np import pytest @@ -110,3 +113,61 @@ def corpus_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: d = build_corpus(tmp_path_factory.mktemp("corpus")) os.environ["HYPERTRAIN_DATA_DIR"] = str(d) return d + + +# Messages meaning "this user/host has no usable system manager", not "the probe broke". +_SYSTEMD_UNAVAILABLE = ( + "Interactive authentication required", + "Access denied", + "Failed to connect to bus", + "System has not been booted with systemd", + "Permission denied", +) + + +def _classify_systemd_probe(returncode: int, stderr: str) -> Literal["ok", "unavailable", "error"]: + if returncode == 0: + return "ok" + return "unavailable" if any(m in stderr for m in _SYSTEMD_UNAVAILABLE) else "error" + + +@pytest.fixture(scope="session") +def systemd_manager() -> None: + """Skip unless this user may start transient units on the system manager, as the product does. + + Non-root skips only when the manager is absent or authorization is denied; any other probe + failure (EAGAIN, failed transient unit, timeout) is a broken host and fails. Root that cannot + is a broken host, not an unprivileged one: fail, so the heavy `native` job (isolated root + environment) can never turn green by skipping. + """ + try: + probe = subprocess.run( + [ + "systemd-run", + "--unit=ht-probe-" + uuid.uuid4().hex, + "--wait", + "--collect", + "--quiet", + "--no-ask-password", + "--property=MemoryMax=64M", + "--property=CPUQuota=50%", + "/bin/true", + ], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + verdict = _classify_systemd_probe(probe.returncode, probe.stderr) + reason = f"rc={probe.returncode} {probe.stderr.strip()[-500:]}" + except FileNotFoundError as exc: + verdict, reason = "unavailable", repr(exc) + except (OSError, subprocess.TimeoutExpired) as exc: + verdict, reason = "error", repr(exc) + if verdict == "ok": + return + if os.geteuid() == 0: + pytest.fail(f"systemd-run as root failed: {reason}") + if verdict == "error": + pytest.fail(f"systemd-run probe failed: {reason}") + pytest.skip(f"system manager transient units unavailable: {reason}") diff --git a/tests/miner/test_capacity_runtime_v2.py b/tests/miner/test_capacity_runtime_v2.py index a9c3021..91ed320 100644 --- a/tests/miner/test_capacity_runtime_v2.py +++ b/tests/miner/test_capacity_runtime_v2.py @@ -23,6 +23,7 @@ def small(monkeypatch, tmp_path): return launch.CapacityAttempt("11" * 32, "22" * 32, tmp_path / "runtime.lock", lambda: True) +@pytest.mark.usefixtures("systemd_manager") @pytest.mark.parametrize("mode", ["success", "oom", "cancel", "deadline"]) def test_actual_small_kernel_attempt_and_cleanup(tmp_path, small, mode): """Subscribe readiness before triggering real bounded command and await pidfd exit.""" @@ -145,6 +146,7 @@ def test_missing_manager_permission_has_no_unbounded_fallback(tmp_path, small, m assert not (tmp_path / "attempt/capacity-observed.json").exists() +@pytest.mark.usefixtures("systemd_manager") def test_real_runner_parent_death_retains_absolute_manager_timer(tmp_path, small): """Fork only a stdlib waiting controller; manager timer survives its SIGKILL.""" path = str(tmp_path / "dead.sock") diff --git a/tests/miner/test_island_launch_v2.py b/tests/miner/test_island_launch_v2.py index 168e042..dcd0fea 100644 --- a/tests/miner/test_island_launch_v2.py +++ b/tests/miner/test_island_launch_v2.py @@ -304,6 +304,7 @@ def test_noisy_failure_retains_only_bounded_error_tail( assert not (tmp_path / "published").exists() +@pytest.mark.usefixtures("systemd_manager") def test_capacity_branch_uses_real_bounded_failure_before_publication(tmp_path, monkeypatch): import os import sys diff --git a/tests/relay/test_transport_v2.py b/tests/relay/test_transport_v2.py index 79f363a..0676969 100644 --- a/tests/relay/test_transport_v2.py +++ b/tests/relay/test_transport_v2.py @@ -1822,6 +1822,7 @@ def test_image_archive_stdin_through_capture_and_native_shim(tmp_path: Path) -> assert json.loads(journal.read_bytes()) == {"cluster": "hypertrain-v2-stream"} +@pytest.mark.usefixtures("systemd_manager") def test_launch_safety_independent_systemd_units(tmp_path: Path) -> None: import json import os diff --git a/tests/test_launch_cuda_run.py b/tests/test_launch_cuda_run.py index fd65461..de49ca2 100644 --- a/tests/test_launch_cuda_run.py +++ b/tests/test_launch_cuda_run.py @@ -8,7 +8,18 @@ def _run(tmp_path: Path, *args: str) -> subprocess.CompletedProcess[str]: - env = dict(os.environ, RD=str(tmp_path), STATE=str(tmp_path / "state"), PY="python3") + # Stub the frozen tree (checked fail-closed before preflight) and pin TOOLS under tmp_path. + tree = tmp_path / "tree" + (tree / "experiments/gpu_network_v2").mkdir(parents=True) + (tree / "experiments/gpu_network_v2/orchestrate.py").touch() + env = dict( + os.environ, + RD=str(tmp_path), + STATE=str(tmp_path / "state"), + PY="python3", + HT_TREE=str(tree), + TOOLS=str(tmp_path / "tools"), + ) return subprocess.run( ["bash", str(SCRIPT), *args], capture_output=True, text=True, env=env, timeout=60 ) diff --git a/tests/test_publish_exclusions.py b/tests/test_publish_exclusions.py index 30c4573..f97a984 100644 --- a/tests/test_publish_exclusions.py +++ b/tests/test_publish_exclusions.py @@ -358,6 +358,21 @@ def test_reviewed_allowlist_is_exact_path_and_line(tmp_path: Path) -> None: ) +def test_reviewed_fixture_lines_do_not_exempt_secrets_elsewhere(tmp_path: Path) -> None: + # The real allowlist exempts these fixture lines only at their exact path and bytes. + source = Path(__file__).resolve().parents[1] + (tmp_path / "scripts").mkdir() + (tmp_path / "scripts/publish-allowlist.txt").write_bytes( + (source / "scripts/publish-allowlist.txt").read_bytes() + ) + for fixture, number in (("tests/beacon/test_push.py", 282), ("tests/datasets/test_pin.py", 19)): + line = (source / fixture).read_text().splitlines()[number - 1] + assert run_scanner(tmp_path, fixture, line + "\n").returncode == 0 + assert run_scanner(tmp_path, "src/hypertrain/leak.py", line + "\n").returncode == 1 + genuine = 'PATH_SECRET = "sk_" + "' + "Zp9Qw2Lm7Xc" + "4Vb8Nk3Rt6Yh1Jd5" + '"' + assert run_scanner(tmp_path, "tests/beacon/test_push.py", genuine + "\n").returncode == 1 + + def test_current_repo_tree_scan_clean() -> None: source = Path(__file__).resolve().parents[1] script = (source / "scripts/publish.sh").read_text() diff --git a/tests/test_systemd_probe_classify.py b/tests/test_systemd_probe_classify.py new file mode 100644 index 0000000..59c8bf3 --- /dev/null +++ b/tests/test_systemd_probe_classify.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import pytest + +from conftest import _classify_systemd_probe + +DENIED = [ + "Failed to start transient service unit: Interactive authentication required.", + "Failed to start transient service unit: Access denied", + "Failed to connect to bus: No such file or directory", + "System has not been booted with systemd as init system (PID 1). Can't operate.", + "Failed to create bus connection: Permission denied", +] + + +def test_ok() -> None: + assert _classify_systemd_probe(0, "") == "ok" + + +@pytest.mark.parametrize("stderr", DENIED) +def test_unavailable(stderr: str) -> None: + assert _classify_systemd_probe(1, stderr) == "unavailable" + + +@pytest.mark.parametrize( + "stderr", + ["Failed to start transient service unit: Resource temporarily unavailable (EAGAIN)", ""], +) +def test_error(stderr: str) -> None: + assert _classify_systemd_probe(1, stderr) == "error"