From efb8f53132951d6e56047d9a02efbce9efce6316 Mon Sep 17 00:00:00 2001 From: NumberWan Date: Fri, 30 Oct 2026 15:15:42 +0800 Subject: [PATCH 01/23] feat(harness): add an offline parallel-round coordinator Give one round a plan-supplied attempt budget and a fail-closed execution boundary so lanes can submit synthetic candidates without sharing a device lease. Refs #249 Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_round.py | 305 +++++++++++++++++++++++++ src/inferrsi/harness/round_budget.py | 215 +++++++++++++++++ src/inferrsi/infra/confinement.py | 206 +++++++++++++++++ src/inferrsi/infra/device_lease.py | 40 ++++ tests/test_confinement.py | 85 +++++++ tests/test_parallel_round.py | 196 ++++++++++++++++ tests/test_round_budget.py | 68 ++++++ tools/parallel_round.py | 62 +++++ 8 files changed, 1177 insertions(+) create mode 100644 src/inferrsi/harness/parallel_round.py create mode 100644 src/inferrsi/harness/round_budget.py create mode 100644 src/inferrsi/infra/confinement.py create mode 100644 src/inferrsi/infra/device_lease.py create mode 100644 tests/test_confinement.py create mode 100644 tests/test_parallel_round.py create mode 100644 tests/test_round_budget.py create mode 100644 tools/parallel_round.py diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py new file mode 100644 index 000000000..8a37780ea --- /dev/null +++ b/src/inferrsi/harness/parallel_round.py @@ -0,0 +1,305 @@ +"""Prepare, submit and measure one offline parallel round. + +Directory diffs stand in for git clones. ``close`` and incumbent advancement are +not implemented. A synthetic latency number is not a campaign verdict. +""" + +import json +import shutil +import time +from datetime import datetime +from fnmatch import fnmatch +from pathlib import Path + +from inferrsi.harness.round_budget import ( + Binding, + BudgetError, + Entry, + Genesis, + LaneKey, + Ledger, + admit_attempt, + advance, + empty_ledger, +) +from inferrsi.infra.artifacts import digest, sha256, write_json +from inferrsi.infra.confinement import ConfinementUnavailable, confined_execute +from inferrsi.infra.device_lease import DeviceLease, LeaseError + +PLANES = {"harness", "model", "infra"} + + +class RoundPlanError(ValueError): + """The frozen round plan is internally inconsistent.""" + + +def _match(path: str, patterns: list[str]) -> bool: + return any(fnmatch(path, pattern) for pattern in patterns) + + +def _files(root: Path) -> list[str]: + found = [] + for path in sorted(root.rglob("*")): + if not path.is_file() or ".git" in path.parts: + continue + found.append(path.relative_to(root).as_posix()) + return found + + +def _lane(plan: dict, lane_id: str) -> dict: + matches = [lane for lane in plan["lanes"] if lane["id"] == lane_id] + if len(matches) != 1: + raise RoundPlanError(f"unknown lane: {lane_id}") + return matches[0] + + +def _hashes(root: Path) -> dict[str, str]: + return {relative: sha256(root / relative) for relative in _files(root)} + + +def _changed(incumbent: Path, work: Path) -> list[str]: + before, after = _hashes(incumbent), _hashes(work) + return sorted(path for path in set(before) | set(after) if before.get(path) != after.get(path)) + + +def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: + """Validate lane scopes and write ``round-plan.json``. The digest covers the validated body.""" + if not isinstance(spec, dict): + raise RoundPlanError("round spec must be an object") + lanes = spec.get("lanes") + if not isinstance(lanes, list) or not lanes: + raise RoundPlanError("round spec needs at least one lane") + protected = spec.get("protected", []) + if not isinstance(protected, list): + raise RoundPlanError("protected paths must be a list of globs") + seen_entries: set[str] = set() + seen_ids: set[str] = set() + files = _files(incumbent) + normalized = [] + for lane in lanes: + lane_id = lane.get("id") + entry = lane.get("entry") + plane = lane.get("plane") + globs = lane.get("globs") + if not isinstance(lane_id, str) or not lane_id or lane_id in seen_ids: + raise RoundPlanError("each lane needs its own id") + if not isinstance(entry, str) or not entry: + raise RoundPlanError("each lane needs one search-plan entry") + if entry in seen_entries: + raise RoundPlanError(f"two lanes pursue {entry}") + if type(plane) is not str or plane not in PLANES: + raise RoundPlanError("a lane must name exactly one plane") + if not isinstance(globs, list) or not globs or any(type(item) is not str for item in globs): + raise RoundPlanError("a lane needs path globs") + if type(lane.get("cap")) is not int or type(lane.get("per_attempt_worst_case")) is not int: + raise RoundPlanError("cap and per-attempt cost must come from the plan") + if not isinstance(lane.get("deadline_utc"), str): + raise RoundPlanError("each lane needs a deadline") + if any(_match(path, globs) and _match(path, protected) for path in files): + raise RoundPlanError(f"{lane_id} reaches a protected path") + seen_ids.add(lane_id) + seen_entries.add(entry) + normalized.append(dict(lane)) + for index, lane in enumerate(normalized): + for other in normalized[index + 1 :]: + shared = set(lane["globs"]) & set(other["globs"]) + shared.update( + path + for path in files + if _match(path, lane["globs"]) and _match(path, other["globs"]) + ) + if shared: + raise RoundPlanError(f"{lane['id']} overlaps {other['id']}") + body = { + "round_id": spec["round_id"], + "device_id": spec["device_id"], + "incumbent_id": spec["incumbent_id"], + "protected": protected, + "build": list(spec.get("build", [])), + "trial": list(spec.get("trial", [])), + "lanes": normalized, + } + body["round_digest"] = digest(body) + round_dir.mkdir(parents=True, exist_ok=True) + write_json(round_dir / "round-plan.json", body) + return body + + +def submit(round_dir: Path, lane_id: str, incumbent: Path, work: Path) -> dict: + """Copy in-glob changes to the outbox. The same manifest always returns the same ticket.""" + plan = json.loads((round_dir / "round-plan.json").read_text(encoding="utf-8")) + lane = _lane(plan, lane_id) + changed = _changed(incumbent, work) + outside = [path for path in changed if not _match(path, lane["globs"])] + if outside or not changed: + return {"submitted": False, "outside": outside, "ticket": None} + manifest = {path: sha256(work / path) for path in changed} + ticket = digest( + {"round_digest": plan["round_digest"], "lane_id": lane_id, "manifest": manifest} + )[:16] + destination = round_dir / "outbox" / ticket + if not destination.exists(): + destination.mkdir(parents=True) + for path in changed: + target = destination / path + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(work / path, target) + write_json( + destination / "manifest.json", + {"ticket": ticket, "lane_id": lane_id, "files": manifest}, + ) + return {"submitted": True, "outside": [], "ticket": ticket} + + +def _ledger_path(round_dir: Path, lane_id: str) -> Path: + return round_dir / "ledgers" / f"{lane_id}.json" + + +def _load_ledger(round_dir: Path, plan: dict, lane: dict): + path = _ledger_path(round_dir, lane["id"]) + if not path.exists(): + return empty_ledger(plan["round_digest"], lane["id"], lane["cap"]) + raw = json.loads(path.read_text(encoding="utf-8")) + entries = tuple( + Entry( + Binding(item["plan_digest"], item["run_id"], item["worst_case"]), + item["phase"], + item["actual_cost"], + ) + for item in raw["entries"] + ) + return Ledger( + LaneKey(raw["round_digest"], raw["lane_id"]), + Genesis(raw["evidence"], raw["legacy_charge"]), + raw["cap"], + raw["revision"], + entries, + ) + + +def _save_ledger(round_dir: Path, ledger) -> None: + write_json( + _ledger_path(round_dir, ledger.key.lane_id), + { + "round_digest": ledger.key.round_digest, + "lane_id": ledger.key.lane_id, + "evidence": ledger.genesis.evidence, + "legacy_charge": ledger.genesis.legacy_charge, + "cap": ledger.cap, + "revision": ledger.revision, + "entries": [ + { + "plan_digest": entry.binding.plan_digest, + "run_id": entry.binding.run_id, + "worst_case": entry.binding.worst_case, + "phase": entry.phase, + "actual_cost": entry.actual_cost, + } + for entry in ledger.entries + ], + }, + ) + + +def _materialize(incumbent: Path, overlay: Path, destination: Path) -> None: + if destination.exists(): + shutil.rmtree(destination) + shutil.copytree(incumbent, destination, ignore=shutil.ignore_patterns("manifest.json")) + manifest = json.loads((overlay / "manifest.json").read_text(encoding="utf-8")) + for relative in manifest["files"]: + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(overlay / relative, target) + + +def _command(template: list[str], result: Path) -> tuple[str, ...]: + return tuple(part.replace("{result}", str(result)) for part in template) + + +def serve_ticket(round_dir: Path, ticket: str, incumbent: Path, *, backend, now: datetime) -> dict: + """Measure one ticket. A busy device leaves it queued and does not reserve an attempt.""" + plan = json.loads((round_dir / "round-plan.json").read_text(encoding="utf-8")) + result_path = round_dir / "results" / f"{ticket}.json" + if result_path.exists(): + return json.loads(result_path.read_text(encoding="utf-8")) + overlay = round_dir / "outbox" / ticket + manifest = json.loads((overlay / "manifest.json").read_text(encoding="utf-8")) + lane = _lane(plan, manifest["lane_id"]) + try: + held = DeviceLease(round_dir / "leases").acquire(plan["device_id"]) + except LeaseError: + return {"ticket": ticket, "status": "queued"} + try: + try: + loaded = _load_ledger(round_dir, plan, lane) + ledger, binding = admit_attempt(plan, loaded, lane["id"], now) + except BudgetError as exc: + record = {"ticket": ticket, "status": "rejected", "reason": str(exc), "synthetic": True} + write_json(result_path, record) + return record + _save_ledger(round_dir, ledger) + candidate = round_dir / "candidates" / ticket + trial_dir = round_dir / "trials" / ticket + build_dir = candidate / "build-output" + try: + _materialize(incumbent, overlay, candidate) + build_dir.mkdir(parents=True, exist_ok=True) + trial_dir.mkdir(parents=True, exist_ok=True) + if plan["build"]: + confined_execute( + tuple(plan["build"]), + candidate, + (candidate, build_dir), + (), + time.monotonic() + 30, + backend=backend, + ) + candidate_result = trial_dir / "candidate.json" + confined_execute( + _command(plan["trial"], candidate_result), + candidate, + (trial_dir,), + (candidate,), + time.monotonic() + 30, + backend=backend, + ) + incumbent_result = trial_dir / "incumbent.json" + confined_execute( + _command(plan["trial"], incumbent_result), + incumbent, + (trial_dir,), + (incumbent,), + time.monotonic() + 30, + backend=backend, + ) + except ConfinementUnavailable as exc: + _save_ledger(round_dir, advance(ledger, binding, "cancelled")) + record = {"ticket": ticket, "status": "rejected", "reason": str(exc), "synthetic": True} + write_json(result_path, record) + return record + candidate_body = json.loads(candidate_result.read_text(encoding="utf-8")) + incumbent_body = json.loads(incumbent_result.read_text(encoding="utf-8")) + candidate_metric = candidate_body["metrics"]["latency_ms"] + incumbent_metric = incumbent_body["metrics"]["latency_ms"] + finished = advance(advance(ledger, binding, "claimed"), binding, "finished", actual_cost=0) + _save_ledger(round_dir, finished) + record = { + "ticket": ticket, + "status": "measured", + "synthetic": True, + "candidate_latency_ms": candidate_metric, + "incumbent_latency_ms": incumbent_metric, + "gain_ms": incumbent_metric - candidate_metric, + "confinement": getattr(backend, "name", "unknown"), + } + write_json(result_path, record) + return record + finally: + held.release() + + +def ticket_status(round_dir: Path, ticket: str) -> dict: + path = round_dir / "results" / f"{ticket}.json" + if path.exists(): + return json.loads(path.read_text(encoding="utf-8")) + return {"ticket": ticket, "status": "queued"} diff --git a/src/inferrsi/harness/round_budget.py b/src/inferrsi/harness/round_budget.py new file mode 100644 index 000000000..64cbee56c --- /dev/null +++ b/src/inferrsi/harness/round_budget.py @@ -0,0 +1,215 @@ +"""Lane attempt budgets for one parallel round. + +The transitions match ``tools/skill_eval_budget.py``: a reservation is a +permanent charge, and the same plan digest cannot be reserved twice. The +identity here is a lane, and each attempt mints its own plan digest from the +unchanged round digest, lane id and attempt ordinal. The cap comes from the +frozen round plan. This module does not authorize a skill-eval pull-request +budget, and it does not confine the filesystem. +""" + +import hashlib +import json +from dataclasses import dataclass, replace +from datetime import datetime +from pathlib import Path + + +class BudgetError(ValueError): + """Invalid, exhausted, expired, or unsafe lane-budget state.""" + + +def _text(value: str) -> None: + if type(value) is not str or not value.strip() or len(value) > 512: + raise BudgetError("Expected a bounded nonempty identifier") + + +def _digest(value: str) -> None: + digits = "0123456789abcdef" + if type(value) is not str or len(value) != 64 or any(char not in digits for char in value): + raise BudgetError("Expected SHA-256 digest") + + +def _amount(value: int, *, positive: bool = False) -> None: + if type(value) is not int or value < int(positive): + raise BudgetError("Expected nonnegative integer cost") + + +def attempt_plan_digest(round_digest: str, lane_id: str, attempt_ordinal: int) -> str: + """One immutable plan identity per attempt. The round digest stays unchanged.""" + _digest(round_digest) + _text(lane_id) + _amount(attempt_ordinal, positive=True) + payload = f"{round_digest}\n{lane_id}\n{attempt_ordinal}".encode() + return hashlib.sha256(payload).hexdigest() + + +@dataclass(frozen=True) +class LaneKey: + round_digest: str + lane_id: str + + def __post_init__(self) -> None: + _digest(self.round_digest) + _text(self.lane_id) + + +@dataclass(frozen=True) +class Genesis: + evidence: str + legacy_charge: int + + def __post_init__(self) -> None: + _text(self.evidence) + _amount(self.legacy_charge) + + +@dataclass(frozen=True) +class Binding: + plan_digest: str + run_id: str + worst_case: int + + def __post_init__(self) -> None: + _digest(self.plan_digest) + _text(self.run_id) + _amount(self.worst_case, positive=True) + + +@dataclass(frozen=True) +class Entry: + binding: Binding + phase: str = "reserved" + actual_cost: int | None = None + + def __post_init__(self) -> None: + if type(self.binding) is not Binding or self.phase not in { + "reserved", "claimed", "finished", "cancelled" + }: + raise BudgetError("Invalid reservation") + if self.phase == "finished": + _amount(self.actual_cost) + if self.actual_cost > self.binding.worst_case: + raise BudgetError("Actual cost exceeds reservation") + elif self.actual_cost is not None: + raise BudgetError("Only finished reservations have actual cost") + + +@dataclass(frozen=True) +class Ledger: + key: LaneKey + genesis: Genesis + cap: int + revision: int = 0 + entries: tuple[Entry, ...] = () + + def __post_init__(self) -> None: + if type(self.key) is not LaneKey or type(self.genesis) is not Genesis: + raise BudgetError("Missing ledger identity") + _amount(self.cap, positive=True) + _amount(self.revision) + entries_ok = type(self.entries) is tuple and all( + type(entry) is Entry for entry in self.entries + ) + if not entries_ok: + raise BudgetError("Invalid ledger entries") + expected = sum( + {"reserved": 1, "claimed": 2, "finished": 3, "cancelled": 2}[entry.phase] + for entry in self.entries + ) + if self.revision != expected: + raise BudgetError("Inconsistent transition revision") + plans = [entry.binding.plan_digest for entry in self.entries] + runs = [entry.binding.run_id for entry in self.entries] + if len(set(plans)) != len(plans) or len(set(runs)) != len(runs): + raise BudgetError("Duplicate plan or run binding") + if self.charged > self.cap: + raise BudgetError("lane cumulative budget exhausted") + + @property + def charged(self) -> int: + return self.genesis.legacy_charge + sum( + entry.binding.worst_case for entry in self.entries + ) + + +def reserve(ledger: Ledger, binding: Binding) -> Ledger: + """Idempotent for the exact binding only. Does not authorize execution.""" + if type(ledger) is not Ledger or type(binding) is not Binding: + raise BudgetError("Invalid ledger or binding") + for entry in ledger.entries: + if entry.binding == binding: + return ledger + same_plan = entry.binding.plan_digest == binding.plan_digest + same_run = entry.binding.run_id == binding.run_id + if same_plan or same_run: + raise BudgetError("Plan or run already bound differently") + return replace(ledger, revision=ledger.revision + 1, entries=ledger.entries + (Entry(binding),)) + + +def advance( + ledger: Ledger, binding: Binding, phase: str, *, actual_cost: int | None = None +) -> Ledger: + """Claim is not idempotent. Cancelling a reservation does not free capacity.""" + if type(ledger) is not Ledger or type(binding) is not Binding: + raise BudgetError("Invalid ledger or binding") + for index, entry in enumerate(ledger.entries): + if entry.binding != binding: + continue + updated = Entry(binding, phase, actual_cost) + if updated == entry and phase in {"finished", "cancelled"}: + return ledger + allowed = {"reserved": {"claimed", "cancelled"}, "claimed": {"finished"}} + if phase not in allowed.get(entry.phase, set()): + raise BudgetError("Unsafe or duplicate execution transition") + entries = ledger.entries[:index] + (updated,) + ledger.entries[index + 1 :] + return replace(ledger, revision=ledger.revision + 1, entries=entries) + raise BudgetError("No exact reservation for this plan/run") + + +def ensure_open(deadline_utc: str, now: datetime) -> None: + """Refuse a new attempt after the lane deadline. Reservation does not check time.""" + if type(deadline_utc) is not str: + raise BudgetError("deadline must be a timezone-aware timestamp") + try: + deadline = datetime.fromisoformat(deadline_utc) + except ValueError as exc: + raise BudgetError("deadline must be a timezone-aware timestamp") from exc + if deadline.tzinfo is None or now.tzinfo is None: + raise BudgetError("deadline and now must be timezone-aware") + if now > deadline: + raise BudgetError("lane deadline passed") + + +def empty_ledger(round_digest: str, lane_id: str, cap: int) -> Ledger: + return Ledger( + LaneKey(round_digest, lane_id), + Genesis("frozen-round-plan", 0), + cap, + ) + + +def admit_attempt( + plan: dict, ledger: Ledger, lane_id: str, now: datetime +) -> tuple[Ledger, Binding]: + """Read the lane contract, enforce the deadline, then reserve the next attempt.""" + lanes = [lane for lane in plan.get("lanes", []) if lane.get("id") == lane_id] + if len(lanes) != 1: + raise BudgetError("lane is missing from the round plan") + lane = lanes[0] + ensure_open(lane["deadline_utc"], now) + ordinal = len(ledger.entries) + 1 + binding = Binding( + attempt_plan_digest(plan["round_digest"], lane_id, ordinal), + f"{lane_id}#{ordinal}", + lane["per_attempt_worst_case"], + ) + return reserve(ledger, binding), binding + + +def admit_from_path( + plan_path: Path, ledger: Ledger, lane_id: str, now: datetime +) -> tuple[Ledger, Binding]: + """Re-read the frozen plan from disk so a restarted coordinator cannot skip expiry.""" + plan = json.loads(Path(plan_path).read_text(encoding="utf-8")) + return admit_attempt(plan, ledger, lane_id, now) diff --git a/src/inferrsi/infra/confinement.py b/src/inferrsi/infra/confinement.py new file mode 100644 index 000000000..af78261f7 --- /dev/null +++ b/src/inferrsi/infra/confinement.py @@ -0,0 +1,206 @@ +"""Filesystem confinement for candidate build and trial commands. + +``execute_command`` is unchanged. The default backend uses a user and mount +namespace when the kernel allows it, and refuses to run otherwise. It does not +fall back to an unconfined subprocess. ``FixtureBackend`` only refuses argv +paths outside the writable set; it is a test double, not an OS boundary. +""" + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + + +class ConfinementUnavailable(RuntimeError): + """The command was not executed because confinement could not be enforced.""" + + +def namespaces_available() -> bool: + try: + limit = int(Path("/proc/sys/user/max_user_namespaces").read_text(encoding="utf-8")) + except (OSError, ValueError): + return False + return limit > 0 + + +def _within(path: Path, roots: tuple[Path, ...]) -> bool: + resolved = path.resolve() + return any(resolved == root or root in resolved.parents for root in roots) + + +def _outside_writable(argv: tuple[str, ...], writable: tuple[Path, ...]) -> Path | None: + for arg in argv[1:]: + if not arg.startswith("/"): + continue + path = Path(arg) + if not _within(path, writable): + return path + return None + + +def _run_local(argv: tuple[str, ...], cwd: Path, deadline: float) -> dict: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise ConfinementUnavailable("execution deadline exhausted before start") + try: + completed = subprocess.run( + argv, + cwd=cwd, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=remaining, + check=False, + ) + except subprocess.TimeoutExpired as exc: + return { + "status": "timeout", + "returncode": None, + "stdout": exc.stdout or "", + "stderr": exc.stderr or "", + } + status = "completed" if completed.returncode == 0 else "failed" + return { + "status": status, + "returncode": completed.returncode, + "stdout": completed.stdout, + "stderr": completed.stderr, + } + + +class UnavailableBackend: + """Fail closed. Does not execute the candidate.""" + + name = "unavailable" + + def run( + self, + argv: tuple[str, ...], + cwd: Path, + writable: tuple[Path, ...], + readable: tuple[Path, ...], + deadline: float, + ) -> dict: + del argv, cwd, writable, readable, deadline + raise ConfinementUnavailable("confinement backend unavailable") + + +class FixtureBackend: + """Refuse argv paths outside ``writable``. Not mount-namespace confinement.""" + + name = "fixture" + + def run( + self, + argv: tuple[str, ...], + cwd: Path, + writable: tuple[Path, ...], + readable: tuple[Path, ...], + deadline: float, + ) -> dict: + del readable + outside = _outside_writable(argv, writable) + if outside is not None: + raise ConfinementUnavailable(f"path outside writable set: {outside}") + return _run_local(argv, cwd, deadline) + + +class UnshareBackend: + """Run argv inside a user and mount namespace. Fail closed if unshare fails.""" + + name = "unshare" + + def run( + self, + argv: tuple[str, ...], + cwd: Path, + writable: tuple[Path, ...], + readable: tuple[Path, ...], + deadline: float, + ) -> dict: + del readable + if not writable: + raise ConfinementUnavailable("refusing to run with an empty writable set") + config = writable[0] / ".inferrsi-confine.json" + payload = { + "argv": list(argv), + "writable": [str(path) for path in writable], + "env": dict(os.environ), + } + config.write_text(json.dumps(payload), encoding="utf-8") + remaining = deadline - time.monotonic() + if remaining <= 0: + raise ConfinementUnavailable("execution deadline exhausted before start") + completed = subprocess.run( + [ + "unshare", + "--user", + "--map-root-user", + "--mount", + "--propagation", + "private", + "--", + sys.executable, + "-c", + "import sys; from inferrsi.infra.confinement import enter_namespace; " + "enter_namespace(sys.argv[1])", + str(config), + ], + cwd=cwd, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=remaining, + check=False, + ) + if completed.returncode != 0: + raise ConfinementUnavailable(completed.stderr.strip() or "unshare confinement failed") + return { + "status": "completed", + "returncode": 0, + "stdout": completed.stdout, + "stderr": completed.stderr, + } + + +def enter_namespace(config_path: str) -> None: + """Remount read-only, restore writable paths, and exit before exec on failure.""" + config = json.loads(Path(config_path).read_text(encoding="utf-8")) + + def mount(args: list[str]) -> None: + completed = subprocess.run(["mount", *args], capture_output=True, text=True, check=False) + if completed.returncode != 0: + raise SystemExit(completed.stderr.strip() or "mount failed") + + mount(["--make-rprivate", "/"]) + mount(["-o", "remount,bind,ro", "/"]) + for path in config["writable"]: + mount(["-o", "bind", path, path]) + mount(["-o", "remount,bind,rw", path]) + argv = config["argv"] + os.execvpe(argv[0], argv, config["env"]) + + +def select_backend(): + if namespaces_available(): + return UnshareBackend() + return UnavailableBackend() + + +def confined_execute( + argv: tuple[str, ...], + cwd: Path, + writable_paths: tuple[Path, ...], + readable_paths: tuple[Path, ...], + deadline: float, + *, + backend=None, +) -> dict: + """Execute argv only inside the selected boundary. Never falls back to a plain subprocess.""" + chosen = select_backend() if backend is None else backend + writable = tuple(Path(path).resolve() for path in writable_paths) + readable = tuple(Path(path).resolve() for path in readable_paths) + return chosen.run(tuple(argv), Path(cwd), writable, readable, deadline) diff --git a/src/inferrsi/infra/device_lease.py b/src/inferrsi/infra/device_lease.py new file mode 100644 index 000000000..7c006f042 --- /dev/null +++ b/src/inferrsi/infra/device_lease.py @@ -0,0 +1,40 @@ +"""Host-local device leases. The key is the caller's device identity, not a CUDA probe.""" + +import fcntl +from pathlib import Path + + +class LeaseError(RuntimeError): + """The device identity is already held.""" + + +class _Held: + def __init__(self, handle) -> None: + self._handle = handle + + def release(self) -> None: + fcntl.flock(self._handle.fileno(), fcntl.LOCK_UN) + self._handle.close() + + def __enter__(self): + return self + + def __exit__(self, *exc) -> None: + self.release() + + +class DeviceLease: + def __init__(self, directory: Path) -> None: + self.directory = Path(directory) + self.directory.mkdir(parents=True, exist_ok=True) + + def acquire(self, device_id: str) -> _Held: + if not device_id or any(char in device_id for char in "/\\"): + raise LeaseError("device identity must be a single path segment") + handle = (self.directory / device_id).open("a+") + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + handle.close() + raise LeaseError(f"device busy: {device_id}") from None + return _Held(handle) diff --git a/tests/test_confinement.py b/tests/test_confinement.py new file mode 100644 index 000000000..97e465bdc --- /dev/null +++ b/tests/test_confinement.py @@ -0,0 +1,85 @@ +"""Confinement policy. The fixture backend is not an operating-system boundary.""" + +import sys +import time + +import pytest + +from inferrsi.infra.confinement import ( + ConfinementUnavailable, + FixtureBackend, + UnavailableBackend, + UnshareBackend, + confined_execute, + select_backend, +) + + +def test_build_write_outside_the_checkout_is_denied_and_does_not_run(tmp_path): + checkout = tmp_path / "candidate" + checkout.mkdir() + ledger = tmp_path / "ledger.json" + ledger.write_text("safe", encoding="utf-8") + argv = ( + sys.executable, + "-c", + "open(__import__('sys').argv[1], 'w').write('pwned')", + str(ledger), + ) + with pytest.raises(ConfinementUnavailable, match="outside writable"): + confined_execute( + argv, + checkout, + (checkout,), + (), + time.monotonic() + 5, + backend=FixtureBackend(), + ) + assert ledger.read_text(encoding="utf-8") == "safe" + + +def test_missing_backend_refuses_without_running(tmp_path): + marker = tmp_path / "ran" + argv = (sys.executable, "-c", "open('ran','w').write('yes')") + with pytest.raises(ConfinementUnavailable, match="unavailable"): + confined_execute( + argv, + tmp_path, + (tmp_path,), + (), + time.monotonic() + 5, + backend=UnavailableBackend(), + ) + assert not marker.exists() + + +def test_select_backend_fails_closed_when_namespaces_are_disabled(monkeypatch): + monkeypatch.setattr("inferrsi.infra.confinement.namespaces_available", lambda: False) + assert isinstance(select_backend(), UnavailableBackend) + + +def test_unshare_failure_does_not_fall_back(tmp_path, monkeypatch): + calls = [] + + def fake_run(argv, **kwargs): + del kwargs + calls.append(argv) + + class Completed: + returncode = 1 + stdout = "" + stderr = "unshare failed" + + return Completed() + + monkeypatch.setattr("inferrsi.infra.confinement.subprocess.run", fake_run) + with pytest.raises(ConfinementUnavailable, match="unshare failed"): + UnshareBackend().run( + (sys.executable, "-c", "print('should-not-run')"), + tmp_path, + (tmp_path,), + (), + time.monotonic() + 5, + ) + assert calls and calls[0][0] == "unshare" + assert "should-not-run" not in calls[0] diff --git a/tests/test_parallel_round.py b/tests/test_parallel_round.py new file mode 100644 index 000000000..57c3f98b5 --- /dev/null +++ b/tests/test_parallel_round.py @@ -0,0 +1,196 @@ +"""Synthetic parallel round. Latency figures here are fixture inputs, not measurements.""" + +import json +import sys +import threading +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from inferrsi.harness.parallel_round import ( + RoundPlanError, + prepare, + serve_ticket, + submit, + ticket_status, +) +from inferrsi.infra.confinement import FixtureBackend +from inferrsi.infra.device_lease import DeviceLease, LeaseError + +WHEN = datetime(2026, 10, 30, tzinfo=timezone.utc) +BENCH = ( + "import json, sys\n" + "from pathlib import Path\n" + "value = float(Path('knobs/a.txt').read_text())\n" + "body = {'metrics': {'latency_ms': value}, 'synthetic': True}\n" + "Path(sys.argv[1]).write_text(json.dumps(body))\n" +) + + +def _tree(root: Path, knob: str) -> None: + (root / "knobs").mkdir(parents=True) + (root / "knobs" / "a.txt").write_text(knob, encoding="utf-8") + (root / "knobs" / "b.txt").write_text("10", encoding="utf-8") + (root / "bench.py").write_text(BENCH, encoding="utf-8") + + +def _spec(**overrides): + lanes = overrides.pop( + "lanes", + [ + { + "id": "L1", + "entry": "D-01", + "plane": "infra", + "globs": ["knobs/a.txt"], + "cap": 2, + "per_attempt_worst_case": 1, + "deadline_utc": "2099-01-01T00:00:00+00:00", + }, + { + "id": "L2", + "entry": "D-02", + "plane": "harness", + "globs": ["knobs/b.txt"], + "cap": 2, + "per_attempt_worst_case": 1, + "deadline_utc": "2099-01-01T00:00:00+00:00", + }, + ], + ) + spec = { + "round_id": "r1", + "device_id": "cpu-0", + "incumbent_id": "fixture-incumbent", + "protected": ["bench.py"], + "build": [sys.executable, "-c", "open('build-stamp','w').write('ok')"], + "trial": [sys.executable, "bench.py", "{result}"], + "lanes": lanes, + } + spec.update(overrides) + return spec + + +def test_prepare_rejects_overlapping_globs_protected_paths_and_shared_entries(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + overlap = _spec( + lanes=[ + { + "id": "L1", + "entry": "D-01", + "plane": "infra", + "globs": ["knobs/*.txt"], + "cap": 1, + "per_attempt_worst_case": 1, + "deadline_utc": "2099-01-01T00:00:00+00:00", + }, + { + "id": "L2", + "entry": "D-02", + "plane": "harness", + "globs": ["knobs/a.txt"], + "cap": 1, + "per_attempt_worst_case": 1, + "deadline_utc": "2099-01-01T00:00:00+00:00", + }, + ] + ) + with pytest.raises(RoundPlanError, match="overlaps"): + prepare(overlap, incumbent, tmp_path / "round-overlap") + protected = _spec( + lanes=[ + { + "id": "L1", + "entry": "D-01", + "plane": "infra", + "globs": ["bench.py"], + "cap": 1, + "per_attempt_worst_case": 1, + "deadline_utc": "2099-01-01T00:00:00+00:00", + } + ] + ) + with pytest.raises(RoundPlanError, match="protected"): + prepare(protected, incumbent, tmp_path / "round-protected") + shared = _spec() + shared["lanes"][1]["entry"] = "D-01" + with pytest.raises(RoundPlanError, match="pursue"): + prepare(shared, incumbent, tmp_path / "round-shared") + many = _spec() + many["lanes"][0]["plane"] = ["infra", "harness"] + with pytest.raises(RoundPlanError, match="exactly one plane"): + prepare(many, incumbent, tmp_path / "round-plane") + + +def test_prepare_records_plan_cap_and_deadline(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + plan = prepare(_spec(), incumbent, tmp_path / "round") + written = json.loads((tmp_path / "round" / "round-plan.json").read_text(encoding="utf-8")) + assert written["lanes"][0]["cap"] == 2 + assert written["lanes"][0]["deadline_utc"].startswith("2099-") + assert written["round_digest"] == plan["round_digest"] + assert len(written["round_digest"]) == 64 + + +def test_submit_rejects_paths_outside_the_lane_and_is_idempotent(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare(_spec(), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + (work / "README").write_text("extra", encoding="utf-8") + rejected = submit(round_dir, "L1", incumbent, work) + assert rejected["submitted"] is False + assert rejected["outside"] == ["README"] + assert not (round_dir / "outbox").exists() + (work / "README").unlink() + first = submit(round_dir, "L1", incumbent, work) + second = submit(round_dir, "L1", incumbent, work) + assert first["submitted"] is True + assert first["ticket"] == second["ticket"] + + +def test_serve_records_a_synthetic_gain_and_a_held_device_stays_queued(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare(_spec(), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work)["ticket"] + held = DeviceLease(round_dir / "leases").acquire("cpu-0") + queued = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + assert queued["status"] == "queued" + assert ticket_status(round_dir, ticket)["status"] == "queued" + assert not (round_dir / "ledgers").exists() + held.release() + measured = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + assert measured["synthetic"] is True + assert measured["candidate_latency_ms"] == 8 + assert measured["incumbent_latency_ms"] == 10 + assert measured["gain_ms"] == 2 + assert measured["confinement"] == "fixture" + assert (round_dir / "candidates" / ticket / "build-stamp").read_text(encoding="utf-8") == "ok" + + +def test_two_acquires_do_not_share_one_device(tmp_path): + lease = DeviceLease(tmp_path) + held = lease.acquire("cpu-0") + failed = threading.Event() + + def other() -> None: + try: + lease.acquire("cpu-0") + except LeaseError: + failed.set() + + thread = threading.Thread(target=other) + thread.start() + thread.join(2) + assert failed.is_set() + held.release() + lease.acquire("cpu-1").release() diff --git a/tests/test_round_budget.py b/tests/test_round_budget.py new file mode 100644 index 000000000..4f08cb32f --- /dev/null +++ b/tests/test_round_budget.py @@ -0,0 +1,68 @@ +"""Lane attempt budgets. Costs in these tests are plan fields, not campaign gates.""" + +import json +from datetime import datetime, timezone + +import pytest + +from inferrsi.harness.round_budget import ( + BudgetError, + admit_from_path, + advance, + attempt_plan_digest, + empty_ledger, +) + +ROUND = "a" * 64 +WHEN = datetime(2026, 10, 30, tzinfo=timezone.utc) + + +def _plan(path, *, cap=2, worst=1, deadline="2099-01-01T00:00:00+00:00"): + document = { + "round_digest": ROUND, + "lanes": [ + { + "id": "L1", + "cap": cap, + "per_attempt_worst_case": worst, + "deadline_utc": deadline, + } + ], + } + path.write_text(json.dumps(document), encoding="utf-8") + return document + + +def test_retry_with_budget_remaining_keeps_both_charges(tmp_path): + plan = _plan(tmp_path / "round-plan.json", cap=2, worst=1) + ledger = empty_ledger(ROUND, "L1", cap=2) + first, binding = admit_from_path(tmp_path / "round-plan.json", ledger, "L1", WHEN) + cancelled = advance(first, binding, "cancelled") + second, _ = admit_from_path(tmp_path / "round-plan.json", cancelled, "L1", WHEN) + assert second.charged == 2 + assert len(second.entries) == 2 + assert second.entries[0].phase == "cancelled" + assert second.entries[1].phase == "reserved" + assert attempt_plan_digest(ROUND, "L1", 1) != attempt_plan_digest(ROUND, "L1", 2) + assert plan["round_digest"] == ROUND + + +def test_exhausted_budget_rejects_the_next_attempt(tmp_path): + _plan(tmp_path / "round-plan.json", cap=1, worst=1) + ledger = empty_ledger(ROUND, "L1", cap=1) + reserved, binding = admit_from_path(tmp_path / "round-plan.json", ledger, "L1", WHEN) + cancelled = advance(reserved, binding, "cancelled") + with pytest.raises(BudgetError, match="exhausted"): + admit_from_path(tmp_path / "round-plan.json", cancelled, "L1", WHEN) + + +def test_past_deadline_rejects_even_when_budget_remains(tmp_path): + plan_path = tmp_path / "round-plan.json" + _plan(plan_path, cap=5, worst=1, deadline="2020-01-01T00:00:00+00:00") + ledger = empty_ledger(ROUND, "L1", cap=5) + before = ledger.charged + with pytest.raises(BudgetError, match="deadline"): + admit_from_path(plan_path, ledger, "L1", WHEN) + reloaded = json.loads(plan_path.read_text(encoding="utf-8")) + assert reloaded["lanes"][0]["deadline_utc"].startswith("2020-") + assert ledger.charged == before diff --git a/tools/parallel_round.py b/tools/parallel_round.py new file mode 100644 index 000000000..a4fd49ec0 --- /dev/null +++ b/tools/parallel_round.py @@ -0,0 +1,62 @@ +"""Offline coordinator for one parallel round. Synthetic output is not campaign evidence.""" + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +from inferrsi.harness.parallel_round import prepare, serve_ticket, submit, ticket_status +from inferrsi.infra.confinement import select_backend + + +def _load(path: str) -> dict: + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + + prepare_command = commands.add_parser("prepare") + prepare_command.add_argument("--spec", required=True) + prepare_command.add_argument("--incumbent", required=True, type=Path) + prepare_command.add_argument("--round-dir", required=True, type=Path) + + submit_command = commands.add_parser("submit") + submit_command.add_argument("--round-dir", required=True, type=Path) + submit_command.add_argument("--lane", required=True) + submit_command.add_argument("--incumbent", required=True, type=Path) + submit_command.add_argument("--work", required=True, type=Path) + + serve_command = commands.add_parser("serve") + serve_command.add_argument("--round-dir", required=True, type=Path) + serve_command.add_argument("--ticket", required=True) + serve_command.add_argument("--incumbent", required=True, type=Path) + + status_command = commands.add_parser("status") + status_command.add_argument("--round-dir", required=True, type=Path) + status_command.add_argument("--ticket", required=True) + + args = parser.parse_args(argv) + if args.command == "prepare": + result = prepare(_load(args.spec), args.incumbent, args.round_dir) + elif args.command == "submit": + result = submit(args.round_dir, args.lane, args.incumbent, args.work) + elif args.command == "serve": + result = serve_ticket( + args.round_dir, + args.ticket, + args.incumbent, + backend=select_backend(), + now=datetime.now(timezone.utc), + ) + else: + result = ticket_status(args.round_dir, args.ticket) + json.dump(result, sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 6b051bb44f0a3688371b7aff609c2ad36cea6c91 Mon Sep 17 00:00:00 2001 From: NumberWan Date: Mon, 2 Nov 2026 10:23:22 +0800 Subject: [PATCH 02/23] fix(harness): close the parallel-round execution boundary Refuse symlinks and in-scope deletions before any content read, persist a claim before the first candidate command, and bound every launch by the frozen deadline. Confinement now builds a fresh root and a minimal environment instead of remounting the host root. Refs #249 Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_round.py | 300 ++++++++++++++++++------- src/inferrsi/harness/round_budget.py | 29 ++- src/inferrsi/infra/confinement.py | 204 +++++++++++++---- tests/test_confinement.py | 51 ++++- tests/test_parallel_round.py | 123 +++++++++- tools/parallel_round.py | 8 +- 6 files changed, 577 insertions(+), 138 deletions(-) diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py index 8a37780ea..fb424e526 100644 --- a/src/inferrsi/harness/parallel_round.py +++ b/src/inferrsi/harness/parallel_round.py @@ -4,10 +4,13 @@ not implemented. A synthetic latency number is not a campaign verdict. """ +import hashlib import json +import os import shutil +import stat import time -from datetime import datetime +from datetime import datetime, timezone from fnmatch import fnmatch from pathlib import Path @@ -21,8 +24,10 @@ admit_attempt, advance, empty_ledger, + ensure_open, + seconds_remaining, ) -from inferrsi.infra.artifacts import digest, sha256, write_json +from inferrsi.infra.artifacts import digest, write_json from inferrsi.infra.confinement import ConfinementUnavailable, confined_execute from inferrsi.infra.device_lease import DeviceLease, LeaseError @@ -38,12 +43,101 @@ def _match(path: str, patterns: list[str]) -> bool: def _files(root: Path) -> list[str]: - found = [] - for path in sorted(root.rglob("*")): - if not path.is_file() or ".git" in path.parts: + hashes, _blocked = _snapshot(root) + return sorted(hashes) + + +def _snapshot(root: Path) -> tuple[dict[str, str], list[str]]: + """Hash regular files without following links. Symlinks are reported, not read.""" + hashes: dict[str, str] = {} + blocked: list[str] = [] + root_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY) + try: + _scan(root_fd, "", hashes, blocked) + finally: + os.close(root_fd) + return hashes, blocked + + +def _scan(dirfd: int, prefix: str, hashes: dict[str, str], blocked: list[str]) -> None: + for name in os.listdir(dirfd): + if name in {".", "..", ".git"}: continue - found.append(path.relative_to(root).as_posix()) - return found + relative = f"{prefix}/{name}" if prefix else name + try: + info = os.stat(name, dir_fd=dirfd, follow_symlinks=False) + except OSError: + blocked.append(relative) + continue + if stat.S_ISLNK(info.st_mode): + blocked.append(relative) + continue + if stat.S_ISDIR(info.st_mode): + try: + child = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=dirfd) + except OSError: + blocked.append(relative) + continue + try: + _scan(child, relative, hashes, blocked) + finally: + os.close(child) + continue + if not stat.S_ISREG(info.st_mode): + blocked.append(relative) + continue + try: + file_fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=dirfd) + except OSError: + blocked.append(relative) + continue + try: + hasher = hashlib.sha256() + while True: + block = os.read(file_fd, 1024 * 1024) + if not block: + break + hasher.update(block) + hashes[relative] = hasher.hexdigest() + finally: + os.close(file_fd) + + +def _read_nofollow(root: Path, relative: str) -> bytes | None: + """Read one regular file through no-follow parent components, or return None.""" + dirfd = os.open(root, os.O_RDONLY | os.O_DIRECTORY) + try: + parts = relative.split("/") + for part in parts[:-1]: + try: + child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=dirfd) + except OSError: + return None + os.close(dirfd) + dirfd = child + try: + file_fd = os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW, dir_fd=dirfd) + except OSError: + return None + finally: + os.close(dirfd) + try: + chunks = [] + while True: + block = os.read(file_fd, 1024 * 1024) + if not block: + break + chunks.append(block) + return b"".join(chunks) + finally: + os.close(file_fd) + + +def _refusal(reason: str, **fields) -> dict: + body = {"submitted": False, "outside": [], "deleted": [], "rejected": [], "ticket": None} + body.update(fields) + body["reason"] = reason + return body def _lane(plan: dict, lane_id: str) -> dict: @@ -53,15 +147,6 @@ def _lane(plan: dict, lane_id: str) -> dict: return matches[0] -def _hashes(root: Path) -> dict[str, str]: - return {relative: sha256(root / relative) for relative in _files(root)} - - -def _changed(incumbent: Path, work: Path) -> list[str]: - before, after = _hashes(incumbent), _hashes(work) - return sorted(path for path in set(before) | set(after) if before.get(path) != after.get(path)) - - def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: """Validate lane scopes and write ``round-plan.json``. The digest covers the validated body.""" if not isinstance(spec, dict): @@ -125,30 +210,59 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: return body -def submit(round_dir: Path, lane_id: str, incumbent: Path, work: Path) -> dict: +def submit( + round_dir: Path, lane_id: str, incumbent: Path, work: Path, *, now: datetime | None = None +) -> dict: """Copy in-glob changes to the outbox. The same manifest always returns the same ticket.""" + moment = now or datetime.now(timezone.utc) plan = json.loads((round_dir / "round-plan.json").read_text(encoding="utf-8")) lane = _lane(plan, lane_id) - changed = _changed(incumbent, work) + try: + ensure_open(lane["deadline_utc"], moment) + except BudgetError as exc: + return _refusal(str(exc)) + before, _incumbent_blocked = _snapshot(incumbent) + after, blocked = _snapshot(work) + changed = sorted( + path for path in set(before) | set(after) if before.get(path) != after.get(path) + ) + known = set(before) | set(after) | set(changed) + linked = [ + path + for path in blocked + if any(item == path or item.startswith(path + "/") for item in known) + ] + if linked: + return _refusal("symlink", rejected=sorted(linked)) + deleted = [path for path in changed if path not in after and _match(path, lane["globs"])] + if deleted: + return _refusal("deletion", deleted=deleted) outside = [path for path in changed if not _match(path, lane["globs"])] if outside or not changed: - return {"submitted": False, "outside": outside, "ticket": None} - manifest = {path: sha256(work / path) for path in changed} + return _refusal("unchanged" if not changed else "outside", outside=outside) + manifest = {path: after[path] for path in changed} ticket = digest( {"round_digest": plan["round_digest"], "lane_id": lane_id, "manifest": manifest} )[:16] destination = round_dir / "outbox" / ticket if not destination.exists(): destination.mkdir(parents=True) - for path in changed: - target = destination / path - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(work / path, target) + try: + for path in changed: + payload = _read_nofollow(work, path) + if payload is None: + raise OSError(path) + target = destination / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(payload) + except OSError: + shutil.rmtree(destination) + return _refusal("symlink", rejected=list(changed)) write_json( destination / "manifest.json", - {"ticket": ticket, "lane_id": lane_id, "files": manifest}, + {"ticket": ticket, "lane_id": lane_id, "files": manifest, "deleted": []}, ) - return {"submitted": True, "outside": [], "ticket": ticket} + return {"submitted": True, "outside": [], "deleted": [], "rejected": [], "ticket": ticket} def _ledger_path(round_dir: Path, lane_id: str) -> Path: @@ -216,15 +330,72 @@ def _command(template: list[str], result: Path) -> tuple[str, ...]: return tuple(part.replace("{result}", str(result)) for part in template) -def serve_ticket(round_dir: Path, ticket: str, incumbent: Path, *, backend, now: datetime) -> dict: +def _record(round_dir: Path, ticket: str, **fields) -> dict: + body = {"ticket": ticket, "synthetic": True, **fields} + write_json(round_dir / "results" / f"{ticket}.json", body) + return body + + +def _claimed_entry(ledger, ticket: str): + suffix = f"@{ticket}" + for entry in ledger.entries: + if entry.binding.run_id.endswith(suffix) and entry.phase == "claimed": + return entry + return None + + +def _launch( + plan: dict, lane: dict, candidate: Path, incumbent: Path, trial_dir: Path, *, backend, current +): + """Run build and both trials. Each start is bounded by the same absolute deadline.""" + build_dir = candidate / "build-output" + build_dir.mkdir(parents=True, exist_ok=True) + trial_dir.mkdir(parents=True, exist_ok=True) + + def run(argv, cwd, writable, readable) -> None: + remaining = min(seconds_remaining(lane["deadline_utc"], current()), 30) + confined_execute( + argv, cwd, writable, readable, time.monotonic() + remaining, backend=backend + ) + + if plan["build"]: + run(tuple(plan["build"]), candidate, (candidate, build_dir), ()) + candidate_result = trial_dir / "candidate.json" + run(_command(plan["trial"], candidate_result), candidate, (trial_dir,), (candidate,)) + incumbent_result = trial_dir / "incumbent.json" + run(_command(plan["trial"], incumbent_result), incumbent, (trial_dir,), (incumbent,)) + return candidate_result, incumbent_result + + +def serve_ticket( + round_dir: Path, + ticket: str, + incumbent: Path, + *, + backend, + now: datetime, + clock=None, +) -> dict: """Measure one ticket. A busy device leaves it queued and does not reserve an attempt.""" + def current() -> datetime: + return clock() if clock is not None else now + plan = json.loads((round_dir / "round-plan.json").read_text(encoding="utf-8")) result_path = round_dir / "results" / f"{ticket}.json" if result_path.exists(): return json.loads(result_path.read_text(encoding="utf-8")) overlay = round_dir / "outbox" / ticket manifest = json.loads((overlay / "manifest.json").read_text(encoding="utf-8")) + if manifest.get("deleted"): + return _record(round_dir, ticket, status="rejected", reason="deletion") lane = _lane(plan, manifest["lane_id"]) + if _claimed_entry(_load_ledger(round_dir, plan, lane), ticket) is not None: + return _record( + round_dir, + ticket, + status="unsettled", + reason="claimed execution was not settled", + ) try: held = DeviceLease(round_dir / "leases").acquire(plan["device_id"]) except LeaseError: @@ -232,68 +403,41 @@ def serve_ticket(round_dir: Path, ticket: str, incumbent: Path, *, backend, now: try: try: loaded = _load_ledger(round_dir, plan, lane) - ledger, binding = admit_attempt(plan, loaded, lane["id"], now) + ledger, binding = admit_attempt(plan, loaded, lane["id"], current(), token=ticket) except BudgetError as exc: - record = {"ticket": ticket, "status": "rejected", "reason": str(exc), "synthetic": True} - write_json(result_path, record) - return record + return _record(round_dir, ticket, status="rejected", reason=str(exc)) _save_ledger(round_dir, ledger) candidate = round_dir / "candidates" / ticket trial_dir = round_dir / "trials" / ticket - build_dir = candidate / "build-output" + _materialize(incumbent, overlay, candidate) try: - _materialize(incumbent, overlay, candidate) - build_dir.mkdir(parents=True, exist_ok=True) - trial_dir.mkdir(parents=True, exist_ok=True) - if plan["build"]: - confined_execute( - tuple(plan["build"]), - candidate, - (candidate, build_dir), - (), - time.monotonic() + 30, - backend=backend, - ) - candidate_result = trial_dir / "candidate.json" - confined_execute( - _command(plan["trial"], candidate_result), - candidate, - (trial_dir,), - (candidate,), - time.monotonic() + 30, - backend=backend, - ) - incumbent_result = trial_dir / "incumbent.json" - confined_execute( - _command(plan["trial"], incumbent_result), - incumbent, - (trial_dir,), - (incumbent,), - time.monotonic() + 30, - backend=backend, - ) - except ConfinementUnavailable as exc: + seconds_remaining(lane["deadline_utc"], current()) + except BudgetError as exc: _save_ledger(round_dir, advance(ledger, binding, "cancelled")) - record = {"ticket": ticket, "status": "rejected", "reason": str(exc), "synthetic": True} - write_json(result_path, record) - return record + return _record(round_dir, ticket, status="rejected", reason=str(exc)) + ledger = advance(ledger, binding, "claimed") + _save_ledger(round_dir, ledger) + try: + candidate_result, incumbent_result = _launch( + plan, lane, candidate, incumbent, trial_dir, backend=backend, current=current + ) + except (ConfinementUnavailable, BudgetError) as exc: + return _record(round_dir, ticket, status="unsettled", reason=str(exc)) candidate_body = json.loads(candidate_result.read_text(encoding="utf-8")) incumbent_body = json.loads(incumbent_result.read_text(encoding="utf-8")) candidate_metric = candidate_body["metrics"]["latency_ms"] incumbent_metric = incumbent_body["metrics"]["latency_ms"] - finished = advance(advance(ledger, binding, "claimed"), binding, "finished", actual_cost=0) + finished = advance(ledger, binding, "finished", actual_cost=0) _save_ledger(round_dir, finished) - record = { - "ticket": ticket, - "status": "measured", - "synthetic": True, - "candidate_latency_ms": candidate_metric, - "incumbent_latency_ms": incumbent_metric, - "gain_ms": incumbent_metric - candidate_metric, - "confinement": getattr(backend, "name", "unknown"), - } - write_json(result_path, record) - return record + return _record( + round_dir, + ticket, + status="measured", + candidate_latency_ms=candidate_metric, + incumbent_latency_ms=incumbent_metric, + gain_ms=incumbent_metric - candidate_metric, + confinement=getattr(backend, "name", "unknown"), + ) finally: held.release() diff --git a/src/inferrsi/harness/round_budget.py b/src/inferrsi/harness/round_budget.py index 64cbee56c..ba4e921bf 100644 --- a/src/inferrsi/harness/round_budget.py +++ b/src/inferrsi/harness/round_budget.py @@ -167,18 +167,30 @@ def advance( raise BudgetError("No exact reservation for this plan/run") -def ensure_open(deadline_utc: str, now: datetime) -> None: - """Refuse a new attempt after the lane deadline. Reservation does not check time.""" - if type(deadline_utc) is not str: - raise BudgetError("deadline must be a timezone-aware timestamp") +def _deadline(deadline_utc: str, now: datetime) -> datetime: + if type(deadline_utc) is not str or now.tzinfo is None: + raise BudgetError("deadline and now must be timezone-aware") try: deadline = datetime.fromisoformat(deadline_utc) except ValueError as exc: raise BudgetError("deadline must be a timezone-aware timestamp") from exc - if deadline.tzinfo is None or now.tzinfo is None: + if deadline.tzinfo is None: raise BudgetError("deadline and now must be timezone-aware") - if now > deadline: + return deadline + + +def ensure_open(deadline_utc: str, now: datetime) -> None: + """Refuse a new attempt after the lane deadline. Reservation does not check time.""" + if now > _deadline(deadline_utc, now): + raise BudgetError("lane deadline passed") + + +def seconds_remaining(deadline_utc: str, now: datetime) -> float: + """Seconds left until the same absolute cutoff. Zero or less is already closed.""" + remaining = (_deadline(deadline_utc, now) - now).total_seconds() + if remaining <= 0: raise BudgetError("lane deadline passed") + return remaining def empty_ledger(round_digest: str, lane_id: str, cap: int) -> Ledger: @@ -190,7 +202,7 @@ def empty_ledger(round_digest: str, lane_id: str, cap: int) -> Ledger: def admit_attempt( - plan: dict, ledger: Ledger, lane_id: str, now: datetime + plan: dict, ledger: Ledger, lane_id: str, now: datetime, *, token: str | None = None ) -> tuple[Ledger, Binding]: """Read the lane contract, enforce the deadline, then reserve the next attempt.""" lanes = [lane for lane in plan.get("lanes", []) if lane.get("id") == lane_id] @@ -199,9 +211,10 @@ def admit_attempt( lane = lanes[0] ensure_open(lane["deadline_utc"], now) ordinal = len(ledger.entries) + 1 + run_id = f"{lane_id}#{ordinal}" if token is None else f"{lane_id}#{ordinal}@{token}" binding = Binding( attempt_plan_digest(plan["round_digest"], lane_id, ordinal), - f"{lane_id}#{ordinal}", + run_id, lane["per_attempt_worst_case"], ) return reserve(ledger, binding), binding diff --git a/src/inferrsi/infra/confinement.py b/src/inferrsi/infra/confinement.py index af78261f7..60f6d3d39 100644 --- a/src/inferrsi/infra/confinement.py +++ b/src/inferrsi/infra/confinement.py @@ -1,18 +1,60 @@ """Filesystem confinement for candidate build and trial commands. -``execute_command`` is unchanged. The default backend uses a user and mount -namespace when the kernel allows it, and refuses to run otherwise. It does not -fall back to an unconfined subprocess. ``FixtureBackend`` only refuses argv -paths outside the writable set; it is a test double, not an OS boundary. +``execute_command`` is unchanged. The namespace backend builds a fresh root +from the writable set, the readable set and the Python runtime. It does not +remount the host root, and it does not pass the coordinator environment +through. If that view cannot be built, execution is refused. ``FixtureBackend`` +is a test double: it enforces the same view with an import hook and a minimal +environment, which is not mount-namespace confinement. """ import json import os import subprocess import sys +import tempfile import time from pathlib import Path +_ENV_ALLOW = frozenset({"PATH", "LANG", "LC_ALL", "LC_CTYPE", "TZ", "PYTHONIOENCODING"}) +_GUARD = """ +import builtins +import os + +_roots = [item for item in os.environ.get("INFERRSI_VISIBLE_ROOTS", "").split(os.pathsep) if item] +_builtin_open = builtins.open +_os_open = os.open + + +def _allowed(file): + if not isinstance(file, (str, bytes, os.PathLike)): + return True + text = os.fsdecode(file) + if text.startswith("<"): + return True + try: + resolved = os.path.realpath(text) + except OSError: + return False + return any(resolved == root or resolved.startswith(root + os.sep) for root in _roots) + + +def _open(file, *args, **kwargs): + if not _allowed(file): + raise PermissionError(file) + return _builtin_open(file, *args, **kwargs) + + +def _open_fd(path, flags, mode=0o777, *, dir_fd=None): + if dir_fd is None and not _allowed(path): + raise PermissionError(path) + return _os_open(path, flags, mode, dir_fd=dir_fd) + + +builtins.open = _open +os.open = _open_fd +""" + class ConfinementUnavailable(RuntimeError): """The command was not executed because confinement could not be enforced.""" @@ -26,6 +68,42 @@ def namespaces_available() -> bool: return limit > 0 +def minimal_environment(extra: dict | None = None) -> dict: + """Coordinator credentials stay out. Only a fixed allowlist is copied.""" + environment = {key: os.environ[key] for key in _ENV_ALLOW if key in os.environ} + if extra: + environment.update(extra) + return environment + + +def runtime_roots() -> tuple[Path, ...]: + """Python prefix needed to start the interpreter. Never the host root.""" + candidates = [Path(sys.prefix).resolve(), Path(sys.executable).resolve().parent] + roots: list[Path] = [] + for candidate in candidates: + if any(candidate == earlier or earlier in candidate.parents for earlier in roots): + continue + roots.append(candidate) + return tuple(roots) + + +def namespace_plan( + writable: tuple[Path, ...], readable: tuple[Path, ...] +) -> dict: + """Mount list for a fresh root. The host root itself is not an entry.""" + mounts = [{"source": str(path), "mode": "rw"} for path in writable] + mounts.extend({"source": str(path), "mode": "ro"} for path in readable) + covered = list(writable) + list(readable) + for root in runtime_roots(): + if any(root == path or path in root.parents or root in path.parents for path in covered): + continue + mounts.append({"source": str(root), "mode": "ro"}) + covered.append(root) + if any(item["source"] == "/" for item in mounts): + raise ConfinementUnavailable("refusing a mount plan that exposes the host root") + return {"mounts": mounts, "env": minimal_environment()} + + def _within(path: Path, roots: tuple[Path, ...]) -> bool: resolved = path.resolve() return any(resolved == root or root in resolved.parents for root in roots) @@ -41,7 +119,7 @@ def _outside_writable(argv: tuple[str, ...], writable: tuple[Path, ...]) -> Path return None -def _run_local(argv: tuple[str, ...], cwd: Path, deadline: float) -> dict: +def _run_local(argv: tuple[str, ...], cwd: Path, deadline: float, environment: dict) -> dict: remaining = deadline - time.monotonic() if remaining <= 0: raise ConfinementUnavailable("execution deadline exhausted before start") @@ -49,6 +127,7 @@ def _run_local(argv: tuple[str, ...], cwd: Path, deadline: float) -> dict: completed = subprocess.run( argv, cwd=cwd, + env=environment, stdin=subprocess.DEVNULL, capture_output=True, text=True, @@ -89,7 +168,7 @@ def run( class FixtureBackend: - """Refuse argv paths outside ``writable``. Not mount-namespace confinement.""" + """Enforce the visible roots and minimal environment. Not a mount namespace.""" name = "fixture" @@ -101,15 +180,26 @@ def run( readable: tuple[Path, ...], deadline: float, ) -> dict: - del readable outside = _outside_writable(argv, writable) if outside is not None: raise ConfinementUnavailable(f"path outside writable set: {outside}") - return _run_local(argv, cwd, deadline) + visible = tuple(dict.fromkeys(writable + readable + runtime_roots())) + try: + guard_dir = tempfile.mkdtemp(prefix="inferrsi-guard-") + Path(guard_dir, "sitecustomize.py").write_text(_GUARD, encoding="utf-8") + except OSError as exc: + raise ConfinementUnavailable("confinement guard could not be installed") from exc + environment = minimal_environment( + { + "PYTHONPATH": guard_dir, + "INFERRSI_VISIBLE_ROOTS": os.pathsep.join(str(path) for path in visible), + } + ) + return _run_local(argv, cwd, deadline, environment) class UnshareBackend: - """Run argv inside a user and mount namespace. Fail closed if unshare fails.""" + """Run argv in a fresh root. Fail closed if the namespace cannot be built.""" name = "unshare" @@ -121,43 +211,55 @@ def run( readable: tuple[Path, ...], deadline: float, ) -> dict: - del readable if not writable: raise ConfinementUnavailable("refusing to run with an empty writable set") - config = writable[0] / ".inferrsi-confine.json" - payload = { - "argv": list(argv), - "writable": [str(path) for path in writable], - "env": dict(os.environ), - } - config.write_text(json.dumps(payload), encoding="utf-8") + plan = namespace_plan(writable, readable) remaining = deadline - time.monotonic() if remaining <= 0: raise ConfinementUnavailable("execution deadline exhausted before start") - completed = subprocess.run( - [ - "unshare", - "--user", - "--map-root-user", - "--mount", - "--propagation", - "private", - "--", - sys.executable, - "-c", - "import sys; from inferrsi.infra.confinement import enter_namespace; " - "enter_namespace(sys.argv[1])", - str(config), - ], - cwd=cwd, - stdin=subprocess.DEVNULL, - capture_output=True, - text=True, - timeout=remaining, - check=False, - ) - if completed.returncode != 0: - raise ConfinementUnavailable(completed.stderr.strip() or "unshare confinement failed") + config_path = None + completed = None + try: + handle = tempfile.NamedTemporaryFile( + prefix="inferrsi-confine-", suffix=".json", delete=False + ) + config_path = Path(handle.name) + handle.close() + config_path.write_text( + json.dumps({"argv": list(argv), "cwd": str(cwd), **plan}), + encoding="utf-8", + ) + completed = subprocess.run( + [ + "unshare", + "--user", + "--map-root-user", + "--mount", + "--propagation", + "private", + "--", + sys.executable, + "-c", + "import sys; from inferrsi.infra.confinement import enter_namespace; " + "enter_namespace(sys.argv[1])", + str(config_path), + ], + cwd=cwd, + env=minimal_environment(), + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=remaining, + check=False, + ) + finally: + if config_path is not None: + config_path.unlink(missing_ok=True) + if completed is None or completed.returncode != 0: + detail = "unshare confinement failed" + if completed is not None and completed.stderr.strip(): + detail = completed.stderr.strip() + raise ConfinementUnavailable(detail) return { "status": "completed", "returncode": 0, @@ -167,19 +269,27 @@ def run( def enter_namespace(config_path: str) -> None: - """Remount read-only, restore writable paths, and exit before exec on failure.""" + """Bind only the planned paths into a tmpfs root, then chroot. Exit before exec on failure.""" config = json.loads(Path(config_path).read_text(encoding="utf-8")) + jail = Path(tempfile.mkdtemp(prefix="inferrsi-jail-")) def mount(args: list[str]) -> None: completed = subprocess.run(["mount", *args], capture_output=True, text=True, check=False) if completed.returncode != 0: raise SystemExit(completed.stderr.strip() or "mount failed") - mount(["--make-rprivate", "/"]) - mount(["-o", "remount,bind,ro", "/"]) - for path in config["writable"]: - mount(["-o", "bind", path, path]) - mount(["-o", "remount,bind,rw", path]) + mount(["-t", "tmpfs", "tmpfs", str(jail)]) + for item in config["mounts"]: + source = item["source"] + if source == "/": + raise SystemExit("refusing to mount the host root") + target = jail / Path(source).resolve().relative_to("/") + target.mkdir(parents=True, exist_ok=True) + mount(["--bind", source, str(target)]) + if item["mode"] == "ro": + mount(["-o", "remount,bind,ro", str(target)]) + os.chroot(jail) + os.chdir(config["cwd"]) argv = config["argv"] os.execvpe(argv[0], argv, config["env"]) diff --git a/tests/test_confinement.py b/tests/test_confinement.py index 97e465bdc..feb479243 100644 --- a/tests/test_confinement.py +++ b/tests/test_confinement.py @@ -1,7 +1,9 @@ """Confinement policy. The fixture backend is not an operating-system boundary.""" +import json import sys import time +from pathlib import Path import pytest @@ -11,6 +13,7 @@ UnavailableBackend, UnshareBackend, confined_execute, + namespace_plan, select_backend, ) @@ -60,10 +63,11 @@ def test_select_backend_fails_closed_when_namespaces_are_disabled(monkeypatch): def test_unshare_failure_does_not_fall_back(tmp_path, monkeypatch): calls = [] + monkeypatch.setenv("SECRET_CANARY", "CANARY-ENV") def fake_run(argv, **kwargs): del kwargs - calls.append(argv) + calls.append(json.loads(Path(argv[-1]).read_text(encoding="utf-8"))) class Completed: returncode = 1 @@ -81,5 +85,46 @@ class Completed: (), time.monotonic() + 5, ) - assert calls and calls[0][0] == "unshare" - assert "should-not-run" not in calls[0] + assert calls + assert "SECRET_CANARY" not in calls[0]["env"] + assert "/" not in {item["source"] for item in calls[0]["mounts"]} + + +def test_namespace_plan_keeps_host_root_and_secrets_out(tmp_path, monkeypatch): + monkeypatch.setenv("SECRET_CANARY", "CANARY-ENV") + plan = namespace_plan((tmp_path / "writable",), (tmp_path / "readable",)) + assert "/" not in {item["source"] for item in plan["mounts"]} + assert "SECRET_CANARY" not in plan["env"] + assert str(tmp_path / "private") not in {item["source"] for item in plan["mounts"]} + + +def test_build_and_trial_cannot_see_private_canaries(tmp_path, monkeypatch): + monkeypatch.setenv("SECRET_CANARY", "CANARY-ENV") + private = tmp_path / "private" + private.mkdir() + (private / "canary.txt").write_text("CANARY-FILE", encoding="utf-8") + checkout = tmp_path / "checkout" + checkout.mkdir() + trial = tmp_path / "trial" + trial.mkdir() + script = ( + "import os\n" + "print(os.environ.get('SECRET_CANARY', ''))\n" + "try:\n" + " print(open(%r).read())\n" + "except PermissionError:\n" + " print('FILE-HIDDEN')\n" % str(private / "canary.txt") + ) + argv = (sys.executable, "-c", script) + backend = FixtureBackend() + stages = ( + confined_execute(argv, checkout, (checkout,), (), time.monotonic() + 10, backend=backend), + confined_execute( + argv, checkout, (trial,), (checkout,), time.monotonic() + 10, backend=backend + ), + ) + for record in stages: + assert record["status"] == "completed" + assert "CANARY-ENV" not in record["stdout"] + assert "CANARY-FILE" not in record["stdout"] + assert "FILE-HIDDEN" in record["stdout"] diff --git a/tests/test_parallel_round.py b/tests/test_parallel_round.py index 57c3f98b5..9ced6d585 100644 --- a/tests/test_parallel_round.py +++ b/tests/test_parallel_round.py @@ -15,7 +15,7 @@ submit, ticket_status, ) -from inferrsi.infra.confinement import FixtureBackend +from inferrsi.infra.confinement import ConfinementUnavailable, FixtureBackend from inferrsi.infra.device_lease import DeviceLease, LeaseError WHEN = datetime(2026, 10, 30, tzinfo=timezone.utc) @@ -194,3 +194,124 @@ def other() -> None: assert failed.is_set() held.release() lease.acquire("cpu-1").release() + + +def test_submit_rejects_symlink_without_reading_the_target(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare(_spec(), incumbent, round_dir) + canary = tmp_path / "canary.txt" + canary.write_text("CANARY-SECRET", encoding="utf-8") + work = tmp_path / "work" + _tree(work, "8") + (work / "knobs" / "a.txt").unlink() + (work / "knobs" / "a.txt").symlink_to(canary) + rejected = submit(round_dir, "L1", incumbent, work, now=WHEN) + assert rejected["submitted"] is False + assert rejected["reason"] == "symlink" + assert rejected["ticket"] is None + assert not (round_dir / "outbox").exists() + assert "CANARY-SECRET" not in json.dumps(rejected) + + +def test_in_scope_deletion_is_rejected_before_hashing_and_serve_does_not_run(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare(_spec(), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + (work / "knobs" / "a.txt").unlink() + rejected = submit(round_dir, "L1", incumbent, work, now=WHEN) + assert rejected == { + "submitted": False, + "outside": [], + "deleted": ["knobs/a.txt"], + "rejected": [], + "ticket": None, + "reason": "deletion", + } + ticket = "crafted-ticket" + overlay = round_dir / "outbox" / ticket + overlay.mkdir(parents=True) + (overlay / "manifest.json").write_text( + json.dumps({"ticket": ticket, "lane_id": "L1", "files": {}, "deleted": ["knobs/a.txt"]}), + encoding="utf-8", + ) + served = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + assert served["status"] == "rejected" + assert served["reason"] == "deletion" + assert not (round_dir / "candidates").exists() + + +def test_submit_after_deadline_is_refused_on_a_fresh_read(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + spec = _spec() + spec["lanes"][0]["deadline_utc"] = "2020-01-01T00:00:00+00:00" + round_dir = tmp_path / "round" + prepare(spec, incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + first = submit(round_dir, "L1", incumbent, work, now=WHEN) + second = submit(round_dir, "L1", incumbent, work, now=WHEN) + assert first["submitted"] is False and second["submitted"] is False + assert "deadline" in first["reason"] + assert not (round_dir / "outbox").exists() + + +def test_launch_after_deadline_cancels_before_claim(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare(_spec(), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + moments = [WHEN, datetime(2100, 1, 1, tzinfo=timezone.utc)] + + def clock(): + return moments.pop(0) if len(moments) > 1 else moments[0] + + rejected = serve_ticket( + round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN, clock=clock + ) + assert rejected["status"] == "rejected" + assert "deadline" in rejected["reason"] + ledger = json.loads((round_dir / "ledgers" / "L1.json").read_text(encoding="utf-8")) + assert ledger["entries"][-1]["phase"] == "cancelled" + assert not (round_dir / "candidates" / ticket / "build-stamp").exists() + + +def test_failure_after_claim_stays_unsettled_across_restart(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare(_spec(), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + + class StopAfterClaim: + name = "stop" + + def run(self, *args, **kwargs): + del args, kwargs + raise ConfinementUnavailable("stopped after claim") + + first = serve_ticket( + round_dir, ticket, incumbent, backend=StopAfterClaim(), now=WHEN + ) + assert first["status"] == "unsettled" + ledger_path = round_dir / "ledgers" / "L1.json" + ledger = json.loads(ledger_path.read_text(encoding="utf-8")) + assert [entry["phase"] for entry in ledger["entries"]] == ["claimed"] + (round_dir / "results" / f"{ticket}.json").unlink() + second = serve_ticket( + round_dir, ticket, incumbent, backend=StopAfterClaim(), now=WHEN + ) + assert second["status"] == "unsettled" + reloaded = json.loads(ledger_path.read_text(encoding="utf-8")) + assert [entry["phase"] for entry in reloaded["entries"]] == ["claimed"] + assert reloaded["entries"][0]["worst_case"] == ledger["entries"][0]["worst_case"] diff --git a/tools/parallel_round.py b/tools/parallel_round.py index a4fd49ec0..c093c8055 100644 --- a/tools/parallel_round.py +++ b/tools/parallel_round.py @@ -42,7 +42,13 @@ def main(argv: list[str] | None = None) -> int: if args.command == "prepare": result = prepare(_load(args.spec), args.incumbent, args.round_dir) elif args.command == "submit": - result = submit(args.round_dir, args.lane, args.incumbent, args.work) + result = submit( + args.round_dir, + args.lane, + args.incumbent, + args.work, + now=datetime.now(timezone.utc), + ) elif args.command == "serve": result = serve_ticket( args.round_dir, From 77da47bfeba91cbda0c29027e71bafcb567add3c Mon Sep 17 00:00:00 2001 From: NumberWan Date: Mon, 2 Nov 2026 15:11:05 +0800 Subject: [PATCH 03/23] fix(harness): keep parallel-round launches inside the deadline and the jail A namespace timeout now leaves an unsettled receipt, each launch reads a fresh clock, and the fresh root includes the dynamic loader and its libraries. Refs #249 Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_round.py | 36 ++++++-- src/inferrsi/infra/confinement.py | 112 ++++++++++++++++++++++--- tests/test_confinement.py | 69 +++++++++++++++ tests/test_parallel_round.py | 66 ++++++++++++++- tools/parallel_round.py | 1 - 5 files changed, 262 insertions(+), 22 deletions(-) diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py index fb424e526..d9a01553a 100644 --- a/src/inferrsi/harness/parallel_round.py +++ b/src/inferrsi/harness/parallel_round.py @@ -9,6 +9,7 @@ import os import shutil import stat +import subprocess import time from datetime import datetime, timezone from fnmatch import fnmatch @@ -352,18 +353,35 @@ def _launch( build_dir.mkdir(parents=True, exist_ok=True) trial_dir.mkdir(parents=True, exist_ok=True) - def run(argv, cwd, writable, readable) -> None: + def run(stage: str, argv, cwd, writable, readable) -> None: remaining = min(seconds_remaining(lane["deadline_utc"], current()), 30) - confined_execute( - argv, cwd, writable, readable, time.monotonic() + remaining, backend=backend - ) + try: + record = confined_execute( + argv, cwd, writable, readable, time.monotonic() + remaining, backend=backend + ) + except subprocess.TimeoutExpired as exc: + raise ConfinementUnavailable(f"{stage} timed out") from exc + if record.get("status") != "completed": + raise ConfinementUnavailable(f"{stage} {record.get('status')}") if plan["build"]: - run(tuple(plan["build"]), candidate, (candidate, build_dir), ()) + run("build", tuple(plan["build"]), candidate, (candidate, build_dir), ()) candidate_result = trial_dir / "candidate.json" - run(_command(plan["trial"], candidate_result), candidate, (trial_dir,), (candidate,)) + run( + "candidate", + _command(plan["trial"], candidate_result), + candidate, + (trial_dir,), + (candidate,), + ) incumbent_result = trial_dir / "incumbent.json" - run(_command(plan["trial"], incumbent_result), incumbent, (trial_dir,), (incumbent,)) + run( + "incumbent", + _command(plan["trial"], incumbent_result), + incumbent, + (trial_dir,), + (incumbent,), + ) return candidate_result, incumbent_result @@ -378,7 +396,9 @@ def serve_ticket( ) -> dict: """Measure one ticket. A busy device leaves it queued and does not reserve an attempt.""" def current() -> datetime: - return clock() if clock is not None else now + if clock is not None: + return clock() + return datetime.now(timezone.utc) plan = json.loads((round_dir / "round-plan.json").read_text(encoding="utf-8")) result_path = round_dir / "results" / f"{ticket}.json" diff --git a/src/inferrsi/infra/confinement.py b/src/inferrsi/infra/confinement.py index 60f6d3d39..43545eb38 100644 --- a/src/inferrsi/infra/confinement.py +++ b/src/inferrsi/infra/confinement.py @@ -10,6 +10,7 @@ import json import os +import struct import subprocess import sys import tempfile @@ -87,20 +88,100 @@ def runtime_roots() -> tuple[Path, ...]: return tuple(roots) +def _elf_interpreter(executable: Path) -> str | None: + """PT_INTERP path recorded in the executable, or None for a static binary.""" + data = executable.read_bytes() + if data[:4] != b"\x7fELF": + return None + endian = "<" if data[5] == 1 else ">" + if data[4] == 2: + phoff = struct.unpack_from(endian + "Q", data, 32)[0] + phentsize, phnum = struct.unpack_from(endian + "HH", data, 54) + header = endian + "IIQQQQ" + elif data[4] == 1: + phoff = struct.unpack_from(endian + "I", data, 28)[0] + phentsize, phnum = struct.unpack_from(endian + "HH", data, 42) + header = endian + "IIIIIIII" + else: + raise ConfinementUnavailable("unsupported ELF class") + for index in range(phnum): + start = phoff + index * phentsize + fields = struct.unpack_from(header, data, start) + if fields[0] != 3: + continue + offset, size = (fields[2], fields[5]) if data[4] == 2 else (fields[1], fields[4]) + return data[offset : offset + size].split(b"\0", 1)[0].decode() + return None + + +def _shared_libraries(executable: Path) -> tuple[str, ...]: + """Absolute libraries ldd resolves. The virtual dynamic linker object is skipped.""" + completed = subprocess.run( + ["ldd", os.fspath(executable)], capture_output=True, text=True, check=False + ) + if completed.returncode != 0: + raise ConfinementUnavailable(completed.stderr.strip() or "ldd failed") + found: list[str] = [] + for line in completed.stdout.splitlines(): + if "=>" in line: + token = line.split("=>", 1)[1].strip().split()[0] + else: + token = line.strip().split()[0] + if token.startswith("/"): + found.append(token) + return tuple(found) + + +def runtime_files() -> tuple[Path, ...]: + """Loader and shared libraries that live outside the Python prefix.""" + executable = Path(sys.executable) + names: list[Path] = [] + interpreter = _elf_interpreter(executable) + if interpreter: + names.append(Path(interpreter)) + elif executable.read_bytes()[:4] == b"\x7fELF": + return () + for item in _shared_libraries(executable): + names.append(Path(item)) + cache = Path("/etc/ld.so.cache") + if cache.is_file(): + names.append(cache) + unique: list[Path] = [] + for path in names: + if path not in unique: + unique.append(path) + return tuple(unique) + + +def _covered(path: Path, covered: list[Path]) -> bool: + return any(path == root or root in path.parents for root in covered) + + def namespace_plan( writable: tuple[Path, ...], readable: tuple[Path, ...] ) -> dict: """Mount list for a fresh root. The host root itself is not an entry.""" - mounts = [{"source": str(path), "mode": "rw"} for path in writable] - mounts.extend({"source": str(path), "mode": "ro"} for path in readable) - covered = list(writable) + list(readable) + mounts: list[dict] = [] + covered: list[Path] = [] + + def add(path: Path, mode: str, *, directory: bool) -> None: + source = str(path) + if source == "/" or ".." in path.parts: + raise ConfinementUnavailable("refusing a mount plan that exposes the host root") + if any(item["source"] == source for item in mounts) or _covered(path, covered): + return + mounts.append({"source": source, "mode": mode}) + if directory: + covered.append(path) + + for path in writable: + add(path, "rw", directory=True) + for path in readable: + add(path, "ro", directory=True) for root in runtime_roots(): - if any(root == path or path in root.parents or root in path.parents for path in covered): - continue - mounts.append({"source": str(root), "mode": "ro"}) - covered.append(root) - if any(item["source"] == "/" for item in mounts): - raise ConfinementUnavailable("refusing a mount plan that exposes the host root") + add(root, "ro", directory=True) + for extra in runtime_files(): + add(extra, "ro", directory=False) return {"mounts": mounts, "env": minimal_environment()} @@ -252,6 +333,8 @@ def run( timeout=remaining, check=False, ) + except subprocess.TimeoutExpired as exc: + raise ConfinementUnavailable("namespace command timed out") from exc finally: if config_path is not None: config_path.unlink(missing_ok=True) @@ -281,10 +364,15 @@ def mount(args: list[str]) -> None: mount(["-t", "tmpfs", "tmpfs", str(jail)]) for item in config["mounts"]: source = item["source"] - if source == "/": + source_path = Path(source) + if source == "/" or not source_path.is_absolute() or ".." in source_path.parts: raise SystemExit("refusing to mount the host root") - target = jail / Path(source).resolve().relative_to("/") - target.mkdir(parents=True, exist_ok=True) + target = jail.joinpath(*source_path.parts[1:]) + if source_path.is_dir() and not source_path.is_symlink(): + target.mkdir(parents=True, exist_ok=True) + else: + target.parent.mkdir(parents=True, exist_ok=True) + target.touch() mount(["--bind", source, str(target)]) if item["mode"] == "ro": mount(["-o", "remount,bind,ro", str(target)]) diff --git a/tests/test_confinement.py b/tests/test_confinement.py index feb479243..ca5a52a3c 100644 --- a/tests/test_confinement.py +++ b/tests/test_confinement.py @@ -1,6 +1,7 @@ """Confinement policy. The fixture backend is not an operating-system boundary.""" import json +import subprocess import sys import time from pathlib import Path @@ -12,8 +13,10 @@ FixtureBackend, UnavailableBackend, UnshareBackend, + _elf_interpreter, confined_execute, namespace_plan, + namespaces_available, select_backend, ) @@ -64,8 +67,11 @@ def test_select_backend_fails_closed_when_namespaces_are_disabled(monkeypatch): def test_unshare_failure_does_not_fall_back(tmp_path, monkeypatch): calls = [] monkeypatch.setenv("SECRET_CANARY", "CANARY-ENV") + real_run = subprocess.run def fake_run(argv, **kwargs): + if not argv or argv[0] != "unshare": + return real_run(argv, **kwargs) del kwargs calls.append(json.loads(Path(argv[-1]).read_text(encoding="utf-8"))) @@ -90,6 +96,69 @@ class Completed: assert "/" not in {item["source"] for item in calls[0]["mounts"]} +def test_unshare_timeout_becomes_confinement_failure(tmp_path, monkeypatch): + real_run = subprocess.run + + def fake_run(argv, **kwargs): + if not argv or argv[0] != "unshare": + return real_run(argv, **kwargs) + raise subprocess.TimeoutExpired(argv, kwargs.get("timeout") or 1) + + monkeypatch.setattr("inferrsi.infra.confinement.subprocess.run", fake_run) + with pytest.raises(ConfinementUnavailable, match="timed out"): + UnshareBackend().run( + (sys.executable, "-c", "print('should-not-run')"), + tmp_path, + (tmp_path,), + (), + time.monotonic() + 5, + ) + + +def test_namespace_plan_includes_the_loader_but_not_a_private_file(tmp_path, monkeypatch): + monkeypatch.setenv("SECRET_CANARY", "CANARY-ENV") + private = tmp_path / "private" + private.mkdir() + (private / "canary.txt").write_text("CANARY-FILE", encoding="utf-8") + plan = namespace_plan((tmp_path / "work",), ()) + sources = {item["source"] for item in plan["mounts"]} + interpreter = _elf_interpreter(Path(sys.executable)) + assert interpreter is not None + assert interpreter in sources or any( + Path(interpreter).is_relative_to(Path(source)) for source in sources + ) + assert "/" not in sources + assert str(private) not in sources + assert "SECRET_CANARY" not in plan["env"] + + +@pytest.mark.skipif(not namespaces_available(), reason="user namespaces are unavailable") +def test_unshare_starts_python_and_hides_a_private_canary(tmp_path, monkeypatch): + monkeypatch.setenv("SECRET_CANARY", "CANARY-ENV") + private = tmp_path / "private" + private.mkdir() + (private / "canary.txt").write_text("CANARY-FILE", encoding="utf-8") + work = tmp_path / "work" + work.mkdir() + script = ( + "import os\n" + "print('STARTED')\n" + "print(os.environ.get('SECRET_CANARY', ''))\n" + "print(os.path.exists(%r))\n" % str(private / "canary.txt") + ) + record = UnshareBackend().run( + (sys.executable, "-c", script), + work, + (work,), + (), + time.monotonic() + 30, + ) + assert record["status"] == "completed" + assert "STARTED" in record["stdout"] + assert "CANARY-ENV" not in record["stdout"] + assert "True" not in record["stdout"] + + def test_namespace_plan_keeps_host_root_and_secrets_out(tmp_path, monkeypatch): monkeypatch.setenv("SECRET_CANARY", "CANARY-ENV") plan = namespace_plan((tmp_path / "writable",), (tmp_path / "readable",)) diff --git a/tests/test_parallel_round.py b/tests/test_parallel_round.py index 9ced6d585..a467a5c13 100644 --- a/tests/test_parallel_round.py +++ b/tests/test_parallel_round.py @@ -1,9 +1,10 @@ """Synthetic parallel round. Latency figures here are fixture inputs, not measurements.""" import json +import subprocess import sys import threading -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import Path import pytest @@ -284,6 +285,69 @@ def clock(): assert not (round_dir / "candidates" / ticket / "build-stamp").exists() +def test_deadline_expiring_between_build_and_candidate_stops_the_trial(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + spec = _spec() + deadline = (WHEN + timedelta(seconds=10)).isoformat() + for lane in spec["lanes"]: + lane["deadline_utc"] = deadline + round_dir = tmp_path / "round" + prepare(spec, incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + moments = [WHEN, WHEN, WHEN + timedelta(seconds=1), WHEN + timedelta(seconds=11)] + + def clock(): + return moments.pop(0) + + class Counting: + name = "counting" + + def __init__(self) -> None: + self.calls = 0 + + def run(self, *args, **kwargs): + del args, kwargs + self.calls += 1 + return {"status": "completed", "returncode": 0, "stdout": "", "stderr": ""} + + backend = Counting() + rejected = serve_ticket(round_dir, ticket, incumbent, backend=backend, now=WHEN, clock=clock) + assert rejected["status"] == "unsettled" + assert "deadline" in rejected["reason"] + assert backend.calls == 1 + ledger = json.loads((round_dir / "ledgers" / "L1.json").read_text(encoding="utf-8")) + assert [entry["phase"] for entry in ledger["entries"]] == ["claimed"] + assert ticket_status(round_dir, ticket)["status"] == "unsettled" + + +def test_namespace_timeout_after_claim_is_an_unsettled_receipt(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare(_spec(), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + + class TimesOut: + name = "timeout" + + def run(self, *args, **kwargs): + del args, kwargs + raise subprocess.TimeoutExpired(["build"], 1) + + result = serve_ticket(round_dir, ticket, incumbent, backend=TimesOut(), now=WHEN) + assert result["status"] == "unsettled" + assert "timed out" in result["reason"] + assert result["reason"].startswith("build") + ledger = json.loads((round_dir / "ledgers" / "L1.json").read_text(encoding="utf-8")) + assert [entry["phase"] for entry in ledger["entries"]] == ["claimed"] + assert ticket_status(round_dir, ticket)["status"] == "unsettled" + + def test_failure_after_claim_stays_unsettled_across_restart(tmp_path): incumbent = tmp_path / "incumbent" _tree(incumbent, "10") diff --git a/tools/parallel_round.py b/tools/parallel_round.py index c093c8055..40ddaff59 100644 --- a/tools/parallel_round.py +++ b/tools/parallel_round.py @@ -55,7 +55,6 @@ def main(argv: list[str] | None = None) -> int: args.ticket, args.incumbent, backend=select_backend(), - now=datetime.now(timezone.utc), ) else: result = ticket_status(args.round_dir, args.ticket) From bd6ba21fec7b2eb757a5f1218069687fd8fb16e6 Mon Sep 17 00:00:00 2001 From: NumberWan Date: Tue, 10 Nov 2026 17:55:49 +0800 Subject: [PATCH 04/23] fix(harness): give serve a receipt when the trial result is missing The production serve command required a clock it never read, a blocked unshare was treated as a usable namespace, and a zero exit without a latency file left the ticket without a result. Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_round.py | 25 ++++++++--- src/inferrsi/infra/confinement.py | 19 ++++++++- tests/test_confinement.py | 17 ++++++++ tests/test_parallel_round.py | 58 ++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 7 deletions(-) diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py index d9a01553a..6d5490dfa 100644 --- a/src/inferrsi/harness/parallel_round.py +++ b/src/inferrsi/harness/parallel_round.py @@ -391,10 +391,15 @@ def serve_ticket( incumbent: Path, *, backend, - now: datetime, + now: datetime | None = None, clock=None, ) -> dict: - """Measure one ticket. A busy device leaves it queued and does not reserve an attempt.""" + """Measure one ticket. A busy device leaves it queued and does not reserve an attempt. + + ``now`` is accepted for older callers and is not a frozen deadline. Every + admission and command reads ``clock`` or the current time again. + """ + del now def current() -> datetime: if clock is not None: return clock() @@ -443,10 +448,18 @@ def current() -> datetime: ) except (ConfinementUnavailable, BudgetError) as exc: return _record(round_dir, ticket, status="unsettled", reason=str(exc)) - candidate_body = json.loads(candidate_result.read_text(encoding="utf-8")) - incumbent_body = json.loads(incumbent_result.read_text(encoding="utf-8")) - candidate_metric = candidate_body["metrics"]["latency_ms"] - incumbent_metric = incumbent_body["metrics"]["latency_ms"] + try: + candidate_body = json.loads(candidate_result.read_text(encoding="utf-8")) + incumbent_body = json.loads(incumbent_result.read_text(encoding="utf-8")) + candidate_metric = candidate_body["metrics"]["latency_ms"] + incumbent_metric = incumbent_body["metrics"]["latency_ms"] + except (OSError, json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: + return _record( + round_dir, + ticket, + status="unsettled", + reason=f"trial finished without a readable latency result: {exc}", + ) finished = advance(ledger, binding, "finished", actual_cost=0) _save_ledger(round_dir, finished) return _record( diff --git a/src/inferrsi/infra/confinement.py b/src/inferrsi/infra/confinement.py index 43545eb38..38581dc38 100644 --- a/src/inferrsi/infra/confinement.py +++ b/src/inferrsi/infra/confinement.py @@ -62,11 +62,28 @@ class ConfinementUnavailable(RuntimeError): def namespaces_available() -> bool: + """True only when a user namespace can actually be created. + + A positive ``max_user_namespaces`` is not enough: unprivileged ``unshare`` + can still return EPERM. Callers that skip live tests must see that as + unavailable instead of starting a command that errors. + """ try: limit = int(Path("/proc/sys/user/max_user_namespaces").read_text(encoding="utf-8")) except (OSError, ValueError): return False - return limit > 0 + if limit <= 0: + return False + try: + probe = subprocess.run( + ["unshare", "--user", "--map-root-user", "--mount", "--", "true"], + capture_output=True, + timeout=5, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return False + return probe.returncode == 0 def minimal_environment(extra: dict | None = None) -> dict: diff --git a/tests/test_confinement.py b/tests/test_confinement.py index ca5a52a3c..4bf61ed5b 100644 --- a/tests/test_confinement.py +++ b/tests/test_confinement.py @@ -159,6 +159,23 @@ def test_unshare_starts_python_and_hides_a_private_canary(tmp_path, monkeypatch) assert "True" not in record["stdout"] +def test_blocked_unshare_reports_namespaces_unavailable(monkeypatch): + real = Path.read_text + + def fake_read(self, *args, **kwargs): + if self.name == "max_user_namespaces": + return "1" + return real(self, *args, **kwargs) + + def blocked(argv, **kwargs): + del kwargs + return subprocess.CompletedProcess(argv, 1, stderr=b"Operation not permitted") + + monkeypatch.setattr(Path, "read_text", fake_read) + monkeypatch.setattr("inferrsi.infra.confinement.subprocess.run", blocked) + assert namespaces_available() is False + + def test_namespace_plan_keeps_host_root_and_secrets_out(tmp_path, monkeypatch): monkeypatch.setenv("SECRET_CANARY", "CANARY-ENV") plan = namespace_plan((tmp_path / "writable",), (tmp_path / "readable",)) diff --git a/tests/test_parallel_round.py b/tests/test_parallel_round.py index a467a5c13..707642fb9 100644 --- a/tests/test_parallel_round.py +++ b/tests/test_parallel_round.py @@ -1,6 +1,7 @@ """Synthetic parallel round. Latency figures here are fixture inputs, not measurements.""" import json +import os import subprocess import sys import threading @@ -379,3 +380,60 @@ def run(self, *args, **kwargs): reloaded = json.loads(ledger_path.read_text(encoding="utf-8")) assert [entry["phase"] for entry in reloaded["entries"]] == ["claimed"] assert reloaded["entries"][0]["worst_case"] == ledger["entries"][0]["worst_case"] + + +def test_completed_trial_without_a_latency_file_is_unsettled(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare(_spec(), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + + class CompletesWithoutResult: + name = "empty" + + def run(self, *args, **kwargs): + del args, kwargs + return {"status": "completed", "stdout": "", "stderr": ""} + + result = serve_ticket( + round_dir, ticket, incumbent, backend=CompletesWithoutResult(), now=WHEN + ) + assert result["status"] == "unsettled" + assert "readable latency result" in result["reason"] + assert ticket_status(round_dir, ticket)["status"] == "unsettled" + + +def test_serve_cli_runs_without_a_caller_supplied_clock(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare(_spec(), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + tool = Path(__file__).parents[1] / "tools" / "parallel_round.py" + completed = subprocess.run( + [ + sys.executable, + str(tool), + "serve", + "--round-dir", + str(round_dir), + "--ticket", + ticket, + "--incumbent", + str(incumbent), + ], + cwd=Path(__file__).parents[1], + env={**os.environ, "PYTHONPATH": str(Path(__file__).parents[1] / "src")}, + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0, completed.stderr + body = json.loads(completed.stdout) + assert body["ticket"] == ticket + assert body["status"] in {"measured", "unsettled", "rejected", "queued"} From a751e760dd374c2364f2e119d847d4c2684d3199 Mon Sep 17 00:00:00 2001 From: NumberWan Date: Thu, 12 Nov 2026 10:26:54 +0800 Subject: [PATCH 05/23] feat(harness): let a parallel round use several devices and a declared metric The round plan now lists device_ids and declares one metric with its direction. serve takes any free device, substitutes {device} into the build and trial commands, and reports gain so that positive is better. A lane lock serializes tickets of the same lane, so two devices never update one ledger at the same time. A non-numeric metric leaves an unsettled receipt instead of raising after the claim. Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_round.py | 208 ++++++++++++++++++------- tests/test_parallel_round.py | 112 ++++++++++++- 2 files changed, 258 insertions(+), 62 deletions(-) diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py index 6d5490dfa..0dba6a330 100644 --- a/src/inferrsi/harness/parallel_round.py +++ b/src/inferrsi/harness/parallel_round.py @@ -1,11 +1,12 @@ """Prepare, submit and measure one offline parallel round. Directory diffs stand in for git clones. ``close`` and incumbent advancement are -not implemented. A synthetic latency number is not a campaign verdict. +not implemented. A synthetic metric value is not a campaign verdict. """ import hashlib import json +import math import os import shutil import stat @@ -33,12 +34,17 @@ from inferrsi.infra.device_lease import DeviceLease, LeaseError PLANES = {"harness", "model", "infra"} +DIRECTIONS = {"lower", "higher"} class RoundPlanError(ValueError): """The frozen round plan is internally inconsistent.""" +def _segment(value) -> bool: + return isinstance(value, str) and bool(value) and not any(char in value for char in "/\\") + + def _match(path: str, patterns: list[str]) -> bool: return any(fnmatch(path, pattern) for pattern in patterns) @@ -158,6 +164,24 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: protected = spec.get("protected", []) if not isinstance(protected, list): raise RoundPlanError("protected paths must be a list of globs") + devices = spec.get("device_ids") + if ( + not isinstance(devices, list) + or not devices + or not all(_segment(item) for item in devices) + or len(set(devices)) != len(devices) + ): + raise RoundPlanError("device_ids must list distinct device identities") + metric = spec.get("metric") + if ( + not isinstance(metric, dict) + or set(metric) != {"name", "better"} + or not isinstance(metric["name"], str) + or not metric["name"] + or not isinstance(metric["better"], str) + or metric["better"] not in DIRECTIONS + ): + raise RoundPlanError("metric needs a name and better: lower or higher") seen_entries: set[str] = set() seen_ids: set[str] = set() files = _files(incumbent) @@ -167,7 +191,7 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: entry = lane.get("entry") plane = lane.get("plane") globs = lane.get("globs") - if not isinstance(lane_id, str) or not lane_id or lane_id in seen_ids: + if not _segment(lane_id) or lane_id in seen_ids: raise RoundPlanError("each lane needs its own id") if not isinstance(entry, str) or not entry: raise RoundPlanError("each lane needs one search-plan entry") @@ -198,7 +222,8 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: raise RoundPlanError(f"{lane['id']} overlaps {other['id']}") body = { "round_id": spec["round_id"], - "device_id": spec["device_id"], + "device_ids": list(devices), + "metric": dict(metric), "incumbent_id": spec["incumbent_id"], "protected": protected, "build": list(spec.get("build", [])), @@ -327,8 +352,18 @@ def _materialize(incumbent: Path, overlay: Path, destination: Path) -> None: shutil.copy2(overlay / relative, target) -def _command(template: list[str], result: Path) -> tuple[str, ...]: - return tuple(part.replace("{result}", str(result)) for part in template) +def _command(template: list[str], result: Path | None, device: str) -> tuple[str, ...]: + parts = (part.replace("{device}", device) for part in template) + if result is None: + return tuple(parts) + return tuple(part.replace("{result}", str(result)) for part in parts) + + +def _metric_value(path: Path, name: str) -> float: + value = json.loads(path.read_text(encoding="utf-8"))["metrics"][name] + if type(value) not in (int, float) or not math.isfinite(value): + raise ValueError(f"{name} is not a finite number") + return value def _record(round_dir: Path, ticket: str, **fields) -> dict: @@ -346,9 +381,17 @@ def _claimed_entry(ledger, ticket: str): def _launch( - plan: dict, lane: dict, candidate: Path, incumbent: Path, trial_dir: Path, *, backend, current + plan: dict, + lane: dict, + candidate: Path, + incumbent: Path, + trial_dir: Path, + device: str, + *, + backend, + current, ): - """Run build and both trials. Each start is bounded by the same absolute deadline.""" + """Run build and both trials on one device. Each start uses the same absolute deadline.""" build_dir = candidate / "build-output" build_dir.mkdir(parents=True, exist_ok=True) trial_dir.mkdir(parents=True, exist_ok=True) @@ -365,11 +408,11 @@ def run(stage: str, argv, cwd, writable, readable) -> None: raise ConfinementUnavailable(f"{stage} {record.get('status')}") if plan["build"]: - run("build", tuple(plan["build"]), candidate, (candidate, build_dir), ()) + run("build", _command(plan["build"], None, device), candidate, (candidate, build_dir), ()) candidate_result = trial_dir / "candidate.json" run( "candidate", - _command(plan["trial"], candidate_result), + _command(plan["trial"], candidate_result, device), candidate, (trial_dir,), (candidate,), @@ -377,7 +420,7 @@ def run(stage: str, argv, cwd, writable, readable) -> None: incumbent_result = trial_dir / "incumbent.json" run( "incumbent", - _command(plan["trial"], incumbent_result), + _command(plan["trial"], incumbent_result, device), incumbent, (trial_dir,), (incumbent,), @@ -394,7 +437,8 @@ def serve_ticket( now: datetime | None = None, clock=None, ) -> dict: - """Measure one ticket. A busy device leaves it queued and does not reserve an attempt. + """Measure one ticket on any free device. A busy lane or no free device leaves it + queued and does not reserve an attempt. ``now`` is accepted for older callers and is not a frozen deadline. Every admission and command reads ``clock`` or the current time again. @@ -414,65 +458,115 @@ def current() -> datetime: if manifest.get("deleted"): return _record(round_dir, ticket, status="rejected", reason="deletion") lane = _lane(plan, manifest["lane_id"]) - if _claimed_entry(_load_ledger(round_dir, plan, lane), ticket) is not None: - return _record( - round_dir, - ticket, - status="unsettled", - reason="claimed execution was not settled", - ) try: - held = DeviceLease(round_dir / "leases").acquire(plan["device_id"]) + lane_lock = DeviceLease(round_dir / "lane-locks").acquire(lane["id"]) except LeaseError: return {"ticket": ticket, "status": "queued"} try: - try: - loaded = _load_ledger(round_dir, plan, lane) - ledger, binding = admit_attempt(plan, loaded, lane["id"], current(), token=ticket) - except BudgetError as exc: - return _record(round_dir, ticket, status="rejected", reason=str(exc)) - _save_ledger(round_dir, ledger) - candidate = round_dir / "candidates" / ticket - trial_dir = round_dir / "trials" / ticket - _materialize(incumbent, overlay, candidate) - try: - seconds_remaining(lane["deadline_utc"], current()) - except BudgetError as exc: - _save_ledger(round_dir, advance(ledger, binding, "cancelled")) - return _record(round_dir, ticket, status="rejected", reason=str(exc)) - ledger = advance(ledger, binding, "claimed") - _save_ledger(round_dir, ledger) - try: - candidate_result, incumbent_result = _launch( - plan, lane, candidate, incumbent, trial_dir, backend=backend, current=current - ) - except (ConfinementUnavailable, BudgetError) as exc: - return _record(round_dir, ticket, status="unsettled", reason=str(exc)) - try: - candidate_body = json.loads(candidate_result.read_text(encoding="utf-8")) - incumbent_body = json.loads(incumbent_result.read_text(encoding="utf-8")) - candidate_metric = candidate_body["metrics"]["latency_ms"] - incumbent_metric = incumbent_body["metrics"]["latency_ms"] - except (OSError, json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: + if result_path.exists(): + return json.loads(result_path.read_text(encoding="utf-8")) + if _claimed_entry(_load_ledger(round_dir, plan, lane), ticket) is not None: return _record( round_dir, ticket, status="unsettled", - reason=f"trial finished without a readable latency result: {exc}", + reason="claimed execution was not settled", ) - finished = advance(ledger, binding, "finished", actual_cost=0) - _save_ledger(round_dir, finished) + leases = DeviceLease(round_dir / "leases") + for device in plan["device_ids"]: + try: + held = leases.acquire(device) + except LeaseError: + continue + try: + return _measure( + round_dir, + plan, + lane, + ticket, + overlay, + incumbent, + device, + backend=backend, + current=current, + ) + finally: + held.release() + return {"ticket": ticket, "status": "queued"} + finally: + lane_lock.release() + + +def _measure( + round_dir: Path, + plan: dict, + lane: dict, + ticket: str, + overlay: Path, + incumbent: Path, + device: str, + *, + backend, + current, +) -> dict: + try: + loaded = _load_ledger(round_dir, plan, lane) + ledger, binding = admit_attempt(plan, loaded, lane["id"], current(), token=ticket) + except BudgetError as exc: + return _record(round_dir, ticket, status="rejected", reason=str(exc)) + _save_ledger(round_dir, ledger) + candidate = round_dir / "candidates" / ticket + trial_dir = round_dir / "trials" / ticket + _materialize(incumbent, overlay, candidate) + try: + seconds_remaining(lane["deadline_utc"], current()) + except BudgetError as exc: + _save_ledger(round_dir, advance(ledger, binding, "cancelled")) + return _record(round_dir, ticket, status="rejected", reason=str(exc)) + ledger = advance(ledger, binding, "claimed") + _save_ledger(round_dir, ledger) + try: + candidate_result, incumbent_result = _launch( + plan, + lane, + candidate, + incumbent, + trial_dir, + device, + backend=backend, + current=current, + ) + except (ConfinementUnavailable, BudgetError) as exc: + return _record(round_dir, ticket, status="unsettled", reason=str(exc), device=device) + metric = plan["metric"] + try: + candidate_value = _metric_value(candidate_result, metric["name"]) + incumbent_value = _metric_value(incumbent_result, metric["name"]) + except (OSError, json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: return _record( round_dir, ticket, - status="measured", - candidate_latency_ms=candidate_metric, - incumbent_latency_ms=incumbent_metric, - gain_ms=incumbent_metric - candidate_metric, - confinement=getattr(backend, "name", "unknown"), + status="unsettled", + reason=f"trial finished without a readable {metric['name']} result: {exc}", + device=device, ) - finally: - held.release() + finished = advance(ledger, binding, "finished", actual_cost=0) + _save_ledger(round_dir, finished) + gain = incumbent_value - candidate_value + if metric["better"] == "higher": + gain = -gain + return _record( + round_dir, + ticket, + status="measured", + metric=metric["name"], + better=metric["better"], + candidate=candidate_value, + incumbent=incumbent_value, + gain=gain, + device=device, + confinement=getattr(backend, "name", "unknown"), + ) def ticket_status(round_dir: Path, ticket: str) -> dict: diff --git a/tests/test_parallel_round.py b/tests/test_parallel_round.py index 707642fb9..28e9ebaf0 100644 --- a/tests/test_parallel_round.py +++ b/tests/test_parallel_round.py @@ -63,7 +63,8 @@ def _spec(**overrides): ) spec = { "round_id": "r1", - "device_id": "cpu-0", + "device_ids": ["cpu-0"], + "metric": {"name": "latency_ms", "better": "lower"}, "incumbent_id": "fixture-incumbent", "protected": ["bench.py"], "build": [sys.executable, "-c", "open('build-stamp','w').write('ok')"], @@ -126,6 +127,23 @@ def test_prepare_rejects_overlapping_globs_protected_paths_and_shared_entries(tm prepare(many, incumbent, tmp_path / "round-plane") +def test_prepare_rejects_bad_devices_and_metric(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + for devices in ([], ["cpu-0", "cpu-0"], ["a/b"], "cpu-0"): + with pytest.raises(RoundPlanError, match="device_ids"): + prepare(_spec(device_ids=devices), incumbent, tmp_path / "round-devices") + for metric in ( + None, + {"name": "latency_ms"}, + {"name": "", "better": "lower"}, + {"name": "latency_ms", "better": "faster"}, + {"name": "latency_ms", "better": ["lower"]}, + ): + with pytest.raises(RoundPlanError, match="metric"): + prepare(_spec(metric=metric), incumbent, tmp_path / "round-metric") + + def test_prepare_records_plan_cap_and_deadline(tmp_path): incumbent = tmp_path / "incumbent" _tree(incumbent, "10") @@ -172,13 +190,97 @@ def test_serve_records_a_synthetic_gain_and_a_held_device_stays_queued(tmp_path) held.release() measured = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) assert measured["synthetic"] is True - assert measured["candidate_latency_ms"] == 8 - assert measured["incumbent_latency_ms"] == 10 - assert measured["gain_ms"] == 2 + assert measured["metric"] == "latency_ms" + assert measured["candidate"] == 8 + assert measured["incumbent"] == 10 + assert measured["gain"] == 2 + assert measured["device"] == "cpu-0" assert measured["confinement"] == "fixture" assert (round_dir / "candidates" / ticket / "build-stamp").read_text(encoding="utf-8") == "ok" +def test_higher_is_better_metric_reports_gain_in_the_better_direction(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare(_spec(metric={"name": "latency_ms", "better": "higher"}), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + measured = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + assert measured["status"] == "measured" + assert measured["better"] == "higher" + assert measured["gain"] == -2 + + +def test_serve_uses_a_free_device_and_passes_its_identity_to_the_commands(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + stamp_device = "import sys; open('build-stamp', 'w').write(sys.argv[1])" + spec = _spec( + device_ids=["gpu-0", "gpu-1"], + build=[sys.executable, "-c", stamp_device, "{device}"], + ) + prepare(spec, incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + held = DeviceLease(round_dir / "leases").acquire("gpu-0") + try: + measured = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + finally: + held.release() + assert measured["status"] == "measured" + assert measured["device"] == "gpu-1" + stamp = round_dir / "candidates" / ticket / "build-stamp" + assert stamp.read_text(encoding="utf-8") == "gpu-1" + + +def test_a_busy_lane_stays_queued_without_touching_its_ledger(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare(_spec(device_ids=["gpu-0", "gpu-1"]), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + held = DeviceLease(round_dir / "lane-locks").acquire("L1") + try: + queued = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + finally: + held.release() + assert queued["status"] == "queued" + assert not (round_dir / "ledgers").exists() + assert not (round_dir / "results").exists() + + +def test_a_non_numeric_metric_is_an_unsettled_receipt(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare(_spec(), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + + class WritesText: + name = "text" + + def run(self, argv, *args, **kwargs): + del args, kwargs + if argv[-1].endswith(".json"): + body = {"metrics": {"latency_ms": "fast"}} + Path(argv[-1]).write_text(json.dumps(body), encoding="utf-8") + return {"status": "completed", "stdout": "", "stderr": ""} + + result = serve_ticket(round_dir, ticket, incumbent, backend=WritesText(), now=WHEN) + assert result["status"] == "unsettled" + assert "not a finite number" in result["reason"] + ledger = json.loads((round_dir / "ledgers" / "L1.json").read_text(encoding="utf-8")) + assert [entry["phase"] for entry in ledger["entries"]] == ["claimed"] + + def test_two_acquires_do_not_share_one_device(tmp_path): lease = DeviceLease(tmp_path) held = lease.acquire("cpu-0") @@ -402,7 +504,7 @@ def run(self, *args, **kwargs): round_dir, ticket, incumbent, backend=CompletesWithoutResult(), now=WHEN ) assert result["status"] == "unsettled" - assert "readable latency result" in result["reason"] + assert "readable latency_ms result" in result["reason"] assert ticket_status(round_dir, ticket)["status"] == "unsettled" From 34f60e20948953908d9b8d0cea3b4fca5de44a35 Mon Sep 17 00:00:00 2001 From: NumberWan Date: Thu, 12 Nov 2026 13:06:22 +0800 Subject: [PATCH 06/23] fix(harness): reject dot device and lane names, document trial placeholders "." and ".." passed the single-segment check and later made DeviceLease open a directory, so serve raised instead of queueing. prepare now rejects them. The module docstring states which placeholders build and trial receive and the result file shape. The serve CLI test asserts the exact status for the namespace branch the host takes. Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_round.py | 11 ++++++++++- tests/test_parallel_round.py | 19 ++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py index 0dba6a330..e92e285e2 100644 --- a/src/inferrsi/harness/parallel_round.py +++ b/src/inferrsi/harness/parallel_round.py @@ -2,6 +2,11 @@ Directory diffs stand in for git clones. ``close`` and incumbent advancement are not implemented. A synthetic metric value is not a campaign verdict. + +Command placeholders: ``build`` gets ``{device}`` only. ``trial`` gets ``{device}`` +and ``{result}``; it runs once for the candidate and once for the incumbent and +must write ``{"metrics": {: }}`` to ``{result}``. +Other text, including ``{result}`` inside ``build``, is passed through unchanged. """ import hashlib @@ -42,7 +47,11 @@ class RoundPlanError(ValueError): def _segment(value) -> bool: - return isinstance(value, str) and bool(value) and not any(char in value for char in "/\\") + return ( + isinstance(value, str) + and value not in {"", ".", ".."} + and not any(char in value for char in "/\\") + ) def _match(path: str, patterns: list[str]) -> bool: diff --git a/tests/test_parallel_round.py b/tests/test_parallel_round.py index 28e9ebaf0..2060a5339 100644 --- a/tests/test_parallel_round.py +++ b/tests/test_parallel_round.py @@ -17,7 +17,11 @@ submit, ticket_status, ) -from inferrsi.infra.confinement import ConfinementUnavailable, FixtureBackend +from inferrsi.infra.confinement import ( + ConfinementUnavailable, + FixtureBackend, + namespaces_available, +) from inferrsi.infra.device_lease import DeviceLease, LeaseError WHEN = datetime(2026, 10, 30, tzinfo=timezone.utc) @@ -130,9 +134,14 @@ def test_prepare_rejects_overlapping_globs_protected_paths_and_shared_entries(tm def test_prepare_rejects_bad_devices_and_metric(tmp_path): incumbent = tmp_path / "incumbent" _tree(incumbent, "10") - for devices in ([], ["cpu-0", "cpu-0"], ["a/b"], "cpu-0"): + for devices in ([], ["cpu-0", "cpu-0"], ["a/b"], "cpu-0", ["."], [".."]): with pytest.raises(RoundPlanError, match="device_ids"): prepare(_spec(device_ids=devices), incumbent, tmp_path / "round-devices") + for lane_id in (".", ".."): + dotted = _spec() + dotted["lanes"][0]["id"] = lane_id + with pytest.raises(RoundPlanError, match="own id"): + prepare(dotted, incumbent, tmp_path / "round-lane-id") for metric in ( None, {"name": "latency_ms"}, @@ -538,4 +547,8 @@ def test_serve_cli_runs_without_a_caller_supplied_clock(tmp_path): assert completed.returncode == 0, completed.stderr body = json.loads(completed.stdout) assert body["ticket"] == ticket - assert body["status"] in {"measured", "unsettled", "rejected", "queued"} + if namespaces_available(): + assert body["status"] == "measured" + else: + assert body["status"] == "unsettled" + assert body["reason"] == "confinement backend unavailable" From f6bd0694061ad71a970bcfde12c56ad48b1398ed Mon Sep 17 00:00:00 2001 From: NumberWan Date: Thu, 12 Nov 2026 15:37:04 +0800 Subject: [PATCH 07/23] feat(harness): let each lane declare how many devices one trial needs A lane may set device_count (default 1). prepare rejects a plan whose lanes ask for more devices in total than device_ids lists, so a round stops adding lanes once the devices are used up. serve runs a ticket only after it holds that many free devices at once; otherwise it releases what it took and leaves the ticket queued. {device} becomes the held identities joined with commas, so device names may not contain a comma. Results record the held devices as a list. Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_round.py | 79 ++++++++++++++++---------- tests/test_parallel_round.py | 60 +++++++++++++++++-- 2 files changed, 102 insertions(+), 37 deletions(-) diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py index e92e285e2..8f2fd5924 100644 --- a/src/inferrsi/harness/parallel_round.py +++ b/src/inferrsi/harness/parallel_round.py @@ -3,10 +3,14 @@ Directory diffs stand in for git clones. ``close`` and incumbent advancement are not implemented. A synthetic metric value is not a campaign verdict. +Each lane declares ``device_count`` (default 1). The counts of all lanes must fit +in ``device_ids``. A ticket runs only after it holds that many devices at once. + Command placeholders: ``build`` gets ``{device}`` only. ``trial`` gets ``{device}`` and ``{result}``; it runs once for the candidate and once for the incumbent and must write ``{"metrics": {: }}`` to ``{result}``. -Other text, including ``{result}`` inside ``build``, is passed through unchanged. +``{device}`` is the held device identities joined with commas. Other text, +including ``{result}`` inside ``build``, is passed through unchanged. """ import hashlib @@ -177,7 +181,7 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: if ( not isinstance(devices, list) or not devices - or not all(_segment(item) for item in devices) + or not all(_segment(item) and "," not in item for item in devices) or len(set(devices)) != len(devices) ): raise RoundPlanError("device_ids must list distinct device identities") @@ -214,11 +218,19 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: raise RoundPlanError("cap and per-attempt cost must come from the plan") if not isinstance(lane.get("deadline_utc"), str): raise RoundPlanError("each lane needs a deadline") + count = lane.get("device_count", 1) + if type(count) is not int or count < 1: + raise RoundPlanError(f"{lane_id} device_count must be a positive integer") if any(_match(path, globs) and _match(path, protected) for path in files): raise RoundPlanError(f"{lane_id} reaches a protected path") seen_ids.add(lane_id) seen_entries.add(entry) - normalized.append(dict(lane)) + normalized.append({**lane, "device_count": count}) + demand = sum(lane["device_count"] for lane in normalized) + if demand > len(devices): + raise RoundPlanError( + f"lanes ask for {demand} devices in total but device_ids lists {len(devices)}" + ) for index, lane in enumerate(normalized): for other in normalized[index + 1 :]: shared = set(lane["globs"]) & set(other["globs"]) @@ -400,7 +412,7 @@ def _launch( backend, current, ): - """Run build and both trials on one device. Each start uses the same absolute deadline.""" + """Run build and both trials on the held devices. Each start uses the same deadline.""" build_dir = candidate / "build-output" build_dir.mkdir(parents=True, exist_ok=True) trial_dir.mkdir(parents=True, exist_ok=True) @@ -446,8 +458,8 @@ def serve_ticket( now: datetime | None = None, clock=None, ) -> dict: - """Measure one ticket on any free device. A busy lane or no free device leaves it - queued and does not reserve an attempt. + """Measure one ticket on the lane's ``device_count`` free devices. A busy lane or too + few free devices leaves it queued and does not reserve an attempt. ``now`` is accepted for older callers and is not a frozen deadline. Every admission and command reads ``clock`` or the current time again. @@ -482,26 +494,31 @@ def current() -> datetime: reason="claimed execution was not settled", ) leases = DeviceLease(round_dir / "leases") - for device in plan["device_ids"]: - try: - held = leases.acquire(device) - except LeaseError: - continue - try: - return _measure( - round_dir, - plan, - lane, - ticket, - overlay, - incumbent, - device, - backend=backend, - current=current, - ) - finally: - held.release() - return {"ticket": ticket, "status": "queued"} + held: list[tuple[str, object]] = [] + try: + for device in plan["device_ids"]: + if len(held) == lane["device_count"]: + break + try: + held.append((device, leases.acquire(device))) + except LeaseError: + continue + if len(held) < lane["device_count"]: + return {"ticket": ticket, "status": "queued"} + return _measure( + round_dir, + plan, + lane, + ticket, + overlay, + incumbent, + [device for device, _lease in held], + backend=backend, + current=current, + ) + finally: + for _device, lease in held: + lease.release() finally: lane_lock.release() @@ -513,7 +530,7 @@ def _measure( ticket: str, overlay: Path, incumbent: Path, - device: str, + devices: list[str], *, backend, current, @@ -541,12 +558,12 @@ def _measure( candidate, incumbent, trial_dir, - device, + ",".join(devices), backend=backend, current=current, ) except (ConfinementUnavailable, BudgetError) as exc: - return _record(round_dir, ticket, status="unsettled", reason=str(exc), device=device) + return _record(round_dir, ticket, status="unsettled", reason=str(exc), devices=devices) metric = plan["metric"] try: candidate_value = _metric_value(candidate_result, metric["name"]) @@ -557,7 +574,7 @@ def _measure( ticket, status="unsettled", reason=f"trial finished without a readable {metric['name']} result: {exc}", - device=device, + devices=devices, ) finished = advance(ledger, binding, "finished", actual_cost=0) _save_ledger(round_dir, finished) @@ -573,7 +590,7 @@ def _measure( candidate=candidate_value, incumbent=incumbent_value, gain=gain, - device=device, + devices=devices, confinement=getattr(backend, "name", "unknown"), ) diff --git a/tests/test_parallel_round.py b/tests/test_parallel_round.py index 2060a5339..7f5c9cac9 100644 --- a/tests/test_parallel_round.py +++ b/tests/test_parallel_round.py @@ -67,7 +67,7 @@ def _spec(**overrides): ) spec = { "round_id": "r1", - "device_ids": ["cpu-0"], + "device_ids": ["cpu-0", "cpu-1"], "metric": {"name": "latency_ms", "better": "lower"}, "incumbent_id": "fixture-incumbent", "protected": ["bench.py"], @@ -134,7 +134,7 @@ def test_prepare_rejects_overlapping_globs_protected_paths_and_shared_entries(tm def test_prepare_rejects_bad_devices_and_metric(tmp_path): incumbent = tmp_path / "incumbent" _tree(incumbent, "10") - for devices in ([], ["cpu-0", "cpu-0"], ["a/b"], "cpu-0", ["."], [".."]): + for devices in ([], ["cpu-0", "cpu-0"], ["a/b"], "cpu-0", ["."], [".."], ["0,1", "2"]): with pytest.raises(RoundPlanError, match="device_ids"): prepare(_spec(device_ids=devices), incumbent, tmp_path / "round-devices") for lane_id in (".", ".."): @@ -191,23 +191,71 @@ def test_serve_records_a_synthetic_gain_and_a_held_device_stays_queued(tmp_path) work = tmp_path / "work" _tree(work, "8") ticket = submit(round_dir, "L1", incumbent, work)["ticket"] - held = DeviceLease(round_dir / "leases").acquire("cpu-0") + leases = DeviceLease(round_dir / "leases") + held = [leases.acquire("cpu-0"), leases.acquire("cpu-1")] queued = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) assert queued["status"] == "queued" assert ticket_status(round_dir, ticket)["status"] == "queued" assert not (round_dir / "ledgers").exists() - held.release() + for lease in held: + lease.release() measured = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) assert measured["synthetic"] is True assert measured["metric"] == "latency_ms" assert measured["candidate"] == 8 assert measured["incumbent"] == 10 assert measured["gain"] == 2 - assert measured["device"] == "cpu-0" + assert measured["devices"] == ["cpu-0"] assert measured["confinement"] == "fixture" assert (round_dir / "candidates" / ticket / "build-stamp").read_text(encoding="utf-8") == "ok" +def test_prepare_rejects_bad_device_counts_and_lanes_that_need_more_devices(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + for count in (0, -1, True, "2"): + spec = _spec() + spec["lanes"][0]["device_count"] = count + with pytest.raises(RoundPlanError, match="device_count"): + prepare(spec, incumbent, tmp_path / "round-count") + spec = _spec(device_ids=["0", "1", "2"]) + spec["lanes"][1]["device_count"] = 3 + with pytest.raises(RoundPlanError, match="4 devices in total"): + prepare(spec, incumbent, tmp_path / "round-demand") + spec["lanes"][1]["device_count"] = 2 + plan = prepare(spec, incumbent, tmp_path / "round-fits") + assert [lane["device_count"] for lane in plan["lanes"]] == [1, 2] + + +def test_a_lane_needing_two_devices_waits_until_it_can_hold_both(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + stamp_device = "import sys; open('build-stamp', 'w').write(sys.argv[1])" + spec = _spec( + device_ids=["0", "1", "2"], + build=[sys.executable, "-c", stamp_device, "{device}"], + ) + spec["lanes"][0]["device_count"] = 2 + prepare(spec, incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + leases = DeviceLease(round_dir / "leases") + held = [leases.acquire("0"), leases.acquire("1")] + queued = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + assert queued["status"] == "queued" + assert not (round_dir / "ledgers").exists() + leases.acquire("2").release() + held.pop().release() + measured = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + held[0].release() + assert measured["status"] == "measured" + assert measured["devices"] == ["1", "2"] + stamp = round_dir / "candidates" / ticket / "build-stamp" + assert stamp.read_text(encoding="utf-8") == "1,2" + + def test_higher_is_better_metric_reports_gain_in_the_better_direction(tmp_path): incumbent = tmp_path / "incumbent" _tree(incumbent, "10") @@ -241,7 +289,7 @@ def test_serve_uses_a_free_device_and_passes_its_identity_to_the_commands(tmp_pa finally: held.release() assert measured["status"] == "measured" - assert measured["device"] == "gpu-1" + assert measured["devices"] == ["gpu-1"] stamp = round_dir / "candidates" / ticket / "build-stamp" assert stamp.read_text(encoding="utf-8") == "gpu-1" From fb5f0fc4c2de5a188ed71ec1ffdfacfdebf368d1 Mon Sep 17 00:00:00 2001 From: NumberWan Date: Thu, 12 Nov 2026 16:03:03 +0800 Subject: [PATCH 08/23] feat(harness): drive a cluster round end to end and close it drive runs one worker per lane: the lane's agent command edits a work tree copied from the incumbent, then submit and serve run until the agent has nothing new, a receipt is not measured, or the lane cap or deadline is reached. close then picks the next incumbent and writes it with close.json. serve now runs metric.repeats candidate and incumbent trials in ABBA order and records every sample, the medians, gain and relative_gain. A lane passes close when its best ticket's relative gain reaches metric.margin. Two or more passing lanes are stacked and measured once more; the stack is chosen only if it gains at least as much as the best single lane. That extra measurement is not charged to a lane. GPU trials: device_paths per device id, shared_device_paths, runtime_paths and env (values may use {device}) extend the confined view. Device nodes are bound read-write; with device nodes the command gets a new PID namespace and a fresh /proc, never the host /proc. Every jail gets an empty /tmp and /dev/shm. command_timeout_seconds replaces the fixed 30 second cap per command. Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_close.py | 155 +++++++++++++ src/inferrsi/harness/parallel_drive.py | 120 ++++++++++ src/inferrsi/harness/parallel_round.py | 291 ++++++++++++++++++------- src/inferrsi/infra/confinement.py | 59 ++++- tests/test_confinement.py | 111 ++++++++++ tests/test_parallel_close.py | 159 ++++++++++++++ tests/test_parallel_drive.py | 151 +++++++++++++ tests/test_parallel_round.py | 113 ++++++++++ tools/parallel_round.py | 25 +++ 9 files changed, 1094 insertions(+), 90 deletions(-) create mode 100644 src/inferrsi/harness/parallel_close.py create mode 100644 src/inferrsi/harness/parallel_drive.py create mode 100644 tests/test_parallel_close.py create mode 100644 tests/test_parallel_drive.py diff --git a/src/inferrsi/harness/parallel_close.py b/src/inferrsi/harness/parallel_close.py new file mode 100644 index 000000000..d3fba7e7e --- /dev/null +++ b/src/inferrsi/harness/parallel_close.py @@ -0,0 +1,155 @@ +"""Close one parallel round: pick passing lanes and write the next incumbent. + +A lane passes when its best measured ticket beats the incumbent by at least +``metric.margin`` (relative gain). With two or more passing lanes, their overlays +are stacked and measured against the incumbent once more. The stack is chosen only +if its gain is at least the best single lane's gain; otherwise the best single lane +is. The extra measurement is not charged to any lane's attempt budget. +""" + +import json +from datetime import datetime, timezone +from pathlib import Path + +from inferrsi.harness.parallel_round import ( + MeasurementError, + _snapshot, + compare, + hold_devices, + materialize, + summarize, +) +from inferrsi.harness.round_budget import BudgetError +from inferrsi.infra.artifacts import write_json +from inferrsi.infra.confinement import ConfinementUnavailable + + +class CloseError(RuntimeError): + """The round cannot be closed as asked.""" + + +def _score(result: dict) -> float: + relative = result.get("relative_gain") + return result["gain"] if relative is None else relative + + +def _passes(result: dict, margin: float) -> bool: + relative = result.get("relative_gain") + return result["gain"] > 0 and (relative is None or relative >= margin) + + +def _best_per_lane(round_dir: Path) -> dict[str, dict]: + best: dict[str, dict] = {} + for path in sorted((round_dir / "results").glob("*.json")): + result = json.loads(path.read_text(encoding="utf-8")) + if result.get("status") != "measured": + continue + lane_id = result.get("lane_id") + if lane_id is None: + manifest = round_dir / "outbox" / result["ticket"] / "manifest.json" + lane_id = json.loads(manifest.read_text(encoding="utf-8"))["lane_id"] + if lane_id not in best or _score(result) > _score(best[lane_id]): + best[lane_id] = {**result, "lane_id": lane_id} + return best + + +def close_round(round_dir: Path, incumbent: Path, output: Path, *, backend, clock=None) -> dict: + """Write the next incumbent to ``output`` and ``close.json``. Repeated calls return + the first decision without measuring again.""" + record_path = round_dir / "close.json" + if record_path.exists(): + return json.loads(record_path.read_text(encoding="utf-8")) + if output.exists() and any(output.iterdir()): + raise CloseError(f"output {output} already has files") + + def current() -> datetime: + if clock is not None: + return clock() + return datetime.now(timezone.utc) + + plan = json.loads((round_dir / "round-plan.json").read_text(encoding="utf-8")) + margin = plan["metric"]["margin"] + lane_count = {lane["id"]: lane["device_count"] for lane in plan["lanes"]} + best = _best_per_lane(round_dir) + passing = sorted( + (result for result in best.values() if _passes(result, margin)), + key=_score, + reverse=True, + ) + composite = None + chosen = passing[:1] + if len(passing) >= 2: + composite = _measure_composite( + round_dir, plan, incumbent, passing, lane_count, backend=backend, current=current + ) + if composite.get("status") == "measured" and _score(composite) >= _score(passing[0]): + chosen = passing + overlays = [round_dir / "outbox" / result["ticket"] for result in chosen] + if output.exists(): + output.rmdir() + materialize(incumbent, overlays, output) + hashes, _blocked = _snapshot(output) + body = { + "round_digest": plan["round_digest"], + "metric": plan["metric"], + "lanes": { + lane_id: { + "ticket": result["ticket"], + "gain": result["gain"], + "relative_gain": result.get("relative_gain"), + "passed": _passes(result, margin), + } + for lane_id, result in sorted(best.items()) + }, + "passing": [result["lane_id"] for result in passing], + "composite": composite, + "choice": ("kept_incumbent" if not chosen else "composite" if len(chosen) > 1 else "lane"), + "chosen_tickets": [result["ticket"] for result in chosen], + "output": str(output), + "files": hashes, + } + write_json(record_path, body) + return body + + +def _measure_composite( + round_dir: Path, + plan: dict, + incumbent: Path, + passing: list[dict], + lane_count: dict[str, int], + *, + backend, + current, +) -> dict: + count = max(lane_count[result["lane_id"]] for result in passing) + held = hold_devices(round_dir, plan, count) + if held is None: + return {"status": "unsettled", "reason": f"could not hold {count} free devices"} + devices = [device for device, _lease in held] + composite = round_dir / "close" / "composite" + try: + materialize( + incumbent, [round_dir / "outbox" / result["ticket"] for result in passing], composite + ) + candidate_samples, incumbent_samples = compare( + plan, + composite, + incumbent, + round_dir / "close" / "trials", + devices, + None, + backend=backend, + current=current, + ) + except (ConfinementUnavailable, BudgetError, MeasurementError) as exc: + return {"status": "unsettled", "reason": str(exc), "devices": devices} + finally: + for _device, lease in held: + lease.release() + return { + "status": "measured", + "tickets": [result["ticket"] for result in passing], + **summarize(plan, candidate_samples, incumbent_samples), + "devices": devices, + } diff --git a/src/inferrsi/harness/parallel_drive.py b/src/inferrsi/harness/parallel_drive.py new file mode 100644 index 000000000..fe85b2e06 --- /dev/null +++ b/src/inferrsi/harness/parallel_drive.py @@ -0,0 +1,120 @@ +"""Drive one parallel round from a single command, then close it. + +Each lane gets one worker. A worker keeps a work tree at ``round_dir/work/`` +(copied from the incumbent once) and loops: run the lane's ``agent`` command, +submit the work tree, serve the ticket until it is no longer queued. + +The agent command runs on the host with the caller's environment; it is not +confined. Placeholders: ``{work}`` is the work tree, ``{lane}`` the lane id and +``{results}`` a JSON file listing this lane's earlier receipts. + +A worker stops when the agent exits non-zero, submit refuses (``unchanged`` means +the agent has nothing more to try), a ticket repeats, or serve returns anything +other than ``measured``. A lane that reaches its cap stops on the rejected receipt. +""" + +import shutil +import subprocess +import threading +import time +from datetime import datetime, timezone +from pathlib import Path + +from inferrsi.harness.parallel_close import close_round +from inferrsi.harness.parallel_round import RoundPlanError, prepare, serve_ticket, submit +from inferrsi.harness.round_budget import BudgetError, seconds_remaining +from inferrsi.infra.artifacts import write_json + + +def drive_round( + spec: dict, + incumbent: Path, + round_dir: Path, + output: Path, + *, + backend, + poll_seconds: float = 5.0, + clock=None, +) -> dict: + """Prepare, run every lane until it stops, close, and write ``drive.json``.""" + plan = prepare(spec, incumbent, round_dir) + missing = [lane["id"] for lane in plan["lanes"] if not lane.get("agent")] + if missing: + raise RoundPlanError(f"lanes without an agent command: {', '.join(missing)}") + + def current() -> datetime: + if clock is not None: + return clock() + return datetime.now(timezone.utc) + + outcomes: dict[str, dict] = {} + + def work(lane: dict) -> None: + try: + outcomes[lane["id"]] = _drive_lane( + round_dir, incumbent, lane, backend, poll_seconds, current + ) + except Exception as exc: # noqa: BLE001 - one lane's crash must not hide the others + outcomes[lane["id"]] = {"stop": "error", "reason": repr(exc), "tickets": []} + + workers = [threading.Thread(target=work, args=(lane,)) for lane in plan["lanes"]] + for worker in workers: + worker.start() + for worker in workers: + worker.join() + closed = close_round(round_dir, incumbent, output, backend=backend, clock=clock) + body = { + "round_digest": plan["round_digest"], + "lanes": {lane_id: outcomes[lane_id] for lane_id in sorted(outcomes)}, + "close": closed, + } + write_json(round_dir / "drive.json", body) + return body + + +def _drive_lane(round_dir, incumbent, lane, backend, poll_seconds, current) -> dict: + lane_id = lane["id"] + work = round_dir / "work" / lane_id + if not work.exists(): + shutil.copytree(incumbent, work) + history_path = round_dir / "history" / f"{lane_id}.json" + history: list[dict] = [] + while True: + write_json(history_path, history) + try: + remaining = seconds_remaining(lane["deadline_utc"], current()) + except BudgetError as exc: + return _stop("deadline", str(exc), history) + argv = [ + part.replace("{work}", str(work)) + .replace("{lane}", lane_id) + .replace("{results}", str(history_path)) + for part in lane["agent"] + ] + try: + finished = subprocess.run(argv, cwd=work, timeout=remaining, check=False) + except subprocess.TimeoutExpired: + return _stop("deadline", "agent ran past the lane deadline", history) + except OSError as exc: + return _stop("agent_failed", str(exc), history) + if finished.returncode != 0: + return _stop("agent_failed", f"agent exited {finished.returncode}", history) + submitted = submit(round_dir, lane_id, incumbent, work, now=current()) + if not submitted["submitted"]: + stop = "done" if submitted["reason"] == "unchanged" else "refused" + return _stop(stop, submitted["reason"], history) + ticket = submitted["ticket"] + if any(item["ticket"] == ticket for item in history): + return _stop("done", "agent resubmitted an earlier change", history) + result = serve_ticket(round_dir, ticket, incumbent, backend=backend, clock=current) + while result["status"] == "queued": + time.sleep(poll_seconds) + result = serve_ticket(round_dir, ticket, incumbent, backend=backend, clock=current) + history.append(result) + if result["status"] != "measured": + write_json(history_path, history) + return _stop(result["status"], result.get("reason", ""), history) + + +def _stop(stop: str, reason: str, history: list[dict]) -> dict: + return {"stop": stop, "reason": reason, "tickets": [item["ticket"] for item in history]} diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py index 8f2fd5924..37c365b5a 100644 --- a/src/inferrsi/harness/parallel_round.py +++ b/src/inferrsi/harness/parallel_round.py @@ -1,16 +1,22 @@ """Prepare, submit and measure one offline parallel round. -Directory diffs stand in for git clones. ``close`` and incumbent advancement are -not implemented. A synthetic metric value is not a campaign verdict. +Directory diffs stand in for git clones. ``parallel_close`` picks the next +incumbent and ``parallel_drive`` runs a whole round. A synthetic metric value is +not a campaign verdict. Each lane declares ``device_count`` (default 1). The counts of all lanes must fit in ``device_ids``. A ticket runs only after it holds that many devices at once. Command placeholders: ``build`` gets ``{device}`` only. ``trial`` gets ``{device}`` -and ``{result}``; it runs once for the candidate and once for the incumbent and -must write ``{"metrics": {: }}`` to ``{result}``. -``{device}`` is the held device identities joined with commas. Other text, -including ``{result}`` inside ``build``, is passed through unchanged. +and ``{result}``; it must write ``{"metrics": {: }}`` +to ``{result}``. The candidate is built once, then candidate and incumbent trials +run ``metric.repeats`` times each in ABBA order; the receipt keeps every sample +and the medians. ``{device}`` is the held device identities joined with commas. +Other text, including ``{result}`` inside ``build``, is passed through unchanged. + +``device_paths`` (per device id), ``shared_device_paths``, ``runtime_paths`` and +``env`` (values may use ``{device}``) extend the confined view for GPU trials. +Each command is capped by ``command_timeout_seconds`` and the lane deadline. """ import hashlib @@ -19,6 +25,7 @@ import os import shutil import stat +import statistics import subprocess import time from datetime import datetime, timezone @@ -58,6 +65,16 @@ def _segment(value) -> bool: ) +def _host_paths(value) -> bool: + return isinstance(value, list) and all( + isinstance(item, str) + and Path(item).is_absolute() + and item != "/" + and ".." not in Path(item).parts + for item in value + ) + + def _match(path: str, patterns: list[str]) -> bool: return any(fnmatch(path, pattern) for pattern in patterns) @@ -188,13 +205,38 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: metric = spec.get("metric") if ( not isinstance(metric, dict) - or set(metric) != {"name", "better"} + or not {"name", "better"} <= set(metric) <= {"name", "better", "repeats", "margin"} or not isinstance(metric["name"], str) or not metric["name"] or not isinstance(metric["better"], str) or metric["better"] not in DIRECTIONS ): raise RoundPlanError("metric needs a name and better: lower or higher") + repeats = metric.get("repeats", 1) + if type(repeats) is not int or repeats < 1: + raise RoundPlanError("metric repeats must be a positive integer") + margin = metric.get("margin", 0) + if type(margin) not in (int, float) or not math.isfinite(margin) or margin < 0: + raise RoundPlanError("metric margin must be a finite non-negative number") + timeout = spec.get("command_timeout_seconds", 30) + if type(timeout) not in (int, float) or not math.isfinite(timeout) or timeout <= 0: + raise RoundPlanError("command_timeout_seconds must be a positive number") + device_paths = spec.get("device_paths", {}) + if not isinstance(device_paths, dict) or not all( + _host_paths(paths) for paths in device_paths.values() + ): + raise RoundPlanError("device_paths must map device ids to absolute host paths") + if device_paths and set(device_paths) != set(devices): + raise RoundPlanError("device_paths must list every device id and nothing else") + shared_paths = spec.get("shared_device_paths", []) + runtime_paths = spec.get("runtime_paths", []) + if not _host_paths(shared_paths) or not _host_paths(runtime_paths): + raise RoundPlanError("shared_device_paths and runtime_paths must be absolute host paths") + env = spec.get("env", {}) + if not isinstance(env, dict) or not all( + isinstance(key, str) and key and isinstance(value, str) for key, value in env.items() + ): + raise RoundPlanError("env must map names to strings") seen_entries: set[str] = set() seen_ids: set[str] = set() files = _files(incumbent) @@ -221,6 +263,9 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: count = lane.get("device_count", 1) if type(count) is not int or count < 1: raise RoundPlanError(f"{lane_id} device_count must be a positive integer") + agent = lane.get("agent", []) + if not isinstance(agent, list) or any(type(item) is not str for item in agent): + raise RoundPlanError(f"{lane_id} agent must be a list of strings") if any(_match(path, globs) and _match(path, protected) for path in files): raise RoundPlanError(f"{lane_id} reaches a protected path") seen_ids.add(lane_id) @@ -244,7 +289,17 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: body = { "round_id": spec["round_id"], "device_ids": list(devices), - "metric": dict(metric), + "metric": { + "name": metric["name"], + "better": metric["better"], + "repeats": repeats, + "margin": margin, + }, + "command_timeout_seconds": timeout, + "device_paths": {device: list(device_paths[device]) for device in device_paths}, + "shared_device_paths": list(shared_paths), + "runtime_paths": list(runtime_paths), + "env": dict(env), "incumbent_id": spec["incumbent_id"], "protected": protected, "build": list(spec.get("build", [])), @@ -362,15 +417,17 @@ def _save_ledger(round_dir: Path, ledger) -> None: ) -def _materialize(incumbent: Path, overlay: Path, destination: Path) -> None: +def materialize(incumbent: Path, overlays: list[Path], destination: Path) -> None: + """Copy the incumbent, then apply each outbox overlay in order.""" if destination.exists(): shutil.rmtree(destination) shutil.copytree(incumbent, destination, ignore=shutil.ignore_patterns("manifest.json")) - manifest = json.loads((overlay / "manifest.json").read_text(encoding="utf-8")) - for relative in manifest["files"]: - target = destination / relative - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(overlay / relative, target) + for overlay in overlays: + manifest = json.loads((overlay / "manifest.json").read_text(encoding="utf-8")) + for relative in manifest["files"]: + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(overlay / relative, target) def _command(template: list[str], result: Path | None, device: str) -> tuple[str, ...]: @@ -401,52 +458,143 @@ def _claimed_entry(ledger, ticket: str): return None -def _launch( +class MeasurementError(ValueError): + """A trial finished without a readable metric value.""" + + +def _run_confined( + plan: dict, + stage: str, + argv: tuple[str, ...], + cwd: Path, + writable: tuple[Path, ...], + readable: tuple[Path, ...], + devices: list[str], + deadline_utc: str | None, + *, + backend, + current, +) -> None: + remaining = plan["command_timeout_seconds"] + if deadline_utc is not None: + remaining = min(seconds_remaining(deadline_utc, current()), remaining) + device = ",".join(devices) + nodes = list(plan["shared_device_paths"]) + for held in devices: + nodes.extend(plan["device_paths"].get(held, [])) + try: + record = confined_execute( + argv, + cwd, + writable, + readable, + time.monotonic() + remaining, + backend=backend, + device_paths=tuple(Path(path) for path in nodes), + runtime_paths=tuple(Path(path) for path in plan["runtime_paths"]), + env={key: value.replace("{device}", device) for key, value in plan["env"].items()}, + ) + except subprocess.TimeoutExpired as exc: + raise ConfinementUnavailable(f"{stage} timed out") from exc + if record.get("status") != "completed": + raise ConfinementUnavailable(f"{stage} {record.get('status')}") + + +def compare( plan: dict, - lane: dict, candidate: Path, incumbent: Path, trial_dir: Path, - device: str, + devices: list[str], + deadline_utc: str | None, *, backend, current, -): - """Run build and both trials on the held devices. Each start uses the same deadline.""" +) -> tuple[list[float], list[float]]: + """Build the candidate once, then run ``repeats`` trial pairs in ABBA order. + + Raises ``ConfinementUnavailable`` or ``BudgetError`` when a command cannot run, and + ``MeasurementError`` when a finished trial left no readable metric. + """ + device = ",".join(devices) build_dir = candidate / "build-output" build_dir.mkdir(parents=True, exist_ok=True) trial_dir.mkdir(parents=True, exist_ok=True) - - def run(stage: str, argv, cwd, writable, readable) -> None: - remaining = min(seconds_remaining(lane["deadline_utc"], current()), 30) - try: - record = confined_execute( - argv, cwd, writable, readable, time.monotonic() + remaining, backend=backend + common = {"backend": backend, "current": current} + if plan["build"]: + _run_confined( + plan, + "build", + _command(plan["build"], None, device), + candidate, + (candidate, build_dir), + (), + devices, + deadline_utc, + **common, + ) + sides = {"candidate": candidate, "incumbent": incumbent} + samples: dict[str, list[float]] = {"candidate": [], "incumbent": []} + name = plan["metric"]["name"] + for index in range(plan["metric"]["repeats"]): + order = ("candidate", "incumbent") if index % 2 == 0 else ("incumbent", "candidate") + for side in order: + result = trial_dir / f"{side}-{index}.json" + _run_confined( + plan, + side, + _command(plan["trial"], result, device), + sides[side], + (trial_dir,), + (sides[side],), + devices, + deadline_utc, + **common, ) - except subprocess.TimeoutExpired as exc: - raise ConfinementUnavailable(f"{stage} timed out") from exc - if record.get("status") != "completed": - raise ConfinementUnavailable(f"{stage} {record.get('status')}") + try: + samples[side].append(_metric_value(result, name)) + except (OSError, json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: + raise MeasurementError( + f"trial finished without a readable {name} result: {exc}" + ) from exc + return samples["candidate"], samples["incumbent"] + + +def summarize(plan: dict, candidate_samples: list[float], incumbent_samples: list[float]) -> dict: + """Medians and a gain where positive always means the candidate is better.""" + candidate_value = statistics.median(candidate_samples) + incumbent_value = statistics.median(incumbent_samples) + gain = incumbent_value - candidate_value + if plan["metric"]["better"] == "higher": + gain = -gain + return { + "metric": plan["metric"]["name"], + "better": plan["metric"]["better"], + "candidate_samples": candidate_samples, + "incumbent_samples": incumbent_samples, + "candidate": candidate_value, + "incumbent": incumbent_value, + "gain": gain, + "relative_gain": gain / abs(incumbent_value) if incumbent_value else None, + } - if plan["build"]: - run("build", _command(plan["build"], None, device), candidate, (candidate, build_dir), ()) - candidate_result = trial_dir / "candidate.json" - run( - "candidate", - _command(plan["trial"], candidate_result, device), - candidate, - (trial_dir,), - (candidate,), - ) - incumbent_result = trial_dir / "incumbent.json" - run( - "incumbent", - _command(plan["trial"], incumbent_result, device), - incumbent, - (trial_dir,), - (incumbent,), - ) - return candidate_result, incumbent_result + +def hold_devices(round_dir: Path, plan: dict, count: int) -> list[tuple[str, object]] | None: + """Hold ``count`` free devices at once, or none. The caller releases what it gets.""" + leases = DeviceLease(round_dir / "leases") + held: list[tuple[str, object]] = [] + for device in plan["device_ids"]: + if len(held) == count: + break + try: + held.append((device, leases.acquire(device))) + except LeaseError: + continue + if len(held) < count: + for _device, lease in held: + lease.release() + return None + return held def serve_ticket( @@ -465,6 +613,7 @@ def serve_ticket( admission and command reads ``clock`` or the current time again. """ del now + def current() -> datetime: if clock is not None: return clock() @@ -493,18 +642,10 @@ def current() -> datetime: status="unsettled", reason="claimed execution was not settled", ) - leases = DeviceLease(round_dir / "leases") - held: list[tuple[str, object]] = [] + held = hold_devices(round_dir, plan, lane["device_count"]) + if held is None: + return {"ticket": ticket, "status": "queued"} try: - for device in plan["device_ids"]: - if len(held) == lane["device_count"]: - break - try: - held.append((device, leases.acquire(device))) - except LeaseError: - continue - if len(held) < lane["device_count"]: - return {"ticket": ticket, "status": "queued"} return _measure( round_dir, plan, @@ -543,7 +684,7 @@ def _measure( _save_ledger(round_dir, ledger) candidate = round_dir / "candidates" / ticket trial_dir = round_dir / "trials" / ticket - _materialize(incumbent, overlay, candidate) + materialize(incumbent, [overlay], candidate) try: seconds_remaining(lane["deadline_utc"], current()) except BudgetError as exc: @@ -552,44 +693,26 @@ def _measure( ledger = advance(ledger, binding, "claimed") _save_ledger(round_dir, ledger) try: - candidate_result, incumbent_result = _launch( + candidate_samples, incumbent_samples = compare( plan, - lane, candidate, incumbent, trial_dir, - ",".join(devices), + devices, + lane["deadline_utc"], backend=backend, current=current, ) - except (ConfinementUnavailable, BudgetError) as exc: + except (ConfinementUnavailable, BudgetError, MeasurementError) as exc: return _record(round_dir, ticket, status="unsettled", reason=str(exc), devices=devices) - metric = plan["metric"] - try: - candidate_value = _metric_value(candidate_result, metric["name"]) - incumbent_value = _metric_value(incumbent_result, metric["name"]) - except (OSError, json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: - return _record( - round_dir, - ticket, - status="unsettled", - reason=f"trial finished without a readable {metric['name']} result: {exc}", - devices=devices, - ) finished = advance(ledger, binding, "finished", actual_cost=0) _save_ledger(round_dir, finished) - gain = incumbent_value - candidate_value - if metric["better"] == "higher": - gain = -gain return _record( round_dir, ticket, status="measured", - metric=metric["name"], - better=metric["better"], - candidate=candidate_value, - incumbent=incumbent_value, - gain=gain, + lane_id=lane["id"], + **summarize(plan, candidate_samples, incumbent_samples), devices=devices, confinement=getattr(backend, "name", "unknown"), ) diff --git a/src/inferrsi/infra/confinement.py b/src/inferrsi/infra/confinement.py index 38581dc38..9506f5bfa 100644 --- a/src/inferrsi/infra/confinement.py +++ b/src/inferrsi/infra/confinement.py @@ -6,6 +6,12 @@ through. If that view cannot be built, execution is refused. ``FixtureBackend`` is a test double: it enforces the same view with an import hook and a minimal environment, which is not mount-namespace confinement. + +Callers may add device nodes (bound read-write, e.g. one GPU's ``/dev/nvidiaN``), +extra read-only runtime paths (e.g. the driver's user-space libraries) and extra +environment variables. With device nodes, the command also gets a new PID +namespace and a fresh ``/proc``; the host ``/proc`` is never bound. Every jail has +an empty ``/tmp`` and ``/dev/shm``. """ import json @@ -175,7 +181,11 @@ def _covered(path: Path, covered: list[Path]) -> bool: def namespace_plan( - writable: tuple[Path, ...], readable: tuple[Path, ...] + writable: tuple[Path, ...], + readable: tuple[Path, ...], + devices: tuple[Path, ...] = (), + runtime: tuple[Path, ...] = (), + env: dict | None = None, ) -> dict: """Mount list for a fresh root. The host root itself is not an entry.""" mounts: list[dict] = [] @@ -193,13 +203,17 @@ def add(path: Path, mode: str, *, directory: bool) -> None: for path in writable: add(path, "rw", directory=True) + for path in devices: + add(path, "dev", directory=False) for path in readable: add(path, "ro", directory=True) + for path in runtime: + add(path, "ro", directory=path.is_dir()) for root in runtime_roots(): add(root, "ro", directory=True) for extra in runtime_files(): add(extra, "ro", directory=False) - return {"mounts": mounts, "env": minimal_environment()} + return {"mounts": mounts, "env": minimal_environment(env), "proc": bool(devices)} def _within(path: Path, roots: tuple[Path, ...]) -> bool: @@ -260,8 +274,9 @@ def run( writable: tuple[Path, ...], readable: tuple[Path, ...], deadline: float, + **extras, ) -> dict: - del argv, cwd, writable, readable, deadline + del argv, cwd, writable, readable, deadline, extras raise ConfinementUnavailable("confinement backend unavailable") @@ -277,11 +292,15 @@ def run( writable: tuple[Path, ...], readable: tuple[Path, ...], deadline: float, + *, + devices: tuple[Path, ...] = (), + runtime: tuple[Path, ...] = (), + env: dict | None = None, ) -> dict: outside = _outside_writable(argv, writable) if outside is not None: raise ConfinementUnavailable(f"path outside writable set: {outside}") - visible = tuple(dict.fromkeys(writable + readable + runtime_roots())) + visible = tuple(dict.fromkeys(writable + devices + readable + runtime + runtime_roots())) try: guard_dir = tempfile.mkdtemp(prefix="inferrsi-guard-") Path(guard_dir, "sitecustomize.py").write_text(_GUARD, encoding="utf-8") @@ -289,6 +308,7 @@ def run( raise ConfinementUnavailable("confinement guard could not be installed") from exc environment = minimal_environment( { + **(env or {}), "PYTHONPATH": guard_dir, "INFERRSI_VISIBLE_ROOTS": os.pathsep.join(str(path) for path in visible), } @@ -308,10 +328,14 @@ def run( writable: tuple[Path, ...], readable: tuple[Path, ...], deadline: float, + *, + devices: tuple[Path, ...] = (), + runtime: tuple[Path, ...] = (), + env: dict | None = None, ) -> dict: if not writable: raise ConfinementUnavailable("refusing to run with an empty writable set") - plan = namespace_plan(writable, readable) + plan = namespace_plan(writable, readable, devices, runtime, env) remaining = deadline - time.monotonic() if remaining <= 0: raise ConfinementUnavailable("execution deadline exhausted before start") @@ -335,6 +359,7 @@ def run( "--mount", "--propagation", "private", + *(["--pid", "--fork"] if plan["proc"] else []), "--", sys.executable, "-c", @@ -393,6 +418,16 @@ def mount(args: list[str]) -> None: mount(["--bind", source, str(target)]) if item["mode"] == "ro": mount(["-o", "remount,bind,ro", str(target)]) + bound = {item["source"] for item in config["mounts"]} + for scratch in ("/tmp", "/dev/shm"): + if scratch in bound: + continue + target = jail.joinpath(*Path(scratch).parts[1:]) + target.mkdir(parents=True, exist_ok=True) + target.chmod(0o1777) + if config.get("proc"): + (jail / "proc").mkdir(exist_ok=True) + mount(["-t", "proc", "proc", str(jail / "proc")]) os.chroot(jail) os.chdir(config["cwd"]) argv = config["argv"] @@ -413,9 +448,21 @@ def confined_execute( deadline: float, *, backend=None, + device_paths: tuple[Path, ...] = (), + runtime_paths: tuple[Path, ...] = (), + env: dict | None = None, ) -> dict: """Execute argv only inside the selected boundary. Never falls back to a plain subprocess.""" chosen = select_backend() if backend is None else backend writable = tuple(Path(path).resolve() for path in writable_paths) readable = tuple(Path(path).resolve() for path in readable_paths) - return chosen.run(tuple(argv), Path(cwd), writable, readable, deadline) + return chosen.run( + tuple(argv), + Path(cwd), + writable, + readable, + deadline, + devices=tuple(Path(path) for path in device_paths), + runtime=tuple(Path(path).resolve() for path in runtime_paths), + env=env, + ) diff --git a/tests/test_confinement.py b/tests/test_confinement.py index 4bf61ed5b..361ae426b 100644 --- a/tests/test_confinement.py +++ b/tests/test_confinement.py @@ -1,6 +1,8 @@ """Confinement policy. The fixture backend is not an operating-system boundary.""" +import importlib.util import json +import os import subprocess import sys import time @@ -214,3 +216,112 @@ def test_build_and_trial_cannot_see_private_canaries(tmp_path, monkeypatch): assert "CANARY-ENV" not in record["stdout"] assert "CANARY-FILE" not in record["stdout"] assert "FILE-HIDDEN" in record["stdout"] + + +def test_namespace_plan_binds_device_nodes_and_asks_for_a_fresh_proc(tmp_path): + runtime = tmp_path / "driver" + runtime.mkdir() + plan = namespace_plan( + (tmp_path / "work",), + (), + devices=(Path("/dev/null"),), + runtime=(runtime,), + env={"CUDA_VISIBLE_DEVICES": "0"}, + ) + modes = {item["source"]: item["mode"] for item in plan["mounts"]} + assert modes["/dev/null"] == "dev" + assert modes[str(runtime)] == "ro" + assert "/proc" not in modes + assert plan["proc"] is True + assert plan["env"]["CUDA_VISIBLE_DEVICES"] == "0" + assert namespace_plan((tmp_path / "work",), ())["proc"] is False + + +@pytest.mark.skipif(not namespaces_available(), reason="user namespaces are unavailable") +def test_unshare_binds_a_device_node_with_a_fresh_proc_and_scratch_dirs(tmp_path): + work = tmp_path / "work" + work.mkdir() + script = ( + "import os, stat\n" + "open('/dev/null', 'w').write('x')\n" + "open('/tmp/x', 'w').write('x')\n" + "open('/dev/shm/x', 'w').write('x')\n" + "print('PID', os.getpid())\n" + "print('DEV', sorted(os.listdir('/dev')))\n" + "print('PROC', os.path.isdir('/proc/self'))\n" + "print('TMP', oct(stat.S_IMODE(os.stat('/tmp').st_mode)))\n" + "print('ENV', os.environ.get('GPU_PICK'))\n" + ) + record = UnshareBackend().run( + (sys.executable, "-c", script), + work, + (work,), + (), + time.monotonic() + 30, + devices=(Path("/dev/null"),), + env={"GPU_PICK": "7"}, + ) + assert record["status"] == "completed" + lines = record["stdout"].splitlines() + assert "PID 1" in lines + assert "DEV ['null', 'shm']" in lines + assert "PROC True" in lines + assert "TMP 0o1777" in lines + assert "ENV 7" in lines + + +def _gpu_ready() -> bool: + return ( + namespaces_available() + and Path("/dev/nvidiactl").exists() + and importlib.util.find_spec("torch") is not None + ) + + +@pytest.mark.skipif(not _gpu_ready(), reason="needs user namespaces, an NVIDIA GPU and torch") +def test_unshare_gpu_trial_sees_only_the_held_device(tmp_path): + device = os.environ.get("INFERRSI_GPU_TEST_DEVICE", "0") + nodes = tuple( + path + for path in ( + Path(f"/dev/nvidia{device}"), + Path("/dev/nvidiactl"), + Path("/dev/nvidia-uvm"), + Path("/dev/nvidia-uvm-tools"), + Path("/dev/zero"), + Path("/dev/urandom"), + ) + if path.exists() + ) + found = subprocess.run(["ldconfig", "-p"], capture_output=True, text=True, check=False) + libraries = sorted( + { + str(Path(line.split("=>")[-1].strip()).parent) + for line in found.stdout.splitlines() + if "libcuda.so" in line or "libnvidia-ml.so" in line + } + ) + work = tmp_path / "work" + work.mkdir() + script = ( + "import os, torch\n" + "x = torch.ones(256, 256, device='cuda')\n" + "print('COUNT', torch.cuda.device_count())\n" + "print('SUM', float((x @ x).sum()))\n" + "print('NODES', sorted(n for n in os.listdir('/dev') if n[6:].isdigit()))\n" + ) + record = UnshareBackend().run( + (sys.executable, "-c", script), + work, + (work,), + (), + time.monotonic() + 300, + devices=nodes, + runtime=tuple(Path(item) for item in libraries), + env={"HOME": "/tmp"}, + ) + assert record["status"] == "completed", record + lines = record["stdout"].splitlines() + assert "COUNT 1" in lines + assert f"SUM {256.0 * 256 * 256}" in lines + assert f"NODES ['nvidia{device}']" in lines diff --git a/tests/test_parallel_close.py b/tests/test_parallel_close.py new file mode 100644 index 000000000..0eede5649 --- /dev/null +++ b/tests/test_parallel_close.py @@ -0,0 +1,159 @@ +"""Closing a synthetic round. Metric figures here are fixture inputs, not measurements.""" + +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from inferrsi.harness.parallel_close import CloseError, close_round +from inferrsi.harness.parallel_round import prepare, serve_ticket, submit +from inferrsi.infra.confinement import FixtureBackend + +WHEN = datetime(2026, 10, 30, tzinfo=timezone.utc) +BENCH = ( + "import json, sys\n" + "from pathlib import Path\n" + "a = float(Path('knobs/a.txt').read_text())\n" + "b = float(Path('knobs/b.txt').read_text())\n" + "both = 100 if a != 10 and b != 10 else 0\n" + "value = a + b + (both if Path('clash').exists() else 0)\n" + "Path(sys.argv[1]).write_text(json.dumps({'metrics': {'score': value}}))\n" +) + + +def _tree(root: Path, *, clash: bool = False) -> None: + (root / "knobs").mkdir(parents=True) + (root / "knobs" / "a.txt").write_text("10", encoding="utf-8") + (root / "knobs" / "b.txt").write_text("10", encoding="utf-8") + (root / "bench.py").write_text(BENCH, encoding="utf-8") + if clash: + (root / "clash").write_text("", encoding="utf-8") + + +def _lane(lane_id: str, glob: str) -> dict: + return { + "id": lane_id, + "entry": f"D-{lane_id}", + "plane": "infra", + "globs": [glob], + "cap": 3, + "per_attempt_worst_case": 1, + "deadline_utc": "2099-01-01T00:00:00+00:00", + } + + +def _round(tmp_path, *, better="lower", margin=0, clash=False): + incumbent = tmp_path / "incumbent" + _tree(incumbent, clash=clash) + round_dir = tmp_path / "round" + spec = { + "round_id": "r1", + "device_ids": ["cpu-0", "cpu-1"], + "metric": {"name": "score", "better": better, "margin": margin}, + "incumbent_id": "fixture-incumbent", + "protected": ["bench.py", "clash"], + "build": [], + "trial": [sys.executable, "bench.py", "{result}"], + "lanes": [_lane("L1", "knobs/a.txt"), _lane("L2", "knobs/b.txt")], + } + prepare(spec, incumbent, round_dir) + return incumbent, round_dir + + +def _try(tmp_path, incumbent, round_dir, lane_id, knob, value) -> dict: + work = tmp_path / f"work-{lane_id}-{value}" + _tree(work, clash=(incumbent / "clash").exists()) + (work / "knobs" / knob).write_text(value, encoding="utf-8") + ticket = submit(round_dir, lane_id, incumbent, work, now=WHEN)["ticket"] + return serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + + +def _knobs(root: Path) -> tuple[str, str]: + return ( + (root / "knobs" / "a.txt").read_text(encoding="utf-8"), + (root / "knobs" / "b.txt").read_text(encoding="utf-8"), + ) + + +def test_no_passing_lane_keeps_the_incumbent(tmp_path): + incumbent, round_dir = _round(tmp_path) + assert _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "12")["status"] == "measured" + closed = close_round(round_dir, incumbent, tmp_path / "next", backend=FixtureBackend()) + assert closed["choice"] == "kept_incumbent" + assert closed["lanes"]["L1"]["passed"] is False + assert _knobs(tmp_path / "next") == ("10", "10") + + +def test_one_passing_lane_takes_its_best_ticket(tmp_path): + incumbent, round_dir = _round(tmp_path) + _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "9") + best = _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "7") + closed = close_round(round_dir, incumbent, tmp_path / "next", backend=FixtureBackend()) + assert closed["choice"] == "lane" + assert closed["chosen_tickets"] == [best["ticket"]] + assert closed["composite"] is None + assert _knobs(tmp_path / "next") == ("7", "10") + assert set(closed["files"]) == {"bench.py", "knobs/a.txt", "knobs/b.txt"} + + +def test_margin_filters_small_gains(tmp_path): + incumbent, round_dir = _round(tmp_path, margin=0.1) + _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "9") + closed = close_round(round_dir, incumbent, tmp_path / "next", backend=FixtureBackend()) + assert closed["choice"] == "kept_incumbent" + + +def test_two_passing_lanes_stack_when_the_composite_is_better(tmp_path): + incumbent, round_dir = _round(tmp_path) + _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "8") + _try(tmp_path, incumbent, round_dir, "L2", "b.txt", "7") + closed = close_round(round_dir, incumbent, tmp_path / "next", backend=FixtureBackend()) + assert closed["passing"] == ["L2", "L1"] + assert closed["composite"]["status"] == "measured" + assert closed["composite"]["candidate"] == 15 + assert closed["choice"] == "composite" + assert _knobs(tmp_path / "next") == ("8", "7") + + +def test_a_worse_composite_falls_back_to_the_best_single_lane(tmp_path): + incumbent, round_dir = _round(tmp_path, clash=True) + _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "8") + _try(tmp_path, incumbent, round_dir, "L2", "b.txt", "7") + closed = close_round(round_dir, incumbent, tmp_path / "next", backend=FixtureBackend()) + assert closed["composite"]["candidate"] == 115 + assert closed["choice"] == "lane" + assert _knobs(tmp_path / "next") == ("10", "7") + + +def test_higher_is_better_picks_the_larger_score(tmp_path): + incumbent, round_dir = _round(tmp_path, better="higher") + _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "8") + _try(tmp_path, incumbent, round_dir, "L2", "b.txt", "13") + closed = close_round(round_dir, incumbent, tmp_path / "next", backend=FixtureBackend()) + assert closed["passing"] == ["L2"] + assert _knobs(tmp_path / "next") == ("10", "13") + + +def test_close_is_idempotent_and_refuses_a_non_empty_output(tmp_path): + incumbent, round_dir = _round(tmp_path) + _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "8") + _try(tmp_path, incumbent, round_dir, "L2", "b.txt", "7") + first = close_round(round_dir, incumbent, tmp_path / "next", backend=FixtureBackend()) + + class Refuses: + name = "refuses" + + def run(self, *args, **kwargs): + raise AssertionError("close measured twice") + + again = close_round(round_dir, incumbent, tmp_path / "next", backend=Refuses()) + assert again == first + assert json.loads((round_dir / "close.json").read_text(encoding="utf-8")) == first + taken = tmp_path / "taken" + taken.mkdir() + (taken / "x").write_text("", encoding="utf-8") + (round_dir / "close.json").unlink() + with pytest.raises(CloseError, match="already has files"): + close_round(round_dir, incumbent, taken, backend=FixtureBackend()) diff --git a/tests/test_parallel_drive.py b/tests/test_parallel_drive.py new file mode 100644 index 000000000..0c5d4e75f --- /dev/null +++ b/tests/test_parallel_drive.py @@ -0,0 +1,151 @@ +"""Driving a synthetic round end to end. Metric figures here are fixture inputs.""" + +import json +import sys +from pathlib import Path + +import pytest + +from inferrsi.harness.parallel_drive import drive_round +from inferrsi.harness.parallel_round import RoundPlanError +from inferrsi.infra.confinement import FixtureBackend, UnshareBackend, namespaces_available + +BENCH = ( + "import json, sys\n" + "from pathlib import Path\n" + "a = float(Path('knobs/a.txt').read_text())\n" + "b = float(Path('knobs/b.txt').read_text())\n" + "Path(sys.argv[1]).write_text(json.dumps({'metrics': {'latency_ms': a + b}}))\n" +) +AGENT = ( + "import json, sys\n" + "from pathlib import Path\n" + "history = json.loads(Path(sys.argv[1]).read_text())\n" + "values = sys.argv[3].split(',')\n" + "if values == ['fail']:\n" + " sys.exit(3)\n" + "if len(history) < len(values):\n" + " Path(sys.argv[2]).write_text(values[len(history)])\n" +) + + +def _tree(root: Path) -> None: + (root / "knobs").mkdir(parents=True) + (root / "knobs" / "a.txt").write_text("10", encoding="utf-8") + (root / "knobs" / "b.txt").write_text("10", encoding="utf-8") + (root / "bench.py").write_text(BENCH, encoding="utf-8") + + +def _lane(lane_id: str, knob: str, values: str, tmp_path: Path) -> dict: + agent = tmp_path / "agent.py" + agent.write_text(AGENT, encoding="utf-8") + return { + "id": lane_id, + "entry": f"D-{lane_id}", + "plane": "infra", + "globs": [f"knobs/{knob}"], + "cap": 4, + "per_attempt_worst_case": 1, + "deadline_utc": "2099-01-01T00:00:00+00:00", + "agent": [sys.executable, str(agent), "{results}", f"knobs/{knob}", values], + } + + +def _spec(lanes: list[dict]) -> dict: + return { + "round_id": "r1", + "device_ids": ["cpu-0", "cpu-1"], + "metric": {"name": "latency_ms", "better": "lower", "repeats": 2}, + "incumbent_id": "fixture-incumbent", + "protected": ["bench.py"], + "build": [], + "trial": [sys.executable, "bench.py", "{result}"], + "lanes": lanes, + } + + +def _knobs(root: Path) -> tuple[str, str]: + return ( + (root / "knobs" / "a.txt").read_text(encoding="utf-8"), + (root / "knobs" / "b.txt").read_text(encoding="utf-8"), + ) + + +def test_drive_runs_both_lanes_and_closes_on_the_stacked_change(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent) + round_dir = tmp_path / "round" + lanes = [_lane("L1", "a.txt", "9,8", tmp_path), _lane("L2", "b.txt", "7", tmp_path)] + driven = drive_round( + _spec(lanes), + incumbent, + round_dir, + tmp_path / "next", + backend=FixtureBackend(), + poll_seconds=0.01, + ) + assert driven["lanes"]["L1"]["stop"] == "done" + assert len(driven["lanes"]["L1"]["tickets"]) == 2 + assert driven["lanes"]["L2"]["stop"] == "done" + assert len(driven["lanes"]["L2"]["tickets"]) == 1 + assert driven["close"]["choice"] == "composite" + assert driven["close"]["composite"]["candidate"] == 15 + assert _knobs(tmp_path / "next") == ("8", "7") + assert _knobs(incumbent) == ("10", "10") + history = json.loads((round_dir / "history" / "L1.json").read_text(encoding="utf-8")) + assert [item["candidate"] for item in history] == [19, 18] + assert json.loads((round_dir / "drive.json").read_text(encoding="utf-8")) == driven + + +def test_a_failing_agent_stops_only_its_lane(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent) + lanes = [_lane("L1", "a.txt", "8", tmp_path), _lane("L2", "b.txt", "fail", tmp_path)] + driven = drive_round( + _spec(lanes), + incumbent, + tmp_path / "round", + tmp_path / "next", + backend=FixtureBackend(), + poll_seconds=0.01, + ) + assert driven["lanes"]["L2"] == { + "stop": "agent_failed", + "reason": "agent exited 3", + "tickets": [], + } + assert driven["close"]["choice"] == "lane" + assert _knobs(tmp_path / "next") == ("8", "10") + + +def test_drive_needs_an_agent_command_on_every_lane(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent) + lane = _lane("L1", "a.txt", "8", tmp_path) + del lane["agent"] + with pytest.raises(RoundPlanError, match="without an agent"): + drive_round( + _spec([lane]), + incumbent, + tmp_path / "round", + tmp_path / "next", + backend=FixtureBackend(), + ) + + +@pytest.mark.skipif(not namespaces_available(), reason="user namespaces are unavailable") +def test_drive_under_real_namespaces(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent) + lanes = [_lane("L1", "a.txt", "8", tmp_path), _lane("L2", "b.txt", "7", tmp_path)] + driven = drive_round( + _spec(lanes), + incumbent, + tmp_path / "round", + tmp_path / "next", + backend=UnshareBackend(), + poll_seconds=0.01, + ) + assert driven["close"]["choice"] == "composite" + assert driven["close"]["composite"]["candidate"] == 15 + assert _knobs(tmp_path / "next") == ("8", "7") diff --git a/tests/test_parallel_round.py b/tests/test_parallel_round.py index 7f5c9cac9..4bf46c00f 100644 --- a/tests/test_parallel_round.py +++ b/tests/test_parallel_round.py @@ -256,6 +256,119 @@ def test_a_lane_needing_two_devices_waits_until_it_can_hold_both(tmp_path): assert stamp.read_text(encoding="utf-8") == "1,2" +def test_prepare_rejects_bad_repeats_margin_and_timeout(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + base = {"name": "latency_ms", "better": "lower"} + for extra, message in ( + ({"repeats": 0}, "repeats"), + ({"repeats": True}, "repeats"), + ({"margin": -0.1}, "margin"), + ({"margin": float("nan")}, "margin"), + ({"spread": 1}, "metric needs"), + ): + with pytest.raises(RoundPlanError, match=message): + prepare(_spec(metric={**base, **extra}), incumbent, tmp_path / "round-metric") + for timeout in (0, -1, "30", float("inf")): + with pytest.raises(RoundPlanError, match="command_timeout_seconds"): + prepare(_spec(command_timeout_seconds=timeout), incumbent, tmp_path / "round-t") + plan = prepare(_spec(), incumbent, tmp_path / "round-defaults") + assert plan["metric"] == {**base, "repeats": 1, "margin": 0} + assert plan["command_timeout_seconds"] == 30 + + +def test_prepare_rejects_bad_device_paths_env_and_agent(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + for overrides, message in ( + ({"device_paths": {"cpu-0": ["/dev/a"]}}, "every device id"), + ({"device_paths": {"cpu-0": ["dev/a"], "cpu-1": []}}, "absolute host paths"), + ({"device_paths": {"cpu-0": ["/"], "cpu-1": []}}, "absolute host paths"), + ({"shared_device_paths": ["/dev/../etc"]}, "absolute host paths"), + ({"runtime_paths": "/usr/lib64"}, "absolute host paths"), + ({"env": {"CUDA_VISIBLE_DEVICES": 0}}, "env"), + ): + with pytest.raises(RoundPlanError, match=message): + prepare(_spec(**overrides), incumbent, tmp_path / "round-paths") + lanes = _spec()["lanes"] + lanes[0]["agent"] = "claude -p" + with pytest.raises(RoundPlanError, match="agent must be a list"): + prepare(_spec(lanes=lanes), incumbent, tmp_path / "round-agent") + + +def test_serve_passes_only_the_held_device_nodes_and_env(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare( + _spec( + build=[], + device_paths={"cpu-0": ["/dev/fake0"], "cpu-1": ["/dev/fake1"]}, + shared_device_paths=["/dev/fakectl"], + runtime_paths=["/opt/driver"], + env={"CUDA_VISIBLE_DEVICES": "{device}"}, + ), + incumbent, + round_dir, + ) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + seen = [] + + class Recording: + name = "recording" + + def run(self, argv, *args, **kwargs): + del args + seen.append(kwargs) + body = {"metrics": {"latency_ms": 8 if "candidate" in argv[-1] else 10}} + Path(argv[-1]).write_text(json.dumps(body), encoding="utf-8") + return {"status": "completed", "stdout": "", "stderr": ""} + + with DeviceLease(round_dir / "leases").acquire("cpu-0"): + measured = serve_ticket(round_dir, ticket, incumbent, backend=Recording(), now=WHEN) + assert measured["devices"] == ["cpu-1"] + assert seen[0]["devices"] == (Path("/dev/fakectl"), Path("/dev/fake1")) + assert seen[0]["runtime"] == (Path("/opt/driver"),) + assert seen[0]["env"] == {"CUDA_VISIBLE_DEVICES": "cpu-1"} + + +def test_repeats_alternate_abba_and_report_medians(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + metric = {"name": "latency_ms", "better": "lower", "repeats": 3} + prepare(_spec(metric=metric, build=[]), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + values = {"candidate": [8, 100, 9], "incumbent": [10, 10, 1000]} + order = [] + + class Recording: + name = "recording" + + def run(self, argv, *args, **kwargs): + del args, kwargs + result = Path(argv[-1]) + order.append(result.stem) + side = result.stem.split("-")[0] + body = {"metrics": {"latency_ms": values[side].pop(0)}} + result.write_text(json.dumps(body), encoding="utf-8") + return {"status": "completed", "stdout": "", "stderr": ""} + + measured = serve_ticket(round_dir, ticket, incumbent, backend=Recording(), now=WHEN) + assert order == [ + "candidate-0", "incumbent-0", "incumbent-1", "candidate-1", "candidate-2", "incumbent-2" + ] + assert measured["candidate_samples"] == [8, 100, 9] + assert measured["candidate"] == 9 + assert measured["incumbent"] == 10 + assert measured["gain"] == 1 + assert measured["relative_gain"] == pytest.approx(0.1) + + def test_higher_is_better_metric_reports_gain_in_the_better_direction(tmp_path): incumbent = tmp_path / "incumbent" _tree(incumbent, "10") diff --git a/tools/parallel_round.py b/tools/parallel_round.py index 40ddaff59..a62c39603 100644 --- a/tools/parallel_round.py +++ b/tools/parallel_round.py @@ -6,6 +6,8 @@ from datetime import datetime, timezone from pathlib import Path +from inferrsi.harness.parallel_close import close_round +from inferrsi.harness.parallel_drive import drive_round from inferrsi.harness.parallel_round import prepare, serve_ticket, submit, ticket_status from inferrsi.infra.confinement import select_backend @@ -38,6 +40,18 @@ def main(argv: list[str] | None = None) -> int: status_command.add_argument("--round-dir", required=True, type=Path) status_command.add_argument("--ticket", required=True) + drive_command = commands.add_parser("drive") + drive_command.add_argument("--spec", required=True) + drive_command.add_argument("--incumbent", required=True, type=Path) + drive_command.add_argument("--round-dir", required=True, type=Path) + drive_command.add_argument("--output", required=True, type=Path) + drive_command.add_argument("--poll-seconds", type=float, default=5.0) + + close_command = commands.add_parser("close") + close_command.add_argument("--round-dir", required=True, type=Path) + close_command.add_argument("--incumbent", required=True, type=Path) + close_command.add_argument("--output", required=True, type=Path) + args = parser.parse_args(argv) if args.command == "prepare": result = prepare(_load(args.spec), args.incumbent, args.round_dir) @@ -56,6 +70,17 @@ def main(argv: list[str] | None = None) -> int: args.incumbent, backend=select_backend(), ) + elif args.command == "drive": + result = drive_round( + _load(args.spec), + args.incumbent, + args.round_dir, + args.output, + backend=select_backend(), + poll_seconds=args.poll_seconds, + ) + elif args.command == "close": + result = close_round(args.round_dir, args.incumbent, args.output, backend=select_backend()) else: result = ticket_status(args.round_dir, args.ticket) json.dump(result, sys.stdout, indent=2) From d6bcbbc568a9e3a57aaa7e0252b12914cdf2846d Mon Sep 17 00:00:00 2001 From: NumberWan Date: Thu, 12 Nov 2026 16:05:05 +0800 Subject: [PATCH 09/23] fix(infra): mount runtime paths where the loader looks for them ld.so.cache records driver libraries under the path ldconfig saw, such as /lib/x86_64-linux-gnu on Ubuntu where /lib is a symlink. Resolving runtime_paths moved the mount to /usr/lib and left those cache entries missing inside the jail, so runtime paths are now bound as written, like the Python loader files. Signed-off-by: NumberWan --- src/inferrsi/infra/confinement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inferrsi/infra/confinement.py b/src/inferrsi/infra/confinement.py index 9506f5bfa..329797ddb 100644 --- a/src/inferrsi/infra/confinement.py +++ b/src/inferrsi/infra/confinement.py @@ -463,6 +463,6 @@ def confined_execute( readable, deadline, devices=tuple(Path(path) for path in device_paths), - runtime=tuple(Path(path).resolve() for path in runtime_paths), + runtime=tuple(Path(path) for path in runtime_paths), env=env, ) From cdaa54d3c564e26562e54c7ada68ae69550cdbfb Mon Sep 17 00:00:00 2001 From: NumberWan Date: Thu, 12 Nov 2026 16:11:58 +0800 Subject: [PATCH 10/23] feat(harness): drive hands in one candidate per lane per round A round without a cluster has one candidate; a retry is the next round, which starts again from diagnosis. drive now matches that per lane: each lane runs its agent command once, submits, and is served once, all lanes in parallel, then close picks the next incumbent. The in-round retry loop and the {results} placeholder are gone. Each lane reports a status (measured, rejected, unsettled, unchanged, refused, agent_failed, deadline or error), a reason and its ticket. Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_drive.py | 83 +++++++++++--------------- tests/test_parallel_drive.py | 59 ++++++++++++------ 2 files changed, 76 insertions(+), 66 deletions(-) diff --git a/src/inferrsi/harness/parallel_drive.py b/src/inferrsi/harness/parallel_drive.py index fe85b2e06..44ffd9d87 100644 --- a/src/inferrsi/harness/parallel_drive.py +++ b/src/inferrsi/harness/parallel_drive.py @@ -1,16 +1,17 @@ """Drive one parallel round from a single command, then close it. -Each lane gets one worker. A worker keeps a work tree at ``round_dir/work/`` -(copied from the incumbent once) and loops: run the lane's ``agent`` command, -submit the work tree, serve the ticket until it is no longer queued. +Like a round without a cluster, each lane hands in one candidate per round; a +retry belongs to the next round. Each lane gets one worker, all in parallel: copy +the incumbent to ``round_dir/work/``, run the lane's ``agent`` command +there once, submit the work tree, and serve the ticket until it is no longer +queued. When every lane is done, ``close`` picks the next incumbent. The agent command runs on the host with the caller's environment; it is not -confined. Placeholders: ``{work}`` is the work tree, ``{lane}`` the lane id and -``{results}`` a JSON file listing this lane's earlier receipts. +confined. Placeholders: ``{work}`` is the work tree and ``{lane}`` the lane id. -A worker stops when the agent exits non-zero, submit refuses (``unchanged`` means -the agent has nothing more to try), a ticket repeats, or serve returns anything -other than ``measured``. A lane that reaches its cap stops on the rejected receipt. +A lane's outcome is the serve status (``measured``, ``rejected``, ``unsettled``), +``unchanged`` when the agent changed nothing, ``refused`` when submit refused the +change, ``agent_failed``, ``deadline`` or ``error``. """ import shutil @@ -36,7 +37,7 @@ def drive_round( poll_seconds: float = 5.0, clock=None, ) -> dict: - """Prepare, run every lane until it stops, close, and write ``drive.json``.""" + """Prepare, run one attempt on every lane in parallel, close, and write ``drive.json``.""" plan = prepare(spec, incumbent, round_dir) missing = [lane["id"] for lane in plan["lanes"] if not lane.get("agent")] if missing: @@ -55,7 +56,7 @@ def work(lane: dict) -> None: round_dir, incumbent, lane, backend, poll_seconds, current ) except Exception as exc: # noqa: BLE001 - one lane's crash must not hide the others - outcomes[lane["id"]] = {"stop": "error", "reason": repr(exc), "tickets": []} + outcomes[lane["id"]] = _outcome("error", repr(exc)) workers = [threading.Thread(target=work, args=(lane,)) for lane in plan["lanes"]] for worker in workers: @@ -77,44 +78,30 @@ def _drive_lane(round_dir, incumbent, lane, backend, poll_seconds, current) -> d work = round_dir / "work" / lane_id if not work.exists(): shutil.copytree(incumbent, work) - history_path = round_dir / "history" / f"{lane_id}.json" - history: list[dict] = [] - while True: - write_json(history_path, history) - try: - remaining = seconds_remaining(lane["deadline_utc"], current()) - except BudgetError as exc: - return _stop("deadline", str(exc), history) - argv = [ - part.replace("{work}", str(work)) - .replace("{lane}", lane_id) - .replace("{results}", str(history_path)) - for part in lane["agent"] - ] - try: - finished = subprocess.run(argv, cwd=work, timeout=remaining, check=False) - except subprocess.TimeoutExpired: - return _stop("deadline", "agent ran past the lane deadline", history) - except OSError as exc: - return _stop("agent_failed", str(exc), history) - if finished.returncode != 0: - return _stop("agent_failed", f"agent exited {finished.returncode}", history) - submitted = submit(round_dir, lane_id, incumbent, work, now=current()) - if not submitted["submitted"]: - stop = "done" if submitted["reason"] == "unchanged" else "refused" - return _stop(stop, submitted["reason"], history) - ticket = submitted["ticket"] - if any(item["ticket"] == ticket for item in history): - return _stop("done", "agent resubmitted an earlier change", history) + try: + remaining = seconds_remaining(lane["deadline_utc"], current()) + except BudgetError as exc: + return _outcome("deadline", str(exc)) + argv = [part.replace("{work}", str(work)).replace("{lane}", lane_id) for part in lane["agent"]] + try: + finished = subprocess.run(argv, cwd=work, timeout=remaining, check=False) + except subprocess.TimeoutExpired: + return _outcome("deadline", "agent ran past the lane deadline") + except OSError as exc: + return _outcome("agent_failed", str(exc)) + if finished.returncode != 0: + return _outcome("agent_failed", f"agent exited {finished.returncode}") + submitted = submit(round_dir, lane_id, incumbent, work, now=current()) + if not submitted["submitted"]: + status = "unchanged" if submitted["reason"] == "unchanged" else "refused" + return _outcome(status, submitted["reason"]) + ticket = submitted["ticket"] + result = serve_ticket(round_dir, ticket, incumbent, backend=backend, clock=current) + while result["status"] == "queued": + time.sleep(poll_seconds) result = serve_ticket(round_dir, ticket, incumbent, backend=backend, clock=current) - while result["status"] == "queued": - time.sleep(poll_seconds) - result = serve_ticket(round_dir, ticket, incumbent, backend=backend, clock=current) - history.append(result) - if result["status"] != "measured": - write_json(history_path, history) - return _stop(result["status"], result.get("reason", ""), history) + return _outcome(result["status"], result.get("reason", ""), ticket) -def _stop(stop: str, reason: str, history: list[dict]) -> dict: - return {"stop": stop, "reason": reason, "tickets": [item["ticket"] for item in history]} +def _outcome(status: str, reason: str, ticket: str | None = None) -> dict: + return {"status": status, "reason": reason, "ticket": ticket} diff --git a/tests/test_parallel_drive.py b/tests/test_parallel_drive.py index 0c5d4e75f..1a2ce740c 100644 --- a/tests/test_parallel_drive.py +++ b/tests/test_parallel_drive.py @@ -18,14 +18,14 @@ "Path(sys.argv[1]).write_text(json.dumps({'metrics': {'latency_ms': a + b}}))\n" ) AGENT = ( - "import json, sys\n" + "import sys\n" "from pathlib import Path\n" - "history = json.loads(Path(sys.argv[1]).read_text())\n" - "values = sys.argv[3].split(',')\n" - "if values == ['fail']:\n" + "if sys.argv[2] == 'fail':\n" " sys.exit(3)\n" - "if len(history) < len(values):\n" - " Path(sys.argv[2]).write_text(values[len(history)])\n" + "if sys.argv[2] != 'same':\n" + " Path(sys.argv[1]).write_text(sys.argv[2])\n" + "Path(sys.argv[3]).mkdir(parents=True, exist_ok=True)\n" + "Path(sys.argv[3], 'ran').open('a').write('x')\n" ) @@ -36,7 +36,7 @@ def _tree(root: Path) -> None: (root / "bench.py").write_text(BENCH, encoding="utf-8") -def _lane(lane_id: str, knob: str, values: str, tmp_path: Path) -> dict: +def _lane(lane_id: str, knob: str, value: str, tmp_path: Path) -> dict: agent = tmp_path / "agent.py" agent.write_text(AGENT, encoding="utf-8") return { @@ -47,7 +47,13 @@ def _lane(lane_id: str, knob: str, values: str, tmp_path: Path) -> dict: "cap": 4, "per_attempt_worst_case": 1, "deadline_utc": "2099-01-01T00:00:00+00:00", - "agent": [sys.executable, str(agent), "{results}", f"knobs/{knob}", values], + "agent": [ + sys.executable, + str(agent), + f"knobs/{knob}", + value, + str(tmp_path / "calls" / "{lane}"), + ], } @@ -71,11 +77,11 @@ def _knobs(root: Path) -> tuple[str, str]: ) -def test_drive_runs_both_lanes_and_closes_on_the_stacked_change(tmp_path): +def test_drive_runs_each_lane_once_and_closes_on_the_stacked_change(tmp_path): incumbent = tmp_path / "incumbent" _tree(incumbent) round_dir = tmp_path / "round" - lanes = [_lane("L1", "a.txt", "9,8", tmp_path), _lane("L2", "b.txt", "7", tmp_path)] + lanes = [_lane("L1", "a.txt", "8", tmp_path), _lane("L2", "b.txt", "7", tmp_path)] driven = drive_round( _spec(lanes), incumbent, @@ -84,16 +90,17 @@ def test_drive_runs_both_lanes_and_closes_on_the_stacked_change(tmp_path): backend=FixtureBackend(), poll_seconds=0.01, ) - assert driven["lanes"]["L1"]["stop"] == "done" - assert len(driven["lanes"]["L1"]["tickets"]) == 2 - assert driven["lanes"]["L2"]["stop"] == "done" - assert len(driven["lanes"]["L2"]["tickets"]) == 1 + assert {lane: item["status"] for lane, item in driven["lanes"].items()} == { + "L1": "measured", + "L2": "measured", + } + for lane in ("L1", "L2"): + assert (tmp_path / "calls" / lane / "ran").read_text(encoding="utf-8") == "x" + assert len(list((round_dir / "results").glob("*.json"))) == 2 assert driven["close"]["choice"] == "composite" assert driven["close"]["composite"]["candidate"] == 15 assert _knobs(tmp_path / "next") == ("8", "7") assert _knobs(incumbent) == ("10", "10") - history = json.loads((round_dir / "history" / "L1.json").read_text(encoding="utf-8")) - assert [item["candidate"] for item in history] == [19, 18] assert json.loads((round_dir / "drive.json").read_text(encoding="utf-8")) == driven @@ -110,14 +117,30 @@ def test_a_failing_agent_stops_only_its_lane(tmp_path): poll_seconds=0.01, ) assert driven["lanes"]["L2"] == { - "stop": "agent_failed", + "status": "agent_failed", "reason": "agent exited 3", - "tickets": [], + "ticket": None, } assert driven["close"]["choice"] == "lane" assert _knobs(tmp_path / "next") == ("8", "10") +def test_an_agent_without_changes_leaves_the_incumbent(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent) + driven = drive_round( + _spec([_lane("L1", "a.txt", "same", tmp_path)]), + incumbent, + tmp_path / "round", + tmp_path / "next", + backend=FixtureBackend(), + poll_seconds=0.01, + ) + assert driven["lanes"]["L1"]["status"] == "unchanged" + assert driven["close"]["choice"] == "kept_incumbent" + assert _knobs(tmp_path / "next") == ("10", "10") + + def test_drive_needs_an_agent_command_on_every_lane(tmp_path): incumbent = tmp_path / "incumbent" _tree(incumbent) From 10ab9a3557da76da46373f4a729e55a7c5cf3d01 Mon Sep 17 00:00:00 2001 From: NumberWan Date: Thu, 12 Nov 2026 16:18:11 +0800 Subject: [PATCH 11/23] feat(harness): run cluster rounds up to the campaign round_cap campaign reads budget.round_cap from row 6 of the campaign plan (campaign-plan.yaml) and runs up to that many rounds. Each round the planner command, the decision agent, writes the round spec for drive; the next incumbent of one round is the incumbent of the next. The planner ends the campaign early by exiting 0 without writing a spec, for example when the target is met or no direction is left. Results go to campaign.json. PyYAML is imported only when the campaign plan is read, so the base install stays free of it. Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_campaign.py | 114 ++++++++++++++++++++++ tests/test_parallel_campaign.py | 114 ++++++++++++++++++++++ tools/parallel_round.py | 17 ++++ 3 files changed, 245 insertions(+) create mode 100644 src/inferrsi/harness/parallel_campaign.py create mode 100644 tests/test_parallel_campaign.py diff --git a/src/inferrsi/harness/parallel_campaign.py b/src/inferrsi/harness/parallel_campaign.py new file mode 100644 index 000000000..5e3e537a2 --- /dev/null +++ b/src/inferrsi/harness/parallel_campaign.py @@ -0,0 +1,114 @@ +"""Run cluster rounds until the campaign plan's ``round_cap`` or the planner stops. + +``round_cap`` is read from row 6 of the campaign plan (``budget.round_cap`` in +``campaign-plan.yaml``). Each round, the ``planner`` command (the decision agent) +writes the round spec for ``drive``; the next incumbent of one round is the +incumbent of the next. The planner stops the campaign by exiting 0 without +writing a spec, for example when the target is met or no direction is left. + +The planner runs on the host with the caller's environment; it is not confined. +Placeholders: ``{incumbent}`` the current incumbent, ``{round}`` the round number +from 1, ``{previous}`` the previous round's ``drive.json`` (empty in round 1) and +``{spec}`` where the round spec must be written. +""" + +import json +import subprocess +from pathlib import Path + +from inferrsi.harness.parallel_drive import drive_round +from inferrsi.infra.artifacts import write_json + + +class CampaignError(RuntimeError): + """The campaign cannot start as asked.""" + + +def round_cap(campaign_plan: Path) -> int: + """``budget.round_cap`` as a positive whole number.""" + try: + import yaml + except ImportError as exc: # The base install stays PyYAML-free by contract. + raise CampaignError( + "Reading the campaign plan requires the 'skills' extra (PyYAML): " + "python -m pip install '[skills]'" + ) from exc + plan = yaml.safe_load(campaign_plan.read_text(encoding="utf-8")) + value = (plan.get("budget") or {}).get("round_cap") if isinstance(plan, dict) else None + text = str(value) if type(value) in (int, str) else "" + if not text.isdigit() or int(text) < 1: + raise CampaignError(f"budget.round_cap must be a positive whole number, got {value!r}") + return int(text) + + +def run_campaign( + campaign_plan: Path, + planner: list[str], + incumbent: Path, + work_dir: Path, + *, + backend, + poll_seconds: float = 5.0, + clock=None, +) -> dict: + """Run up to ``round_cap`` rounds and write ``campaign.json``.""" + cap = round_cap(campaign_plan) + if not planner: + raise CampaignError("campaign needs a planner command") + if work_dir.exists() and any(work_dir.iterdir()): + raise CampaignError(f"work dir {work_dir} already has files") + work_dir.mkdir(parents=True, exist_ok=True) + current = incumbent + previous = "" + rounds: list[dict] = [] + stop = {"stop": "round_cap", "reason": f"ran {cap} rounds"} + for number in range(1, cap + 1): + round_dir = work_dir / f"round-{number}" + round_dir.mkdir() + spec_path = round_dir / "spec.json" + values = { + "{incumbent}": str(current), + "{round}": str(number), + "{previous}": previous, + "{spec}": str(spec_path), + } + argv = [] + for part in planner: + for key, value in values.items(): + part = part.replace(key, value) + argv.append(part) + try: + finished = subprocess.run(argv, cwd=work_dir, check=False) + except OSError as exc: + stop = {"stop": "planner_failed", "reason": str(exc)} + break + if finished.returncode != 0: + stop = {"stop": "planner_failed", "reason": f"planner exited {finished.returncode}"} + break + if not spec_path.exists(): + stop = {"stop": "planner_stopped", "reason": f"no round spec for round {number}"} + break + spec = json.loads(spec_path.read_text(encoding="utf-8")) + output = round_dir / "incumbent-next" + driven = drive_round( + spec, + current, + round_dir / "round", + output, + backend=backend, + poll_seconds=poll_seconds, + clock=clock, + ) + rounds.append( + { + "round": number, + "choice": driven["close"]["choice"], + "chosen_tickets": driven["close"]["chosen_tickets"], + "drive": str(round_dir / "round" / "drive.json"), + } + ) + current = output + previous = str(round_dir / "round" / "drive.json") + body = {"round_cap": cap, "rounds": rounds, **stop, "incumbent": str(current)} + write_json(work_dir / "campaign.json", body) + return body diff --git a/tests/test_parallel_campaign.py b/tests/test_parallel_campaign.py new file mode 100644 index 000000000..ab7312781 --- /dev/null +++ b/tests/test_parallel_campaign.py @@ -0,0 +1,114 @@ +"""Running synthetic cluster rounds under a campaign plan's round cap.""" + +import json +import sys +from pathlib import Path + +import pytest + +from inferrsi.harness.parallel_campaign import CampaignError, round_cap, run_campaign +from inferrsi.infra.confinement import FixtureBackend + +pytest.importorskip("yaml") + +BENCH = ( + "import json, sys\n" + "from pathlib import Path\n" + "value = float(Path('knob.txt').read_text())\n" + "Path(sys.argv[1]).write_text(json.dumps({'metrics': {'latency_ms': value}}))\n" +) +AGENT = ( + "from pathlib import Path\n" + "knob = Path('knob.txt')\n" + "knob.write_text(str(int(knob.read_text()) - 1))\n" +) +PLANNER = ( + "import json, sys\n" + "from pathlib import Path\n" + "template, spec, number, last = sys.argv[1:5]\n" + "if int(number) > int(last):\n" + " sys.exit(0)\n" + "body = json.loads(Path(template).read_text())\n" + "body['round_id'] = 'r' + number\n" + "Path(spec).write_text(json.dumps(body))\n" +) + + +def _setup(tmp_path: Path, cap: str, last_round: int) -> tuple[Path, list[str], Path]: + incumbent = tmp_path / "incumbent" + incumbent.mkdir() + (incumbent / "knob.txt").write_text("10", encoding="utf-8") + (incumbent / "bench.py").write_text(BENCH, encoding="utf-8") + agent = tmp_path / "agent.py" + agent.write_text(AGENT, encoding="utf-8") + template = tmp_path / "template.json" + template.write_text( + json.dumps( + { + "round_id": "r", + "device_ids": ["cpu-0"], + "metric": {"name": "latency_ms", "better": "lower"}, + "incumbent_id": "fixture", + "protected": ["bench.py"], + "build": [], + "trial": [sys.executable, "bench.py", "{result}"], + "lanes": [ + { + "id": "L1", + "entry": "D-01", + "plane": "infra", + "globs": ["knob.txt"], + "cap": 1, + "per_attempt_worst_case": 1, + "deadline_utc": "2099-01-01T00:00:00+00:00", + "agent": [sys.executable, str(agent)], + } + ], + } + ), + encoding="utf-8", + ) + planner = tmp_path / "planner.py" + planner.write_text(PLANNER, encoding="utf-8") + campaign_plan = tmp_path / "campaign-plan.yaml" + campaign_plan.write_text(f'budget:\n round_cap: "{cap}"\n', encoding="utf-8") + argv = [sys.executable, str(planner), str(template), "{spec}", "{round}", str(last_round)] + return campaign_plan, argv, incumbent + + +def test_campaign_runs_round_cap_rounds_and_carries_the_incumbent(tmp_path): + campaign_plan, planner, incumbent = _setup(tmp_path, "2", last_round=5) + work = tmp_path / "campaign" + result = run_campaign( + campaign_plan, planner, incumbent, work, backend=FixtureBackend(), poll_seconds=0.01 + ) + assert result["stop"] == "round_cap" + assert [item["choice"] for item in result["rounds"]] == ["lane", "lane"] + assert (Path(result["incumbent"]) / "knob.txt").read_text(encoding="utf-8") == "8" + assert (incumbent / "knob.txt").read_text(encoding="utf-8") == "10" + assert json.loads((work / "campaign.json").read_text(encoding="utf-8")) == result + + +def test_campaign_stops_when_the_planner_writes_no_spec(tmp_path): + campaign_plan, planner, incumbent = _setup(tmp_path, "5", last_round=1) + result = run_campaign( + campaign_plan, + planner, + incumbent, + tmp_path / "campaign", + backend=FixtureBackend(), + poll_seconds=0.01, + ) + assert result["stop"] == "planner_stopped" + assert len(result["rounds"]) == 1 + assert (Path(result["incumbent"]) / "knob.txt").read_text(encoding="utf-8") == "9" + + +def test_round_cap_must_be_a_positive_whole_number(tmp_path): + plan = tmp_path / "campaign-plan.yaml" + for value in ('""', '"0"', '"2.5"', "true", "[]"): + plan.write_text(f"budget:\n round_cap: {value}\n", encoding="utf-8") + with pytest.raises(CampaignError, match="round_cap"): + round_cap(plan) + plan.write_text("budget:\n round_cap: 3\n", encoding="utf-8") + assert round_cap(plan) == 3 diff --git a/tools/parallel_round.py b/tools/parallel_round.py index a62c39603..5e1c81078 100644 --- a/tools/parallel_round.py +++ b/tools/parallel_round.py @@ -6,6 +6,7 @@ from datetime import datetime, timezone from pathlib import Path +from inferrsi.harness.parallel_campaign import run_campaign from inferrsi.harness.parallel_close import close_round from inferrsi.harness.parallel_drive import drive_round from inferrsi.harness.parallel_round import prepare, serve_ticket, submit, ticket_status @@ -47,6 +48,13 @@ def main(argv: list[str] | None = None) -> int: drive_command.add_argument("--output", required=True, type=Path) drive_command.add_argument("--poll-seconds", type=float, default=5.0) + campaign_command = commands.add_parser("campaign") + campaign_command.add_argument("--campaign-plan", required=True, type=Path) + campaign_command.add_argument("--incumbent", required=True, type=Path) + campaign_command.add_argument("--work-dir", required=True, type=Path) + campaign_command.add_argument("--poll-seconds", type=float, default=5.0) + campaign_command.add_argument("--planner", required=True, nargs=argparse.REMAINDER) + close_command = commands.add_parser("close") close_command.add_argument("--round-dir", required=True, type=Path) close_command.add_argument("--incumbent", required=True, type=Path) @@ -79,6 +87,15 @@ def main(argv: list[str] | None = None) -> int: backend=select_backend(), poll_seconds=args.poll_seconds, ) + elif args.command == "campaign": + result = run_campaign( + args.campaign_plan, + args.planner, + args.incumbent, + args.work_dir, + backend=select_backend(), + poll_seconds=args.poll_seconds, + ) elif args.command == "close": result = close_round(args.round_dir, args.incumbent, args.output, backend=select_backend()) else: From fe0c191a581179b6e8b6e1bb3270cb7ae81e5da1 Mon Sep 17 00:00:00 2001 From: NumberWan Date: Fri, 13 Nov 2026 11:53:14 +0800 Subject: [PATCH 12/23] refactor(harness): keep parallel round to prepare/submit/serve/status Drop parallel_close, parallel_drive and parallel_campaign with their CLI subcommands and tests, plus the metric.margin and lane agent fields that only they read. The round now prepares, admits, measures and reports one ticket at a time; choosing the next incumbent and driving lanes are left to follow-up PRs. Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_campaign.py | 114 -------------- src/inferrsi/harness/parallel_close.py | 155 ------------------- src/inferrsi/harness/parallel_drive.py | 107 ------------- src/inferrsi/harness/parallel_round.py | 14 +- tests/test_parallel_campaign.py | 114 -------------- tests/test_parallel_close.py | 159 -------------------- tests/test_parallel_drive.py | 174 ---------------------- tests/test_parallel_round.py | 13 +- tools/parallel_round.py | 42 ------ 9 files changed, 7 insertions(+), 885 deletions(-) delete mode 100644 src/inferrsi/harness/parallel_campaign.py delete mode 100644 src/inferrsi/harness/parallel_close.py delete mode 100644 src/inferrsi/harness/parallel_drive.py delete mode 100644 tests/test_parallel_campaign.py delete mode 100644 tests/test_parallel_close.py delete mode 100644 tests/test_parallel_drive.py diff --git a/src/inferrsi/harness/parallel_campaign.py b/src/inferrsi/harness/parallel_campaign.py deleted file mode 100644 index 5e3e537a2..000000000 --- a/src/inferrsi/harness/parallel_campaign.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Run cluster rounds until the campaign plan's ``round_cap`` or the planner stops. - -``round_cap`` is read from row 6 of the campaign plan (``budget.round_cap`` in -``campaign-plan.yaml``). Each round, the ``planner`` command (the decision agent) -writes the round spec for ``drive``; the next incumbent of one round is the -incumbent of the next. The planner stops the campaign by exiting 0 without -writing a spec, for example when the target is met or no direction is left. - -The planner runs on the host with the caller's environment; it is not confined. -Placeholders: ``{incumbent}`` the current incumbent, ``{round}`` the round number -from 1, ``{previous}`` the previous round's ``drive.json`` (empty in round 1) and -``{spec}`` where the round spec must be written. -""" - -import json -import subprocess -from pathlib import Path - -from inferrsi.harness.parallel_drive import drive_round -from inferrsi.infra.artifacts import write_json - - -class CampaignError(RuntimeError): - """The campaign cannot start as asked.""" - - -def round_cap(campaign_plan: Path) -> int: - """``budget.round_cap`` as a positive whole number.""" - try: - import yaml - except ImportError as exc: # The base install stays PyYAML-free by contract. - raise CampaignError( - "Reading the campaign plan requires the 'skills' extra (PyYAML): " - "python -m pip install '[skills]'" - ) from exc - plan = yaml.safe_load(campaign_plan.read_text(encoding="utf-8")) - value = (plan.get("budget") or {}).get("round_cap") if isinstance(plan, dict) else None - text = str(value) if type(value) in (int, str) else "" - if not text.isdigit() or int(text) < 1: - raise CampaignError(f"budget.round_cap must be a positive whole number, got {value!r}") - return int(text) - - -def run_campaign( - campaign_plan: Path, - planner: list[str], - incumbent: Path, - work_dir: Path, - *, - backend, - poll_seconds: float = 5.0, - clock=None, -) -> dict: - """Run up to ``round_cap`` rounds and write ``campaign.json``.""" - cap = round_cap(campaign_plan) - if not planner: - raise CampaignError("campaign needs a planner command") - if work_dir.exists() and any(work_dir.iterdir()): - raise CampaignError(f"work dir {work_dir} already has files") - work_dir.mkdir(parents=True, exist_ok=True) - current = incumbent - previous = "" - rounds: list[dict] = [] - stop = {"stop": "round_cap", "reason": f"ran {cap} rounds"} - for number in range(1, cap + 1): - round_dir = work_dir / f"round-{number}" - round_dir.mkdir() - spec_path = round_dir / "spec.json" - values = { - "{incumbent}": str(current), - "{round}": str(number), - "{previous}": previous, - "{spec}": str(spec_path), - } - argv = [] - for part in planner: - for key, value in values.items(): - part = part.replace(key, value) - argv.append(part) - try: - finished = subprocess.run(argv, cwd=work_dir, check=False) - except OSError as exc: - stop = {"stop": "planner_failed", "reason": str(exc)} - break - if finished.returncode != 0: - stop = {"stop": "planner_failed", "reason": f"planner exited {finished.returncode}"} - break - if not spec_path.exists(): - stop = {"stop": "planner_stopped", "reason": f"no round spec for round {number}"} - break - spec = json.loads(spec_path.read_text(encoding="utf-8")) - output = round_dir / "incumbent-next" - driven = drive_round( - spec, - current, - round_dir / "round", - output, - backend=backend, - poll_seconds=poll_seconds, - clock=clock, - ) - rounds.append( - { - "round": number, - "choice": driven["close"]["choice"], - "chosen_tickets": driven["close"]["chosen_tickets"], - "drive": str(round_dir / "round" / "drive.json"), - } - ) - current = output - previous = str(round_dir / "round" / "drive.json") - body = {"round_cap": cap, "rounds": rounds, **stop, "incumbent": str(current)} - write_json(work_dir / "campaign.json", body) - return body diff --git a/src/inferrsi/harness/parallel_close.py b/src/inferrsi/harness/parallel_close.py deleted file mode 100644 index d3fba7e7e..000000000 --- a/src/inferrsi/harness/parallel_close.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Close one parallel round: pick passing lanes and write the next incumbent. - -A lane passes when its best measured ticket beats the incumbent by at least -``metric.margin`` (relative gain). With two or more passing lanes, their overlays -are stacked and measured against the incumbent once more. The stack is chosen only -if its gain is at least the best single lane's gain; otherwise the best single lane -is. The extra measurement is not charged to any lane's attempt budget. -""" - -import json -from datetime import datetime, timezone -from pathlib import Path - -from inferrsi.harness.parallel_round import ( - MeasurementError, - _snapshot, - compare, - hold_devices, - materialize, - summarize, -) -from inferrsi.harness.round_budget import BudgetError -from inferrsi.infra.artifacts import write_json -from inferrsi.infra.confinement import ConfinementUnavailable - - -class CloseError(RuntimeError): - """The round cannot be closed as asked.""" - - -def _score(result: dict) -> float: - relative = result.get("relative_gain") - return result["gain"] if relative is None else relative - - -def _passes(result: dict, margin: float) -> bool: - relative = result.get("relative_gain") - return result["gain"] > 0 and (relative is None or relative >= margin) - - -def _best_per_lane(round_dir: Path) -> dict[str, dict]: - best: dict[str, dict] = {} - for path in sorted((round_dir / "results").glob("*.json")): - result = json.loads(path.read_text(encoding="utf-8")) - if result.get("status") != "measured": - continue - lane_id = result.get("lane_id") - if lane_id is None: - manifest = round_dir / "outbox" / result["ticket"] / "manifest.json" - lane_id = json.loads(manifest.read_text(encoding="utf-8"))["lane_id"] - if lane_id not in best or _score(result) > _score(best[lane_id]): - best[lane_id] = {**result, "lane_id": lane_id} - return best - - -def close_round(round_dir: Path, incumbent: Path, output: Path, *, backend, clock=None) -> dict: - """Write the next incumbent to ``output`` and ``close.json``. Repeated calls return - the first decision without measuring again.""" - record_path = round_dir / "close.json" - if record_path.exists(): - return json.loads(record_path.read_text(encoding="utf-8")) - if output.exists() and any(output.iterdir()): - raise CloseError(f"output {output} already has files") - - def current() -> datetime: - if clock is not None: - return clock() - return datetime.now(timezone.utc) - - plan = json.loads((round_dir / "round-plan.json").read_text(encoding="utf-8")) - margin = plan["metric"]["margin"] - lane_count = {lane["id"]: lane["device_count"] for lane in plan["lanes"]} - best = _best_per_lane(round_dir) - passing = sorted( - (result for result in best.values() if _passes(result, margin)), - key=_score, - reverse=True, - ) - composite = None - chosen = passing[:1] - if len(passing) >= 2: - composite = _measure_composite( - round_dir, plan, incumbent, passing, lane_count, backend=backend, current=current - ) - if composite.get("status") == "measured" and _score(composite) >= _score(passing[0]): - chosen = passing - overlays = [round_dir / "outbox" / result["ticket"] for result in chosen] - if output.exists(): - output.rmdir() - materialize(incumbent, overlays, output) - hashes, _blocked = _snapshot(output) - body = { - "round_digest": plan["round_digest"], - "metric": plan["metric"], - "lanes": { - lane_id: { - "ticket": result["ticket"], - "gain": result["gain"], - "relative_gain": result.get("relative_gain"), - "passed": _passes(result, margin), - } - for lane_id, result in sorted(best.items()) - }, - "passing": [result["lane_id"] for result in passing], - "composite": composite, - "choice": ("kept_incumbent" if not chosen else "composite" if len(chosen) > 1 else "lane"), - "chosen_tickets": [result["ticket"] for result in chosen], - "output": str(output), - "files": hashes, - } - write_json(record_path, body) - return body - - -def _measure_composite( - round_dir: Path, - plan: dict, - incumbent: Path, - passing: list[dict], - lane_count: dict[str, int], - *, - backend, - current, -) -> dict: - count = max(lane_count[result["lane_id"]] for result in passing) - held = hold_devices(round_dir, plan, count) - if held is None: - return {"status": "unsettled", "reason": f"could not hold {count} free devices"} - devices = [device for device, _lease in held] - composite = round_dir / "close" / "composite" - try: - materialize( - incumbent, [round_dir / "outbox" / result["ticket"] for result in passing], composite - ) - candidate_samples, incumbent_samples = compare( - plan, - composite, - incumbent, - round_dir / "close" / "trials", - devices, - None, - backend=backend, - current=current, - ) - except (ConfinementUnavailable, BudgetError, MeasurementError) as exc: - return {"status": "unsettled", "reason": str(exc), "devices": devices} - finally: - for _device, lease in held: - lease.release() - return { - "status": "measured", - "tickets": [result["ticket"] for result in passing], - **summarize(plan, candidate_samples, incumbent_samples), - "devices": devices, - } diff --git a/src/inferrsi/harness/parallel_drive.py b/src/inferrsi/harness/parallel_drive.py deleted file mode 100644 index 44ffd9d87..000000000 --- a/src/inferrsi/harness/parallel_drive.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Drive one parallel round from a single command, then close it. - -Like a round without a cluster, each lane hands in one candidate per round; a -retry belongs to the next round. Each lane gets one worker, all in parallel: copy -the incumbent to ``round_dir/work/``, run the lane's ``agent`` command -there once, submit the work tree, and serve the ticket until it is no longer -queued. When every lane is done, ``close`` picks the next incumbent. - -The agent command runs on the host with the caller's environment; it is not -confined. Placeholders: ``{work}`` is the work tree and ``{lane}`` the lane id. - -A lane's outcome is the serve status (``measured``, ``rejected``, ``unsettled``), -``unchanged`` when the agent changed nothing, ``refused`` when submit refused the -change, ``agent_failed``, ``deadline`` or ``error``. -""" - -import shutil -import subprocess -import threading -import time -from datetime import datetime, timezone -from pathlib import Path - -from inferrsi.harness.parallel_close import close_round -from inferrsi.harness.parallel_round import RoundPlanError, prepare, serve_ticket, submit -from inferrsi.harness.round_budget import BudgetError, seconds_remaining -from inferrsi.infra.artifacts import write_json - - -def drive_round( - spec: dict, - incumbent: Path, - round_dir: Path, - output: Path, - *, - backend, - poll_seconds: float = 5.0, - clock=None, -) -> dict: - """Prepare, run one attempt on every lane in parallel, close, and write ``drive.json``.""" - plan = prepare(spec, incumbent, round_dir) - missing = [lane["id"] for lane in plan["lanes"] if not lane.get("agent")] - if missing: - raise RoundPlanError(f"lanes without an agent command: {', '.join(missing)}") - - def current() -> datetime: - if clock is not None: - return clock() - return datetime.now(timezone.utc) - - outcomes: dict[str, dict] = {} - - def work(lane: dict) -> None: - try: - outcomes[lane["id"]] = _drive_lane( - round_dir, incumbent, lane, backend, poll_seconds, current - ) - except Exception as exc: # noqa: BLE001 - one lane's crash must not hide the others - outcomes[lane["id"]] = _outcome("error", repr(exc)) - - workers = [threading.Thread(target=work, args=(lane,)) for lane in plan["lanes"]] - for worker in workers: - worker.start() - for worker in workers: - worker.join() - closed = close_round(round_dir, incumbent, output, backend=backend, clock=clock) - body = { - "round_digest": plan["round_digest"], - "lanes": {lane_id: outcomes[lane_id] for lane_id in sorted(outcomes)}, - "close": closed, - } - write_json(round_dir / "drive.json", body) - return body - - -def _drive_lane(round_dir, incumbent, lane, backend, poll_seconds, current) -> dict: - lane_id = lane["id"] - work = round_dir / "work" / lane_id - if not work.exists(): - shutil.copytree(incumbent, work) - try: - remaining = seconds_remaining(lane["deadline_utc"], current()) - except BudgetError as exc: - return _outcome("deadline", str(exc)) - argv = [part.replace("{work}", str(work)).replace("{lane}", lane_id) for part in lane["agent"]] - try: - finished = subprocess.run(argv, cwd=work, timeout=remaining, check=False) - except subprocess.TimeoutExpired: - return _outcome("deadline", "agent ran past the lane deadline") - except OSError as exc: - return _outcome("agent_failed", str(exc)) - if finished.returncode != 0: - return _outcome("agent_failed", f"agent exited {finished.returncode}") - submitted = submit(round_dir, lane_id, incumbent, work, now=current()) - if not submitted["submitted"]: - status = "unchanged" if submitted["reason"] == "unchanged" else "refused" - return _outcome(status, submitted["reason"]) - ticket = submitted["ticket"] - result = serve_ticket(round_dir, ticket, incumbent, backend=backend, clock=current) - while result["status"] == "queued": - time.sleep(poll_seconds) - result = serve_ticket(round_dir, ticket, incumbent, backend=backend, clock=current) - return _outcome(result["status"], result.get("reason", ""), ticket) - - -def _outcome(status: str, reason: str, ticket: str | None = None) -> dict: - return {"status": status, "reason": reason, "ticket": ticket} diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py index 37c365b5a..80d938abd 100644 --- a/src/inferrsi/harness/parallel_round.py +++ b/src/inferrsi/harness/parallel_round.py @@ -1,8 +1,7 @@ """Prepare, submit and measure one offline parallel round. -Directory diffs stand in for git clones. ``parallel_close`` picks the next -incumbent and ``parallel_drive`` runs a whole round. A synthetic metric value is -not a campaign verdict. +Directory diffs stand in for git clones. A synthetic metric value is not a +campaign verdict. Each lane declares ``device_count`` (default 1). The counts of all lanes must fit in ``device_ids``. A ticket runs only after it holds that many devices at once. @@ -205,7 +204,7 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: metric = spec.get("metric") if ( not isinstance(metric, dict) - or not {"name", "better"} <= set(metric) <= {"name", "better", "repeats", "margin"} + or not {"name", "better"} <= set(metric) <= {"name", "better", "repeats"} or not isinstance(metric["name"], str) or not metric["name"] or not isinstance(metric["better"], str) @@ -215,9 +214,6 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: repeats = metric.get("repeats", 1) if type(repeats) is not int or repeats < 1: raise RoundPlanError("metric repeats must be a positive integer") - margin = metric.get("margin", 0) - if type(margin) not in (int, float) or not math.isfinite(margin) or margin < 0: - raise RoundPlanError("metric margin must be a finite non-negative number") timeout = spec.get("command_timeout_seconds", 30) if type(timeout) not in (int, float) or not math.isfinite(timeout) or timeout <= 0: raise RoundPlanError("command_timeout_seconds must be a positive number") @@ -263,9 +259,6 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: count = lane.get("device_count", 1) if type(count) is not int or count < 1: raise RoundPlanError(f"{lane_id} device_count must be a positive integer") - agent = lane.get("agent", []) - if not isinstance(agent, list) or any(type(item) is not str for item in agent): - raise RoundPlanError(f"{lane_id} agent must be a list of strings") if any(_match(path, globs) and _match(path, protected) for path in files): raise RoundPlanError(f"{lane_id} reaches a protected path") seen_ids.add(lane_id) @@ -293,7 +286,6 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: "name": metric["name"], "better": metric["better"], "repeats": repeats, - "margin": margin, }, "command_timeout_seconds": timeout, "device_paths": {device: list(device_paths[device]) for device in device_paths}, diff --git a/tests/test_parallel_campaign.py b/tests/test_parallel_campaign.py deleted file mode 100644 index ab7312781..000000000 --- a/tests/test_parallel_campaign.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Running synthetic cluster rounds under a campaign plan's round cap.""" - -import json -import sys -from pathlib import Path - -import pytest - -from inferrsi.harness.parallel_campaign import CampaignError, round_cap, run_campaign -from inferrsi.infra.confinement import FixtureBackend - -pytest.importorskip("yaml") - -BENCH = ( - "import json, sys\n" - "from pathlib import Path\n" - "value = float(Path('knob.txt').read_text())\n" - "Path(sys.argv[1]).write_text(json.dumps({'metrics': {'latency_ms': value}}))\n" -) -AGENT = ( - "from pathlib import Path\n" - "knob = Path('knob.txt')\n" - "knob.write_text(str(int(knob.read_text()) - 1))\n" -) -PLANNER = ( - "import json, sys\n" - "from pathlib import Path\n" - "template, spec, number, last = sys.argv[1:5]\n" - "if int(number) > int(last):\n" - " sys.exit(0)\n" - "body = json.loads(Path(template).read_text())\n" - "body['round_id'] = 'r' + number\n" - "Path(spec).write_text(json.dumps(body))\n" -) - - -def _setup(tmp_path: Path, cap: str, last_round: int) -> tuple[Path, list[str], Path]: - incumbent = tmp_path / "incumbent" - incumbent.mkdir() - (incumbent / "knob.txt").write_text("10", encoding="utf-8") - (incumbent / "bench.py").write_text(BENCH, encoding="utf-8") - agent = tmp_path / "agent.py" - agent.write_text(AGENT, encoding="utf-8") - template = tmp_path / "template.json" - template.write_text( - json.dumps( - { - "round_id": "r", - "device_ids": ["cpu-0"], - "metric": {"name": "latency_ms", "better": "lower"}, - "incumbent_id": "fixture", - "protected": ["bench.py"], - "build": [], - "trial": [sys.executable, "bench.py", "{result}"], - "lanes": [ - { - "id": "L1", - "entry": "D-01", - "plane": "infra", - "globs": ["knob.txt"], - "cap": 1, - "per_attempt_worst_case": 1, - "deadline_utc": "2099-01-01T00:00:00+00:00", - "agent": [sys.executable, str(agent)], - } - ], - } - ), - encoding="utf-8", - ) - planner = tmp_path / "planner.py" - planner.write_text(PLANNER, encoding="utf-8") - campaign_plan = tmp_path / "campaign-plan.yaml" - campaign_plan.write_text(f'budget:\n round_cap: "{cap}"\n', encoding="utf-8") - argv = [sys.executable, str(planner), str(template), "{spec}", "{round}", str(last_round)] - return campaign_plan, argv, incumbent - - -def test_campaign_runs_round_cap_rounds_and_carries_the_incumbent(tmp_path): - campaign_plan, planner, incumbent = _setup(tmp_path, "2", last_round=5) - work = tmp_path / "campaign" - result = run_campaign( - campaign_plan, planner, incumbent, work, backend=FixtureBackend(), poll_seconds=0.01 - ) - assert result["stop"] == "round_cap" - assert [item["choice"] for item in result["rounds"]] == ["lane", "lane"] - assert (Path(result["incumbent"]) / "knob.txt").read_text(encoding="utf-8") == "8" - assert (incumbent / "knob.txt").read_text(encoding="utf-8") == "10" - assert json.loads((work / "campaign.json").read_text(encoding="utf-8")) == result - - -def test_campaign_stops_when_the_planner_writes_no_spec(tmp_path): - campaign_plan, planner, incumbent = _setup(tmp_path, "5", last_round=1) - result = run_campaign( - campaign_plan, - planner, - incumbent, - tmp_path / "campaign", - backend=FixtureBackend(), - poll_seconds=0.01, - ) - assert result["stop"] == "planner_stopped" - assert len(result["rounds"]) == 1 - assert (Path(result["incumbent"]) / "knob.txt").read_text(encoding="utf-8") == "9" - - -def test_round_cap_must_be_a_positive_whole_number(tmp_path): - plan = tmp_path / "campaign-plan.yaml" - for value in ('""', '"0"', '"2.5"', "true", "[]"): - plan.write_text(f"budget:\n round_cap: {value}\n", encoding="utf-8") - with pytest.raises(CampaignError, match="round_cap"): - round_cap(plan) - plan.write_text("budget:\n round_cap: 3\n", encoding="utf-8") - assert round_cap(plan) == 3 diff --git a/tests/test_parallel_close.py b/tests/test_parallel_close.py deleted file mode 100644 index 0eede5649..000000000 --- a/tests/test_parallel_close.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Closing a synthetic round. Metric figures here are fixture inputs, not measurements.""" - -import json -import sys -from datetime import datetime, timezone -from pathlib import Path - -import pytest - -from inferrsi.harness.parallel_close import CloseError, close_round -from inferrsi.harness.parallel_round import prepare, serve_ticket, submit -from inferrsi.infra.confinement import FixtureBackend - -WHEN = datetime(2026, 10, 30, tzinfo=timezone.utc) -BENCH = ( - "import json, sys\n" - "from pathlib import Path\n" - "a = float(Path('knobs/a.txt').read_text())\n" - "b = float(Path('knobs/b.txt').read_text())\n" - "both = 100 if a != 10 and b != 10 else 0\n" - "value = a + b + (both if Path('clash').exists() else 0)\n" - "Path(sys.argv[1]).write_text(json.dumps({'metrics': {'score': value}}))\n" -) - - -def _tree(root: Path, *, clash: bool = False) -> None: - (root / "knobs").mkdir(parents=True) - (root / "knobs" / "a.txt").write_text("10", encoding="utf-8") - (root / "knobs" / "b.txt").write_text("10", encoding="utf-8") - (root / "bench.py").write_text(BENCH, encoding="utf-8") - if clash: - (root / "clash").write_text("", encoding="utf-8") - - -def _lane(lane_id: str, glob: str) -> dict: - return { - "id": lane_id, - "entry": f"D-{lane_id}", - "plane": "infra", - "globs": [glob], - "cap": 3, - "per_attempt_worst_case": 1, - "deadline_utc": "2099-01-01T00:00:00+00:00", - } - - -def _round(tmp_path, *, better="lower", margin=0, clash=False): - incumbent = tmp_path / "incumbent" - _tree(incumbent, clash=clash) - round_dir = tmp_path / "round" - spec = { - "round_id": "r1", - "device_ids": ["cpu-0", "cpu-1"], - "metric": {"name": "score", "better": better, "margin": margin}, - "incumbent_id": "fixture-incumbent", - "protected": ["bench.py", "clash"], - "build": [], - "trial": [sys.executable, "bench.py", "{result}"], - "lanes": [_lane("L1", "knobs/a.txt"), _lane("L2", "knobs/b.txt")], - } - prepare(spec, incumbent, round_dir) - return incumbent, round_dir - - -def _try(tmp_path, incumbent, round_dir, lane_id, knob, value) -> dict: - work = tmp_path / f"work-{lane_id}-{value}" - _tree(work, clash=(incumbent / "clash").exists()) - (work / "knobs" / knob).write_text(value, encoding="utf-8") - ticket = submit(round_dir, lane_id, incumbent, work, now=WHEN)["ticket"] - return serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) - - -def _knobs(root: Path) -> tuple[str, str]: - return ( - (root / "knobs" / "a.txt").read_text(encoding="utf-8"), - (root / "knobs" / "b.txt").read_text(encoding="utf-8"), - ) - - -def test_no_passing_lane_keeps_the_incumbent(tmp_path): - incumbent, round_dir = _round(tmp_path) - assert _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "12")["status"] == "measured" - closed = close_round(round_dir, incumbent, tmp_path / "next", backend=FixtureBackend()) - assert closed["choice"] == "kept_incumbent" - assert closed["lanes"]["L1"]["passed"] is False - assert _knobs(tmp_path / "next") == ("10", "10") - - -def test_one_passing_lane_takes_its_best_ticket(tmp_path): - incumbent, round_dir = _round(tmp_path) - _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "9") - best = _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "7") - closed = close_round(round_dir, incumbent, tmp_path / "next", backend=FixtureBackend()) - assert closed["choice"] == "lane" - assert closed["chosen_tickets"] == [best["ticket"]] - assert closed["composite"] is None - assert _knobs(tmp_path / "next") == ("7", "10") - assert set(closed["files"]) == {"bench.py", "knobs/a.txt", "knobs/b.txt"} - - -def test_margin_filters_small_gains(tmp_path): - incumbent, round_dir = _round(tmp_path, margin=0.1) - _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "9") - closed = close_round(round_dir, incumbent, tmp_path / "next", backend=FixtureBackend()) - assert closed["choice"] == "kept_incumbent" - - -def test_two_passing_lanes_stack_when_the_composite_is_better(tmp_path): - incumbent, round_dir = _round(tmp_path) - _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "8") - _try(tmp_path, incumbent, round_dir, "L2", "b.txt", "7") - closed = close_round(round_dir, incumbent, tmp_path / "next", backend=FixtureBackend()) - assert closed["passing"] == ["L2", "L1"] - assert closed["composite"]["status"] == "measured" - assert closed["composite"]["candidate"] == 15 - assert closed["choice"] == "composite" - assert _knobs(tmp_path / "next") == ("8", "7") - - -def test_a_worse_composite_falls_back_to_the_best_single_lane(tmp_path): - incumbent, round_dir = _round(tmp_path, clash=True) - _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "8") - _try(tmp_path, incumbent, round_dir, "L2", "b.txt", "7") - closed = close_round(round_dir, incumbent, tmp_path / "next", backend=FixtureBackend()) - assert closed["composite"]["candidate"] == 115 - assert closed["choice"] == "lane" - assert _knobs(tmp_path / "next") == ("10", "7") - - -def test_higher_is_better_picks_the_larger_score(tmp_path): - incumbent, round_dir = _round(tmp_path, better="higher") - _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "8") - _try(tmp_path, incumbent, round_dir, "L2", "b.txt", "13") - closed = close_round(round_dir, incumbent, tmp_path / "next", backend=FixtureBackend()) - assert closed["passing"] == ["L2"] - assert _knobs(tmp_path / "next") == ("10", "13") - - -def test_close_is_idempotent_and_refuses_a_non_empty_output(tmp_path): - incumbent, round_dir = _round(tmp_path) - _try(tmp_path, incumbent, round_dir, "L1", "a.txt", "8") - _try(tmp_path, incumbent, round_dir, "L2", "b.txt", "7") - first = close_round(round_dir, incumbent, tmp_path / "next", backend=FixtureBackend()) - - class Refuses: - name = "refuses" - - def run(self, *args, **kwargs): - raise AssertionError("close measured twice") - - again = close_round(round_dir, incumbent, tmp_path / "next", backend=Refuses()) - assert again == first - assert json.loads((round_dir / "close.json").read_text(encoding="utf-8")) == first - taken = tmp_path / "taken" - taken.mkdir() - (taken / "x").write_text("", encoding="utf-8") - (round_dir / "close.json").unlink() - with pytest.raises(CloseError, match="already has files"): - close_round(round_dir, incumbent, taken, backend=FixtureBackend()) diff --git a/tests/test_parallel_drive.py b/tests/test_parallel_drive.py deleted file mode 100644 index 1a2ce740c..000000000 --- a/tests/test_parallel_drive.py +++ /dev/null @@ -1,174 +0,0 @@ -"""Driving a synthetic round end to end. Metric figures here are fixture inputs.""" - -import json -import sys -from pathlib import Path - -import pytest - -from inferrsi.harness.parallel_drive import drive_round -from inferrsi.harness.parallel_round import RoundPlanError -from inferrsi.infra.confinement import FixtureBackend, UnshareBackend, namespaces_available - -BENCH = ( - "import json, sys\n" - "from pathlib import Path\n" - "a = float(Path('knobs/a.txt').read_text())\n" - "b = float(Path('knobs/b.txt').read_text())\n" - "Path(sys.argv[1]).write_text(json.dumps({'metrics': {'latency_ms': a + b}}))\n" -) -AGENT = ( - "import sys\n" - "from pathlib import Path\n" - "if sys.argv[2] == 'fail':\n" - " sys.exit(3)\n" - "if sys.argv[2] != 'same':\n" - " Path(sys.argv[1]).write_text(sys.argv[2])\n" - "Path(sys.argv[3]).mkdir(parents=True, exist_ok=True)\n" - "Path(sys.argv[3], 'ran').open('a').write('x')\n" -) - - -def _tree(root: Path) -> None: - (root / "knobs").mkdir(parents=True) - (root / "knobs" / "a.txt").write_text("10", encoding="utf-8") - (root / "knobs" / "b.txt").write_text("10", encoding="utf-8") - (root / "bench.py").write_text(BENCH, encoding="utf-8") - - -def _lane(lane_id: str, knob: str, value: str, tmp_path: Path) -> dict: - agent = tmp_path / "agent.py" - agent.write_text(AGENT, encoding="utf-8") - return { - "id": lane_id, - "entry": f"D-{lane_id}", - "plane": "infra", - "globs": [f"knobs/{knob}"], - "cap": 4, - "per_attempt_worst_case": 1, - "deadline_utc": "2099-01-01T00:00:00+00:00", - "agent": [ - sys.executable, - str(agent), - f"knobs/{knob}", - value, - str(tmp_path / "calls" / "{lane}"), - ], - } - - -def _spec(lanes: list[dict]) -> dict: - return { - "round_id": "r1", - "device_ids": ["cpu-0", "cpu-1"], - "metric": {"name": "latency_ms", "better": "lower", "repeats": 2}, - "incumbent_id": "fixture-incumbent", - "protected": ["bench.py"], - "build": [], - "trial": [sys.executable, "bench.py", "{result}"], - "lanes": lanes, - } - - -def _knobs(root: Path) -> tuple[str, str]: - return ( - (root / "knobs" / "a.txt").read_text(encoding="utf-8"), - (root / "knobs" / "b.txt").read_text(encoding="utf-8"), - ) - - -def test_drive_runs_each_lane_once_and_closes_on_the_stacked_change(tmp_path): - incumbent = tmp_path / "incumbent" - _tree(incumbent) - round_dir = tmp_path / "round" - lanes = [_lane("L1", "a.txt", "8", tmp_path), _lane("L2", "b.txt", "7", tmp_path)] - driven = drive_round( - _spec(lanes), - incumbent, - round_dir, - tmp_path / "next", - backend=FixtureBackend(), - poll_seconds=0.01, - ) - assert {lane: item["status"] for lane, item in driven["lanes"].items()} == { - "L1": "measured", - "L2": "measured", - } - for lane in ("L1", "L2"): - assert (tmp_path / "calls" / lane / "ran").read_text(encoding="utf-8") == "x" - assert len(list((round_dir / "results").glob("*.json"))) == 2 - assert driven["close"]["choice"] == "composite" - assert driven["close"]["composite"]["candidate"] == 15 - assert _knobs(tmp_path / "next") == ("8", "7") - assert _knobs(incumbent) == ("10", "10") - assert json.loads((round_dir / "drive.json").read_text(encoding="utf-8")) == driven - - -def test_a_failing_agent_stops_only_its_lane(tmp_path): - incumbent = tmp_path / "incumbent" - _tree(incumbent) - lanes = [_lane("L1", "a.txt", "8", tmp_path), _lane("L2", "b.txt", "fail", tmp_path)] - driven = drive_round( - _spec(lanes), - incumbent, - tmp_path / "round", - tmp_path / "next", - backend=FixtureBackend(), - poll_seconds=0.01, - ) - assert driven["lanes"]["L2"] == { - "status": "agent_failed", - "reason": "agent exited 3", - "ticket": None, - } - assert driven["close"]["choice"] == "lane" - assert _knobs(tmp_path / "next") == ("8", "10") - - -def test_an_agent_without_changes_leaves_the_incumbent(tmp_path): - incumbent = tmp_path / "incumbent" - _tree(incumbent) - driven = drive_round( - _spec([_lane("L1", "a.txt", "same", tmp_path)]), - incumbent, - tmp_path / "round", - tmp_path / "next", - backend=FixtureBackend(), - poll_seconds=0.01, - ) - assert driven["lanes"]["L1"]["status"] == "unchanged" - assert driven["close"]["choice"] == "kept_incumbent" - assert _knobs(tmp_path / "next") == ("10", "10") - - -def test_drive_needs_an_agent_command_on_every_lane(tmp_path): - incumbent = tmp_path / "incumbent" - _tree(incumbent) - lane = _lane("L1", "a.txt", "8", tmp_path) - del lane["agent"] - with pytest.raises(RoundPlanError, match="without an agent"): - drive_round( - _spec([lane]), - incumbent, - tmp_path / "round", - tmp_path / "next", - backend=FixtureBackend(), - ) - - -@pytest.mark.skipif(not namespaces_available(), reason="user namespaces are unavailable") -def test_drive_under_real_namespaces(tmp_path): - incumbent = tmp_path / "incumbent" - _tree(incumbent) - lanes = [_lane("L1", "a.txt", "8", tmp_path), _lane("L2", "b.txt", "7", tmp_path)] - driven = drive_round( - _spec(lanes), - incumbent, - tmp_path / "round", - tmp_path / "next", - backend=UnshareBackend(), - poll_seconds=0.01, - ) - assert driven["close"]["choice"] == "composite" - assert driven["close"]["composite"]["candidate"] == 15 - assert _knobs(tmp_path / "next") == ("8", "7") diff --git a/tests/test_parallel_round.py b/tests/test_parallel_round.py index 4bf46c00f..ae22a192c 100644 --- a/tests/test_parallel_round.py +++ b/tests/test_parallel_round.py @@ -256,15 +256,14 @@ def test_a_lane_needing_two_devices_waits_until_it_can_hold_both(tmp_path): assert stamp.read_text(encoding="utf-8") == "1,2" -def test_prepare_rejects_bad_repeats_margin_and_timeout(tmp_path): +def test_prepare_rejects_bad_repeats_and_timeout(tmp_path): incumbent = tmp_path / "incumbent" _tree(incumbent, "10") base = {"name": "latency_ms", "better": "lower"} for extra, message in ( ({"repeats": 0}, "repeats"), ({"repeats": True}, "repeats"), - ({"margin": -0.1}, "margin"), - ({"margin": float("nan")}, "margin"), + ({"margin": 0.03}, "metric needs"), ({"spread": 1}, "metric needs"), ): with pytest.raises(RoundPlanError, match=message): @@ -273,11 +272,11 @@ def test_prepare_rejects_bad_repeats_margin_and_timeout(tmp_path): with pytest.raises(RoundPlanError, match="command_timeout_seconds"): prepare(_spec(command_timeout_seconds=timeout), incumbent, tmp_path / "round-t") plan = prepare(_spec(), incumbent, tmp_path / "round-defaults") - assert plan["metric"] == {**base, "repeats": 1, "margin": 0} + assert plan["metric"] == {**base, "repeats": 1} assert plan["command_timeout_seconds"] == 30 -def test_prepare_rejects_bad_device_paths_env_and_agent(tmp_path): +def test_prepare_rejects_bad_device_paths_and_env(tmp_path): incumbent = tmp_path / "incumbent" _tree(incumbent, "10") for overrides, message in ( @@ -290,10 +289,6 @@ def test_prepare_rejects_bad_device_paths_env_and_agent(tmp_path): ): with pytest.raises(RoundPlanError, match=message): prepare(_spec(**overrides), incumbent, tmp_path / "round-paths") - lanes = _spec()["lanes"] - lanes[0]["agent"] = "claude -p" - with pytest.raises(RoundPlanError, match="agent must be a list"): - prepare(_spec(lanes=lanes), incumbent, tmp_path / "round-agent") def test_serve_passes_only_the_held_device_nodes_and_env(tmp_path): diff --git a/tools/parallel_round.py b/tools/parallel_round.py index 5e1c81078..40ddaff59 100644 --- a/tools/parallel_round.py +++ b/tools/parallel_round.py @@ -6,9 +6,6 @@ from datetime import datetime, timezone from pathlib import Path -from inferrsi.harness.parallel_campaign import run_campaign -from inferrsi.harness.parallel_close import close_round -from inferrsi.harness.parallel_drive import drive_round from inferrsi.harness.parallel_round import prepare, serve_ticket, submit, ticket_status from inferrsi.infra.confinement import select_backend @@ -41,25 +38,6 @@ def main(argv: list[str] | None = None) -> int: status_command.add_argument("--round-dir", required=True, type=Path) status_command.add_argument("--ticket", required=True) - drive_command = commands.add_parser("drive") - drive_command.add_argument("--spec", required=True) - drive_command.add_argument("--incumbent", required=True, type=Path) - drive_command.add_argument("--round-dir", required=True, type=Path) - drive_command.add_argument("--output", required=True, type=Path) - drive_command.add_argument("--poll-seconds", type=float, default=5.0) - - campaign_command = commands.add_parser("campaign") - campaign_command.add_argument("--campaign-plan", required=True, type=Path) - campaign_command.add_argument("--incumbent", required=True, type=Path) - campaign_command.add_argument("--work-dir", required=True, type=Path) - campaign_command.add_argument("--poll-seconds", type=float, default=5.0) - campaign_command.add_argument("--planner", required=True, nargs=argparse.REMAINDER) - - close_command = commands.add_parser("close") - close_command.add_argument("--round-dir", required=True, type=Path) - close_command.add_argument("--incumbent", required=True, type=Path) - close_command.add_argument("--output", required=True, type=Path) - args = parser.parse_args(argv) if args.command == "prepare": result = prepare(_load(args.spec), args.incumbent, args.round_dir) @@ -78,26 +56,6 @@ def main(argv: list[str] | None = None) -> int: args.incumbent, backend=select_backend(), ) - elif args.command == "drive": - result = drive_round( - _load(args.spec), - args.incumbent, - args.round_dir, - args.output, - backend=select_backend(), - poll_seconds=args.poll_seconds, - ) - elif args.command == "campaign": - result = run_campaign( - args.campaign_plan, - args.planner, - args.incumbent, - args.work_dir, - backend=select_backend(), - poll_seconds=args.poll_seconds, - ) - elif args.command == "close": - result = close_round(args.round_dir, args.incumbent, args.output, backend=select_backend()) else: result = ticket_status(args.round_dir, args.ticket) json.dump(result, sys.stdout, indent=2) From 03a728817e6c86978b345799fcabdc3bd68aeafe Mon Sep 17 00:00:00 2001 From: NumberWan Date: Fri, 13 Nov 2026 11:54:43 +0800 Subject: [PATCH 13/23] fix(confinement): kill the whole namespace on timeout, isolate the supervisor Every confined command now runs as PID 1 of its own PID namespace under `unshare --pid --kill-child`, started in its own session. On timeout the session is killed and run() returns only after no member is left, so nothing the candidate started can act after the deadline or after the caller releases its device lease. Before, a timeout killed `unshare` only and could leave the payload running. The supervisor starts with `python -I` and gets the package root as an argument. The candidate cwd and PYTHON* variables no longer reach sys.path, so a candidate cannot shadow `inferrsi`, and a PYTHONPATH-only checkout still finds the real package. Signed-off-by: NumberWan --- src/inferrsi/infra/confinement.py | 67 +++++++++++++---- tests/test_confinement.py | 117 ++++++++++++++++++++++++------ 2 files changed, 148 insertions(+), 36 deletions(-) diff --git a/src/inferrsi/infra/confinement.py b/src/inferrsi/infra/confinement.py index 329797ddb..562f4d5ad 100644 --- a/src/inferrsi/infra/confinement.py +++ b/src/inferrsi/infra/confinement.py @@ -9,13 +9,15 @@ Callers may add device nodes (bound read-write, e.g. one GPU's ``/dev/nvidiaN``), extra read-only runtime paths (e.g. the driver's user-space libraries) and extra -environment variables. With device nodes, the command also gets a new PID -namespace and a fresh ``/proc``; the host ``/proc`` is never bound. Every jail has -an empty ``/tmp`` and ``/dev/shm``. +environment variables. Every command runs as PID 1 of its own PID namespace, so +when it exits or is killed nothing it started survives. With device nodes, the +jail also gets a fresh ``/proc``; the host ``/proc`` is never bound. Every jail +has an empty ``/tmp`` and ``/dev/shm``. """ import json import os +import signal import struct import subprocess import sys @@ -24,6 +26,12 @@ from pathlib import Path _ENV_ALLOW = frozenset({"PATH", "LANG", "LC_ALL", "LC_CTYPE", "TZ", "PYTHONIOENCODING"}) +_PACKAGE_ROOT = str(Path(__file__).resolve().parents[2]) +_SUPERVISOR = ( + "import sys; sys.path.insert(0, sys.argv[1]); " + "from inferrsi.infra.confinement import enter_namespace; enter_namespace(sys.argv[2])" +) +_REAP_SECONDS = 10.0 _GUARD = """ import builtins import os @@ -262,6 +270,32 @@ def _run_local(argv: tuple[str, ...], cwd: Path, deadline: float, environment: d } +def supervisor_command(config_path: Path) -> list[str]: + """Isolated interpreter: the candidate cwd and PYTHON* variables do not reach sys.path.""" + return [sys.executable, "-I", "-c", _SUPERVISOR, _PACKAGE_ROOT, str(config_path)] + + +def _kill_group(process: subprocess.Popen) -> None: + """Kill the session and return only once no member of it is left.""" + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + for stream in (process.stdout, process.stderr): + if stream is not None: + stream.close() + give_up = time.monotonic() + _REAP_SECONDS + while True: + try: + os.killpg(process.pid, 0) + except ProcessLookupError: + return + if time.monotonic() > give_up: + raise ConfinementUnavailable("namespace command outlived its deadline") + time.sleep(0.05) + + class UnavailableBackend: """Fail closed. Does not execute the candidate.""" @@ -351,7 +385,7 @@ def run( json.dumps({"argv": list(argv), "cwd": str(cwd), **plan}), encoding="utf-8", ) - completed = subprocess.run( + process = subprocess.Popen( [ "unshare", "--user", @@ -359,24 +393,27 @@ def run( "--mount", "--propagation", "private", - *(["--pid", "--fork"] if plan["proc"] else []), + "--pid", + "--kill-child", "--", - sys.executable, - "-c", - "import sys; from inferrsi.infra.confinement import enter_namespace; " - "enter_namespace(sys.argv[1])", - str(config_path), + *supervisor_command(config_path), ], cwd=cwd, env=minimal_environment(), stdin=subprocess.DEVNULL, - capture_output=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, - timeout=remaining, - check=False, + start_new_session=True, + ) + try: + stdout, stderr = process.communicate(timeout=remaining) + except subprocess.TimeoutExpired as exc: + _kill_group(process) + raise ConfinementUnavailable("namespace command timed out") from exc + completed = subprocess.CompletedProcess( + process.args, process.returncode, stdout, stderr ) - except subprocess.TimeoutExpired as exc: - raise ConfinementUnavailable("namespace command timed out") from exc finally: if config_path is not None: config_path.unlink(missing_ok=True) diff --git a/tests/test_confinement.py b/tests/test_confinement.py index 361ae426b..34c3c4371 100644 --- a/tests/test_confinement.py +++ b/tests/test_confinement.py @@ -10,6 +10,7 @@ import pytest +from inferrsi.infra import confinement from inferrsi.infra.confinement import ( ConfinementUnavailable, FixtureBackend, @@ -17,9 +18,11 @@ UnshareBackend, _elf_interpreter, confined_execute, + minimal_environment, namespace_plan, namespaces_available, select_backend, + supervisor_command, ) @@ -69,22 +72,25 @@ def test_select_backend_fails_closed_when_namespaces_are_disabled(monkeypatch): def test_unshare_failure_does_not_fall_back(tmp_path, monkeypatch): calls = [] monkeypatch.setenv("SECRET_CANARY", "CANARY-ENV") - real_run = subprocess.run + real_popen = subprocess.Popen - def fake_run(argv, **kwargs): - if not argv or argv[0] != "unshare": - return real_run(argv, **kwargs) - del kwargs - calls.append(json.loads(Path(argv[-1]).read_text(encoding="utf-8"))) + class FailedUnshare: + def __init__(self, argv): + self.args = argv + self.returncode = None + calls.append(json.loads(Path(argv[-1]).read_text(encoding="utf-8"))) - class Completed: - returncode = 1 - stdout = "" - stderr = "unshare failed" + def communicate(self, timeout=None): + del timeout + self.returncode = 1 + return "", "unshare failed" - return Completed() + def fake_popen(argv, **kwargs): + if argv and argv[0] == "unshare": + return FailedUnshare(argv) + return real_popen(argv, **kwargs) - monkeypatch.setattr("inferrsi.infra.confinement.subprocess.run", fake_run) + monkeypatch.setattr("inferrsi.infra.confinement.subprocess.Popen", fake_popen) with pytest.raises(ConfinementUnavailable, match="unshare failed"): UnshareBackend().run( (sys.executable, "-c", "print('should-not-run')"), @@ -98,23 +104,68 @@ class Completed: assert "/" not in {item["source"] for item in calls[0]["mounts"]} -def test_unshare_timeout_becomes_confinement_failure(tmp_path, monkeypatch): - real_run = subprocess.run +_LATE_CHILD = """ +import pathlib, sys, time +pathlib.Path(sys.argv[1], "started").write_text("1") +time.sleep(float(sys.argv[2])) +pathlib.Path(sys.argv[1], "late").write_text("1") +""" + + +def test_unshare_timeout_kills_the_whole_process_group(tmp_path, monkeypatch): + real_popen = subprocess.Popen + (tmp_path / "child.py").write_text(_LATE_CHILD, encoding="utf-8") + child = [str(tmp_path / "child.py"), str(tmp_path), "1.5"] + payload = ( + "import subprocess, sys, time\n" + f"subprocess.Popen([sys.executable, *{child!r}])\n" + "time.sleep(30)\n" + ) - def fake_run(argv, **kwargs): - if not argv or argv[0] != "unshare": - return real_run(argv, **kwargs) - raise subprocess.TimeoutExpired(argv, kwargs.get("timeout") or 1) + def fake_popen(argv, **kwargs): + if argv and argv[0] == "unshare": + argv = [sys.executable, "-c", payload] + return real_popen(argv, **kwargs) - monkeypatch.setattr("inferrsi.infra.confinement.subprocess.run", fake_run) + monkeypatch.setattr("inferrsi.infra.confinement.subprocess.Popen", fake_popen) with pytest.raises(ConfinementUnavailable, match="timed out"): UnshareBackend().run( - (sys.executable, "-c", "print('should-not-run')"), + (sys.executable, "-c", "print('unused')"), tmp_path, (tmp_path,), (), - time.monotonic() + 5, + time.monotonic() + 1, ) + time.sleep(2) + assert (tmp_path / "started").exists() + assert not (tmp_path / "late").exists() + + +def test_supervisor_ignores_a_shadow_package_in_the_candidate(tmp_path): + shadow = tmp_path / "candidate" + (shadow / "inferrsi" / "infra").mkdir(parents=True) + canary = tmp_path / "shadow-imported" + (shadow / "inferrsi" / "__init__.py").write_text( + f"open({str(canary)!r}, 'w').write('1')\n", encoding="utf-8" + ) + (shadow / "inferrsi" / "infra" / "__init__.py").write_text("", encoding="utf-8") + (shadow / "inferrsi" / "infra" / "confinement.py").write_text( + "def enter_namespace(path):\n raise SystemExit('shadow')\n", encoding="utf-8" + ) + completed = subprocess.run( + supervisor_command(tmp_path / "missing.json"), + cwd=shadow, + env={**minimal_environment(), "PYTHONPATH": str(shadow)}, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert completed.returncode != 0 + assert not canary.exists() + assert "shadow" not in completed.stderr + assert "FileNotFoundError" in completed.stderr + assert str(Path(confinement.__file__).resolve()) in completed.stderr def test_namespace_plan_includes_the_loader_but_not_a_private_file(tmp_path, monkeypatch): @@ -161,6 +212,30 @@ def test_unshare_starts_python_and_hides_a_private_canary(tmp_path, monkeypatch) assert "True" not in record["stdout"] +@pytest.mark.skipif(not namespaces_available(), reason="user namespaces are unavailable") +def test_unshare_timeout_leaves_no_detached_child(tmp_path): + work = tmp_path / "work" + work.mkdir() + (work / "child.py").write_text(_LATE_CHILD, encoding="utf-8") + script = ( + "import subprocess, sys, time\n" + f"subprocess.Popen([sys.executable, {str(work / 'child.py')!r}, {str(work)!r}, '4'], " + "start_new_session=True)\n" + "time.sleep(30)\n" + ) + with pytest.raises(ConfinementUnavailable, match="timed out"): + UnshareBackend().run( + (sys.executable, "-c", script), + work, + (work,), + (), + time.monotonic() + 3, + ) + time.sleep(3) + assert (work / "started").exists() + assert not (work / "late").exists() + + def test_blocked_unshare_reports_namespaces_unavailable(monkeypatch): real = Path.read_text From f7a20b606a59bab3349b7f6034c9e9b856f1319d Mon Sep 17 00:00:00 2001 From: NumberWan Date: Fri, 13 Nov 2026 12:08:14 +0800 Subject: [PATCH 14/23] fix(harness): resume an interrupted serve without a second attempt serve_ticket now reads this ticket's ledger row in any phase. A row left in `reserved` by an interrupted serve is resumed under the same binding instead of charging a new ordinal. A `finished` or `claimed` row without a receipt is written as unsettled and not launched again; a `cancelled` row is written as rejected. A queued ticket whose lane deadline has passed is now rejected instead of staying queued forever, and an interrupted reservation past the deadline is cancelled. Workspace reads open with O_NONBLOCK and fstat the descriptor, so a file swapped for a FIFO after the stat check is refused instead of blocking the coordinator. Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_round.py | 62 +++++++++++++----- tests/test_parallel_round.py | 90 +++++++++++++++++++++++++- 2 files changed, 134 insertions(+), 18 deletions(-) diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py index 80d938abd..b84d8277e 100644 --- a/src/inferrsi/harness/parallel_round.py +++ b/src/inferrsi/harness/parallel_round.py @@ -95,6 +95,18 @@ def _snapshot(root: Path) -> tuple[dict[str, str], list[str]]: return hashes, blocked +def _open_regular(name: str, dirfd: int) -> int | None: + """Open without following a link or blocking on a FIFO; refuse anything not regular.""" + try: + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=dirfd) + except OSError: + return None + if not stat.S_ISREG(os.fstat(fd).st_mode): + os.close(fd) + return None + return fd + + def _scan(dirfd: int, prefix: str, hashes: dict[str, str], blocked: list[str]) -> None: for name in os.listdir(dirfd): if name in {".", "..", ".git"}: @@ -122,9 +134,8 @@ def _scan(dirfd: int, prefix: str, hashes: dict[str, str], blocked: list[str]) - if not stat.S_ISREG(info.st_mode): blocked.append(relative) continue - try: - file_fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=dirfd) - except OSError: + file_fd = _open_regular(name, dirfd) + if file_fd is None: blocked.append(relative) continue try: @@ -151,9 +162,8 @@ def _read_nofollow(root: Path, relative: str) -> bytes | None: return None os.close(dirfd) dirfd = child - try: - file_fd = os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW, dir_fd=dirfd) - except OSError: + file_fd = _open_regular(parts[-1], dirfd) + if file_fd is None: return None finally: os.close(dirfd) @@ -442,10 +452,10 @@ def _record(round_dir: Path, ticket: str, **fields) -> dict: return body -def _claimed_entry(ledger, ticket: str): +def _ticket_entry(ledger, ticket: str): suffix = f"@{ticket}" for entry in ledger.entries: - if entry.binding.run_id.endswith(suffix) and entry.phase == "claimed": + if entry.binding.run_id.endswith(suffix): return entry return None @@ -627,13 +637,27 @@ def current() -> datetime: try: if result_path.exists(): return json.loads(result_path.read_text(encoding="utf-8")) - if _claimed_entry(_load_ledger(round_dir, plan, lane), ticket) is not None: + entry = _ticket_entry(_load_ledger(round_dir, plan, lane), ticket) + if entry is not None and entry.phase == "cancelled": + return _record( + round_dir, ticket, status="rejected", reason="attempt was cancelled at the deadline" + ) + if entry is not None and entry.phase != "reserved": return _record( round_dir, ticket, status="unsettled", - reason="claimed execution was not settled", + reason=f"{entry.phase} execution was not settled", ) + try: + ensure_open(lane["deadline_utc"], current()) + except BudgetError as exc: + if entry is not None: + _save_ledger( + round_dir, + advance(_load_ledger(round_dir, plan, lane), entry.binding, "cancelled"), + ) + return _record(round_dir, ticket, status="rejected", reason=str(exc)) held = hold_devices(round_dir, plan, lane["device_count"]) if held is None: return {"ticket": ticket, "status": "queued"} @@ -648,6 +672,7 @@ def current() -> datetime: [device for device, _lease in held], backend=backend, current=current, + reserved=entry, ) finally: for _device, lease in held: @@ -667,13 +692,18 @@ def _measure( *, backend, current, + reserved=None, ) -> dict: - try: - loaded = _load_ledger(round_dir, plan, lane) - ledger, binding = admit_attempt(plan, loaded, lane["id"], current(), token=ticket) - except BudgetError as exc: - return _record(round_dir, ticket, status="rejected", reason=str(exc)) - _save_ledger(round_dir, ledger) + """``reserved`` is this ticket's surviving reservation from an interrupted serve.""" + ledger = _load_ledger(round_dir, plan, lane) + if reserved is not None: + binding = reserved.binding + else: + try: + ledger, binding = admit_attempt(plan, ledger, lane["id"], current(), token=ticket) + except BudgetError as exc: + return _record(round_dir, ticket, status="rejected", reason=str(exc)) + _save_ledger(round_dir, ledger) candidate = round_dir / "candidates" / ticket trial_dir = round_dir / "trials" / ticket materialize(incumbent, [overlay], candidate) diff --git a/tests/test_parallel_round.py b/tests/test_parallel_round.py index ae22a192c..ba08ac4ca 100644 --- a/tests/test_parallel_round.py +++ b/tests/test_parallel_round.py @@ -10,8 +10,10 @@ import pytest +from inferrsi.harness import parallel_round from inferrsi.harness.parallel_round import ( RoundPlanError, + _read_nofollow, prepare, serve_ticket, submit, @@ -538,7 +540,7 @@ def test_launch_after_deadline_cancels_before_claim(tmp_path): work = tmp_path / "work" _tree(work, "8") ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] - moments = [WHEN, datetime(2100, 1, 1, tzinfo=timezone.utc)] + moments = [WHEN, WHEN, datetime(2100, 1, 1, tzinfo=timezone.utc)] def clock(): return moments.pop(0) if len(moments) > 1 else moments[0] @@ -565,7 +567,7 @@ def test_deadline_expiring_between_build_and_candidate_stops_the_trial(tmp_path) work = tmp_path / "work" _tree(work, "8") ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] - moments = [WHEN, WHEN, WHEN + timedelta(seconds=1), WHEN + timedelta(seconds=11)] + moments = [WHEN, WHEN, WHEN, WHEN + timedelta(seconds=1), WHEN + timedelta(seconds=11)] def clock(): return moments.pop(0) @@ -649,6 +651,90 @@ def run(self, *args, **kwargs): assert reloaded["entries"][0]["worst_case"] == ledger["entries"][0]["worst_case"] +def _one_ticket(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare(_spec(), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + return incumbent, round_dir, ticket + + +def _phases(round_dir): + ledger = json.loads((round_dir / "ledgers" / "L1.json").read_text(encoding="utf-8")) + return [entry["phase"] for entry in ledger["entries"]] + + +def test_restart_after_reserve_resumes_the_same_attempt(tmp_path, monkeypatch): + incumbent, round_dir, ticket = _one_ticket(tmp_path) + + def crash(*args, **kwargs): + del args, kwargs + raise KeyboardInterrupt + + monkeypatch.setattr(parallel_round, "materialize", crash) + with pytest.raises(KeyboardInterrupt): + serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + assert _phases(round_dir) == ["reserved"] + monkeypatch.undo() + result = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + assert result["status"] == "measured" + assert _phases(round_dir) == ["finished"] + + +def test_restart_after_finish_writes_a_receipt_without_rerunning(tmp_path): + incumbent, round_dir, ticket = _one_ticket(tmp_path) + first = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + assert first["status"] == "measured" + (round_dir / "results" / f"{ticket}.json").unlink() + + class MustNotRun: + name = "must-not-run" + + def run(self, *args, **kwargs): + raise AssertionError("a finished attempt was launched again") + + second = serve_ticket(round_dir, ticket, incumbent, backend=MustNotRun(), now=WHEN) + assert second["status"] == "unsettled" + assert second["reason"] == "finished execution was not settled" + assert _phases(round_dir) == ["finished"] + + +def test_a_queued_ticket_past_its_deadline_is_rejected_without_a_reservation(tmp_path): + incumbent, round_dir, ticket = _one_ticket(tmp_path) + leases = DeviceLease(round_dir / "leases") + held = [leases.acquire("cpu-0"), leases.acquire("cpu-1")] + try: + queued = serve_ticket( + round_dir, ticket, incumbent, backend=FixtureBackend(), clock=lambda: WHEN + ) + assert queued["status"] == "queued" + late = datetime(2100, 1, 1, tzinfo=timezone.utc) + rejected = serve_ticket( + round_dir, ticket, incumbent, backend=FixtureBackend(), clock=lambda: late + ) + finally: + for lease in held: + lease.release() + assert rejected["status"] == "rejected" + assert "deadline" in rejected["reason"] + assert ticket_status(round_dir, ticket)["status"] == "rejected" + assert not (round_dir / "ledgers" / "L1.json").exists() + + +def test_reading_a_fifo_does_not_block(tmp_path): + os.mkfifo(tmp_path / "pipe") + found = [] + reader = threading.Thread(target=lambda: found.append(_read_nofollow(tmp_path, "pipe"))) + reader.daemon = True + reader.start() + reader.join(timeout=5) + assert not reader.is_alive() + assert found == [None] + + def test_completed_trial_without_a_latency_file_is_unsettled(tmp_path): incumbent = tmp_path / "incumbent" _tree(incumbent, "10") From 57618dbb953fbcd344dfef1c27586354570826ac Mon Sep 17 00:00:00 2001 From: NumberWan Date: Fri, 13 Nov 2026 12:17:42 +0800 Subject: [PATCH 15/23] fix(harness): quarantine devices whose command could not be reaped When a timed-out command's process group does not empty within the reap window, confinement raises UnreapedCommand. serve now writes quarantine/ for every device that ticket held, and hold_devices skips quarantined devices until the file is removed. Releasing the flock alone was not enough: the next ticket could be handed a device the old payload still used, and a one-shot serve process drops its flock on exit anyway. The _kill_group docstring now says what it checks: the process group is empty. Processes that called setsid are covered by --kill-child tearing down the PID namespace. Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_round.py | 14 +++++++++++++- src/inferrsi/infra/confinement.py | 12 ++++++++++-- tests/test_confinement.py | 21 +++++++++++++++++++++ tests/test_parallel_round.py | 24 ++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py index b84d8277e..bd013859d 100644 --- a/src/inferrsi/harness/parallel_round.py +++ b/src/inferrsi/harness/parallel_round.py @@ -16,6 +16,8 @@ ``device_paths`` (per device id), ``shared_device_paths``, ``runtime_paths`` and ``env`` (values may use ``{device}``) extend the confined view for GPU trials. Each command is capped by ``command_timeout_seconds`` and the lane deadline. +If a timed-out command cannot be confirmed dead, its devices are written to +``quarantine/`` and are not handed out until that file is removed. """ import hashlib @@ -45,7 +47,11 @@ seconds_remaining, ) from inferrsi.infra.artifacts import digest, write_json -from inferrsi.infra.confinement import ConfinementUnavailable, confined_execute +from inferrsi.infra.confinement import ( + ConfinementUnavailable, + UnreapedCommand, + confined_execute, +) from inferrsi.infra.device_lease import DeviceLease, LeaseError PLANES = {"harness", "model", "infra"} @@ -588,6 +594,8 @@ def hold_devices(round_dir: Path, plan: dict, count: int) -> list[tuple[str, obj for device in plan["device_ids"]: if len(held) == count: break + if (round_dir / "quarantine" / device).exists(): + continue try: held.append((device, leases.acquire(device))) except LeaseError: @@ -725,6 +733,10 @@ def _measure( backend=backend, current=current, ) + except UnreapedCommand as exc: + for device in devices: + write_json(round_dir / "quarantine" / device, {"ticket": ticket, "reason": str(exc)}) + return _record(round_dir, ticket, status="unsettled", reason=str(exc), devices=devices) except (ConfinementUnavailable, BudgetError, MeasurementError) as exc: return _record(round_dir, ticket, status="unsettled", reason=str(exc), devices=devices) finished = advance(ledger, binding, "finished", actual_cost=0) diff --git a/src/inferrsi/infra/confinement.py b/src/inferrsi/infra/confinement.py index 562f4d5ad..162362ae3 100644 --- a/src/inferrsi/infra/confinement.py +++ b/src/inferrsi/infra/confinement.py @@ -75,6 +75,10 @@ class ConfinementUnavailable(RuntimeError): """The command was not executed because confinement could not be enforced.""" +class UnreapedCommand(ConfinementUnavailable): + """A killed command may still hold its devices; they must not be handed out again.""" + + def namespaces_available() -> bool: """True only when a user namespace can actually be created. @@ -276,7 +280,11 @@ def supervisor_command(config_path: Path) -> list[str]: def _kill_group(process: subprocess.Popen) -> None: - """Kill the session and return only once no member of it is left.""" + """Kill the session and return once its process group is empty. + + Processes that called setsid are not in the group; ``--kill-child`` covers + them, because killing PID 1 tears down the whole PID namespace. + """ try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: @@ -292,7 +300,7 @@ def _kill_group(process: subprocess.Popen) -> None: except ProcessLookupError: return if time.monotonic() > give_up: - raise ConfinementUnavailable("namespace command outlived its deadline") + raise UnreapedCommand("namespace command outlived its deadline") time.sleep(0.05) diff --git a/tests/test_confinement.py b/tests/test_confinement.py index 34c3c4371..8e8d7e3db 100644 --- a/tests/test_confinement.py +++ b/tests/test_confinement.py @@ -15,8 +15,10 @@ ConfinementUnavailable, FixtureBackend, UnavailableBackend, + UnreapedCommand, UnshareBackend, _elf_interpreter, + _kill_group, confined_execute, minimal_environment, namespace_plan, @@ -141,6 +143,25 @@ def fake_popen(argv, **kwargs): assert not (tmp_path / "late").exists() +def test_a_group_that_never_empties_is_reported_unreaped(monkeypatch): + process = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(30)"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + ) + real_killpg = os.killpg + + def killpg(pgid, sig): + if sig != 0: + real_killpg(pgid, sig) + + monkeypatch.setattr(confinement, "_REAP_SECONDS", 0.2) + monkeypatch.setattr(confinement.os, "killpg", killpg) + with pytest.raises(UnreapedCommand): + _kill_group(process) + + def test_supervisor_ignores_a_shadow_package_in_the_candidate(tmp_path): shadow = tmp_path / "candidate" (shadow / "inferrsi" / "infra").mkdir(parents=True) diff --git a/tests/test_parallel_round.py b/tests/test_parallel_round.py index ba08ac4ca..e669dcdfb 100644 --- a/tests/test_parallel_round.py +++ b/tests/test_parallel_round.py @@ -14,6 +14,7 @@ from inferrsi.harness.parallel_round import ( RoundPlanError, _read_nofollow, + hold_devices, prepare, serve_ticket, submit, @@ -22,6 +23,7 @@ from inferrsi.infra.confinement import ( ConfinementUnavailable, FixtureBackend, + UnreapedCommand, namespaces_available, ) from inferrsi.infra.device_lease import DeviceLease, LeaseError @@ -724,6 +726,28 @@ def test_a_queued_ticket_past_its_deadline_is_rejected_without_a_reservation(tmp assert not (round_dir / "ledgers" / "L1.json").exists() +def test_an_unreaped_command_quarantines_its_device(tmp_path): + incumbent, round_dir, ticket = _one_ticket(tmp_path) + + class Unreaped: + name = "unreaped" + + def run(self, *args, **kwargs): + del args, kwargs + raise UnreapedCommand("namespace command outlived its deadline") + + result = serve_ticket(round_dir, ticket, incumbent, backend=Unreaped(), now=WHEN) + assert result["status"] == "unsettled" + assert result["devices"] == ["cpu-0"] + assert (round_dir / "quarantine" / "cpu-0").exists() + plan = json.loads((round_dir / "round-plan.json").read_text(encoding="utf-8")) + assert hold_devices(round_dir, plan, 2) is None + held = hold_devices(round_dir, plan, 1) + assert [device for device, _lease in held] == ["cpu-1"] + for _device, lease in held: + lease.release() + + def test_reading_a_fifo_does_not_block(tmp_path): os.mkfifo(tmp_path / "pipe") found = [] From 0c3c94d3bbc6f14e44cf365518ad529f3fbf8ef4 Mon Sep 17 00:00:00 2001 From: NumberWan Date: Fri, 13 Nov 2026 12:24:11 +0800 Subject: [PATCH 16/23] fix(confinement): kill the namespace PID 1 directly on timeout The confined command is PID 1 of its namespace. If it calls setsid it leaves unshare's process group, so killing the group took down unshare only; _kill_group then saw an empty group and returned while PID 1 could still hold the device node, and serve released the lease. On timeout, _kill_group now records unshare's direct children (PID 1) with their start times, kills the group and each child, and returns only once the group is empty and every recorded child is gone or a zombie. A zombie PID 1 has already outlived every other process in its namespace. If that does not happen within the reap window, UnreapedCommand is raised and serve quarantines the devices. Signed-off-by: NumberWan --- src/inferrsi/infra/confinement.py | 42 ++++++++++++++++++++++-- tests/test_confinement.py | 53 +++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/src/inferrsi/infra/confinement.py b/src/inferrsi/infra/confinement.py index 162362ae3..b42be5ab5 100644 --- a/src/inferrsi/infra/confinement.py +++ b/src/inferrsi/infra/confinement.py @@ -279,16 +279,49 @@ def supervisor_command(config_path: Path) -> list[str]: return [sys.executable, "-I", "-c", _SUPERVISOR, _PACKAGE_ROOT, str(config_path)] +def _proc_stat(pid: int) -> list[str] | None: + """Fields of /proc//stat after the command name: state, ppid, ...""" + try: + text = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8") + except OSError: + return None + return text.rsplit(")", 1)[1].split() + + +def _children(pid: int) -> list[tuple[int, str]]: + """Direct children of ``pid`` with their start times.""" + found = [] + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + fields = _proc_stat(int(entry.name)) + if fields is not None and fields[1] == str(pid): + found.append((int(entry.name), fields[19])) + return found + + +def _running(pid: int, started: str) -> bool: + """A zombie PID 1 has already outlived every other process in its namespace.""" + fields = _proc_stat(pid) + return fields is not None and fields[19] == started and fields[0] != "Z" + + def _kill_group(process: subprocess.Popen) -> None: - """Kill the session and return once its process group is empty. + """Kill unshare, its session and the namespace's PID 1; return once all are gone. - Processes that called setsid are not in the group; ``--kill-child`` covers - them, because killing PID 1 tears down the whole PID namespace. + PID 1 is found and killed directly because it may have left the session + with setsid. Killing PID 1 tears down the whole PID namespace. """ + inits = _children(process.pid) try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass + for pid, _started in inits: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass process.wait() for stream in (process.stdout, process.stderr): if stream is not None: @@ -297,7 +330,10 @@ def _kill_group(process: subprocess.Popen) -> None: while True: try: os.killpg(process.pid, 0) + group_left = True except ProcessLookupError: + group_left = False + if not group_left and not any(_running(pid, started) for pid, started in inits): return if time.monotonic() > give_up: raise UnreapedCommand("namespace command outlived its deadline") diff --git a/tests/test_confinement.py b/tests/test_confinement.py index 8e8d7e3db..28d1ad6ee 100644 --- a/tests/test_confinement.py +++ b/tests/test_confinement.py @@ -143,6 +143,35 @@ def fake_popen(argv, **kwargs): assert not (tmp_path / "late").exists() +def test_unshare_timeout_kills_a_direct_child_that_left_the_session(tmp_path, monkeypatch): + real_popen = subprocess.Popen + (tmp_path / "child.py").write_text(_LATE_CHILD, encoding="utf-8") + child = [str(tmp_path / "child.py"), str(tmp_path), "1.5"] + payload = ( + "import subprocess, sys, time\n" + f"subprocess.Popen([sys.executable, *{child!r}], start_new_session=True)\n" + "time.sleep(30)\n" + ) + + def fake_popen(argv, **kwargs): + if argv and argv[0] == "unshare": + argv = [sys.executable, "-c", payload] + return real_popen(argv, **kwargs) + + monkeypatch.setattr("inferrsi.infra.confinement.subprocess.Popen", fake_popen) + with pytest.raises(ConfinementUnavailable, match="timed out"): + UnshareBackend().run( + (sys.executable, "-c", "print('unused')"), + tmp_path, + (tmp_path,), + (), + time.monotonic() + 1, + ) + time.sleep(2) + assert (tmp_path / "started").exists() + assert not (tmp_path / "late").exists() + + def test_a_group_that_never_empties_is_reported_unreaped(monkeypatch): process = subprocess.Popen( [sys.executable, "-c", "import time; time.sleep(30)"], @@ -257,6 +286,30 @@ def test_unshare_timeout_leaves_no_detached_child(tmp_path): assert not (work / "late").exists() +@pytest.mark.skipif(not namespaces_available(), reason="user namespaces are unavailable") +def test_unshare_timeout_kills_a_pid_1_that_left_the_session(tmp_path): + work = tmp_path / "work" + work.mkdir() + (work / "child.py").write_text(_LATE_CHILD, encoding="utf-8") + script = ( + "import os, subprocess, sys, time\n" + "os.setsid()\n" + f"subprocess.Popen([sys.executable, {str(work / 'child.py')!r}, {str(work)!r}, '4'])\n" + "time.sleep(30)\n" + ) + with pytest.raises(ConfinementUnavailable, match="timed out"): + UnshareBackend().run( + (sys.executable, "-c", script), + work, + (work,), + (), + time.monotonic() + 3, + ) + time.sleep(3) + assert (work / "started").exists() + assert not (work / "late").exists() + + def test_blocked_unshare_reports_namespaces_unavailable(monkeypatch): real = Path.read_text From 1836d8f36fe1504b5517e18c8a636614a98c52de Mon Sep 17 00:00:00 2001 From: NumberWan Date: Fri, 13 Nov 2026 12:30:47 +0800 Subject: [PATCH 17/23] fix(harness): keep the quarantine marker in the device lease store Quarantine now lives next to the lease file as leases/.quarantined, written by DeviceLease.quarantine while the lease is still held. DeviceLease.acquire checks the marker after taking the flock and refuses the device until the file is removed, so every acquire through the lease store sees it, not only hold_devices. Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_round.py | 10 +++++----- src/inferrsi/infra/device_lease.py | 13 +++++++++++++ tests/test_parallel_round.py | 6 +++++- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py index bd013859d..a85ea7745 100644 --- a/src/inferrsi/harness/parallel_round.py +++ b/src/inferrsi/harness/parallel_round.py @@ -16,8 +16,9 @@ ``device_paths`` (per device id), ``shared_device_paths``, ``runtime_paths`` and ``env`` (values may use ``{device}``) extend the confined view for GPU trials. Each command is capped by ``command_timeout_seconds`` and the lane deadline. -If a timed-out command cannot be confirmed dead, its devices are written to -``quarantine/`` and are not handed out until that file is removed. +If a timed-out command cannot be confirmed dead, its devices are quarantined: +``leases/.quarantined`` is written before the lease is released, and +the device is refused until an operator removes that file. """ import hashlib @@ -594,8 +595,6 @@ def hold_devices(round_dir: Path, plan: dict, count: int) -> list[tuple[str, obj for device in plan["device_ids"]: if len(held) == count: break - if (round_dir / "quarantine" / device).exists(): - continue try: held.append((device, leases.acquire(device))) except LeaseError: @@ -734,8 +733,9 @@ def _measure( current=current, ) except UnreapedCommand as exc: + leases = DeviceLease(round_dir / "leases") for device in devices: - write_json(round_dir / "quarantine" / device, {"ticket": ticket, "reason": str(exc)}) + leases.quarantine(device, f"{ticket}: {exc}") return _record(round_dir, ticket, status="unsettled", reason=str(exc), devices=devices) except (ConfinementUnavailable, BudgetError, MeasurementError) as exc: return _record(round_dir, ticket, status="unsettled", reason=str(exc), devices=devices) diff --git a/src/inferrsi/infra/device_lease.py b/src/inferrsi/infra/device_lease.py index 7c006f042..d7a9bceb7 100644 --- a/src/inferrsi/infra/device_lease.py +++ b/src/inferrsi/infra/device_lease.py @@ -37,4 +37,17 @@ def acquire(self, device_id: str) -> _Held: except BlockingIOError: handle.close() raise LeaseError(f"device busy: {device_id}") from None + if self._marker(device_id).exists(): + _Held(handle).release() + raise LeaseError(f"device quarantined: {device_id}") return _Held(handle) + + def quarantine(self, device_id: str, reason: str) -> None: + """Refuse every later acquire of ``device_id`` until the marker file is removed. + + Call it while holding the lease, so no acquire can slip in before the marker. + """ + self._marker(device_id).write_text(reason + "\n", encoding="utf-8") + + def _marker(self, device_id: str) -> Path: + return self.directory / f"{device_id}.quarantined" diff --git a/tests/test_parallel_round.py b/tests/test_parallel_round.py index e669dcdfb..027fd6d40 100644 --- a/tests/test_parallel_round.py +++ b/tests/test_parallel_round.py @@ -739,13 +739,17 @@ def run(self, *args, **kwargs): result = serve_ticket(round_dir, ticket, incumbent, backend=Unreaped(), now=WHEN) assert result["status"] == "unsettled" assert result["devices"] == ["cpu-0"] - assert (round_dir / "quarantine" / "cpu-0").exists() + assert (round_dir / "leases" / "cpu-0.quarantined").exists() + with pytest.raises(LeaseError, match="quarantined"): + DeviceLease(round_dir / "leases").acquire("cpu-0") plan = json.loads((round_dir / "round-plan.json").read_text(encoding="utf-8")) assert hold_devices(round_dir, plan, 2) is None held = hold_devices(round_dir, plan, 1) assert [device for device, _lease in held] == ["cpu-1"] for _device, lease in held: lease.release() + (round_dir / "leases" / "cpu-0.quarantined").unlink() + DeviceLease(round_dir / "leases").acquire("cpu-0").release() def test_reading_a_fifo_does_not_block(tmp_path): From 41d6fa8ee3ac08552dab3ff0d5e36d1df7b2f50d Mon Sep 17 00:00:00 2001 From: NumberWan Date: Fri, 13 Nov 2026 12:37:10 +0800 Subject: [PATCH 18/23] fix(confinement): re-check PID 1 start time before killing it _kill_group signals a recorded namespace PID 1 only if /proc still shows the same start time and it is not a zombie, so a recycled pid is left alone. _proc_stat returns None for a short /proc read instead of raising before the kill. Signed-off-by: NumberWan --- src/inferrsi/infra/confinement.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/inferrsi/infra/confinement.py b/src/inferrsi/infra/confinement.py index b42be5ab5..c64a605e1 100644 --- a/src/inferrsi/infra/confinement.py +++ b/src/inferrsi/infra/confinement.py @@ -285,7 +285,8 @@ def _proc_stat(pid: int) -> list[str] | None: text = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8") except OSError: return None - return text.rsplit(")", 1)[1].split() + fields = text.rsplit(")", 1)[-1].split() + return fields if len(fields) > 19 else None def _children(pid: int) -> list[tuple[int, str]]: @@ -317,7 +318,9 @@ def _kill_group(process: subprocess.Popen) -> None: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass - for pid, _started in inits: + for pid, started in inits: + if not _running(pid, started): + continue try: os.kill(pid, signal.SIGKILL) except ProcessLookupError: From 2e2d601c57b265f206020016f4d475986bcca25c Mon Sep 17 00:00:00 2001 From: NumberWan Date: Fri, 13 Nov 2026 14:13:56 +0800 Subject: [PATCH 19/23] refactor(harness): measure parallel-round tickets with run_experiment serve no longer has its own ABBA loop. After the confined build, a ticket is measured by run_experiment with a confined execute, and evaluate gives the verdict, as issue #249 asks (reuse run_experiment, evaluate and qualify_devices; no second runner). - The trial adapter keeps the lane deadline, command timeout, device view and read-only repository; an unreaped command still quarantines the held devices. - The receipt carries verdict, improvement_pct, reasons, medians, samples and run_dir; a ticket without a complete metric set stays unsettled. - Optional plan fields: validation (run once on the candidate), metric.min_improvement_pct (default 0), and device_model, which runs qualify_devices on the held devices before an attempt is reserved and leaves the ticket queued when the check fails. Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_round.py | 311 ++++++++++++++++--------- tests/test_parallel_round.py | 111 ++++++++- 2 files changed, 307 insertions(+), 115 deletions(-) diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py index a85ea7745..5d7383a89 100644 --- a/src/inferrsi/harness/parallel_round.py +++ b/src/inferrsi/harness/parallel_round.py @@ -6,12 +6,18 @@ Each lane declares ``device_count`` (default 1). The counts of all lanes must fit in ``device_ids``. A ticket runs only after it holds that many devices at once. -Command placeholders: ``build`` gets ``{device}`` only. ``trial`` gets ``{device}`` -and ``{result}``; it must write ``{"metrics": {: }}`` -to ``{result}``. The candidate is built once, then candidate and incumbent trials -run ``metric.repeats`` times each in ABBA order; the receipt keeps every sample -and the medians. ``{device}`` is the held device identities joined with commas. -Other text, including ``{result}`` inside ``build``, is passed through unchanged. +Command placeholders: ``build`` and ``validation`` get ``{device}`` only. ``trial`` +gets ``{device}`` and ``{result}``; it must write +``{"metrics": {: }}`` to ``{result}``. The +candidate is built once in confinement; measurement is ``run_experiment`` with a +confined ``execute``, so incumbent and candidate trials alternate AB/BA +``metric.repeats`` times each, the optional ``validation`` runs once on the +candidate, and ``evaluate`` gives the verdict against ``metric.min_improvement_pct``. +``{device}`` is the held device identities joined with commas. Other text, +including ``{result}`` inside ``build``, is passed through unchanged. + +With ``device_model`` set, ``qualify_devices`` checks the held devices before an +attempt is reserved; a device that fails the check leaves the ticket queued. ``device_paths`` (per device id), ``shared_device_paths``, ``runtime_paths`` and ``env`` (values may use ``{device}``) extend the confined view for GPU trials. @@ -27,7 +33,6 @@ import os import shutil import stat -import statistics import subprocess import time from datetime import datetime, timezone @@ -47,6 +52,7 @@ ensure_open, seconds_remaining, ) +from inferrsi.harness.runner import run_experiment from inferrsi.infra.artifacts import digest, write_json from inferrsi.infra.confinement import ( ConfinementUnavailable, @@ -54,6 +60,9 @@ confined_execute, ) from inferrsi.infra.device_lease import DeviceLease, LeaseError +from inferrsi.infra.platforms.cuda import qualify_devices +from inferrsi.model.protocols.metrics import read_metric +from inferrsi.runspec import RunSpec PLANES = {"harness", "model", "infra"} DIRECTIONS = {"lower", "higher"} @@ -221,7 +230,9 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: metric = spec.get("metric") if ( not isinstance(metric, dict) - or not {"name", "better"} <= set(metric) <= {"name", "better", "repeats"} + or not {"name", "better"} + <= set(metric) + <= {"name", "better", "repeats", "min_improvement_pct"} or not isinstance(metric["name"], str) or not metric["name"] or not isinstance(metric["better"], str) @@ -231,6 +242,19 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: repeats = metric.get("repeats", 1) if type(repeats) is not int or repeats < 1: raise RoundPlanError("metric repeats must be a positive integer") + threshold = metric.get("min_improvement_pct", 0) + if type(threshold) not in (int, float) or not math.isfinite(threshold) or threshold < 0: + raise RoundPlanError("metric min_improvement_pct must be a finite non-negative number") + validation = spec.get("validation") + if validation is not None and ( + not isinstance(validation, list) + or not validation + or any(type(item) is not str for item in validation) + ): + raise RoundPlanError("validation must be a non-empty argv list") + device_model = spec.get("device_model") + if device_model is not None and (type(device_model) is not str or not device_model): + raise RoundPlanError("device_model must be a non-empty string") timeout = spec.get("command_timeout_seconds", 30) if type(timeout) not in (int, float) or not math.isfinite(timeout) or timeout <= 0: raise RoundPlanError("command_timeout_seconds must be a positive number") @@ -303,6 +327,7 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: "name": metric["name"], "better": metric["better"], "repeats": repeats, + "min_improvement_pct": threshold, }, "command_timeout_seconds": timeout, "device_paths": {device: list(device_paths[device]) for device in device_paths}, @@ -313,6 +338,8 @@ def prepare(spec: dict, incumbent: Path, round_dir: Path) -> dict: "protected": protected, "build": list(spec.get("build", [])), "trial": list(spec.get("trial", [])), + "validation": list(validation) if validation is not None else None, + "device_model": device_model, "lanes": normalized, } body["round_digest"] = digest(body) @@ -446,13 +473,6 @@ def _command(template: list[str], result: Path | None, device: str) -> tuple[str return tuple(part.replace("{result}", str(result)) for part in parts) -def _metric_value(path: Path, name: str) -> float: - value = json.loads(path.read_text(encoding="utf-8"))["metrics"][name] - if type(value) not in (int, float) or not math.isfinite(value): - raise ValueError(f"{name} is not a finite number") - return value - - def _record(round_dir: Path, ticket: str, **fields) -> dict: body = {"ticket": ticket, "synthetic": True, **fields} write_json(round_dir / "results" / f"{ticket}.json", body) @@ -467,8 +487,17 @@ def _ticket_entry(ledger, ticket: str): return None -class MeasurementError(ValueError): - """A trial finished without a readable metric value.""" +def _device_view(plan: dict, devices: list[str]) -> dict: + """Confined-view extras for the held devices: their nodes, the runtime and the env.""" + device = ",".join(devices) + nodes = list(plan["shared_device_paths"]) + for held in devices: + nodes.extend(plan["device_paths"].get(held, [])) + return { + "device_paths": tuple(Path(path) for path in nodes), + "runtime_paths": tuple(Path(path) for path in plan["runtime_paths"]), + "env": {key: value.replace("{device}", device) for key, value in plan["env"].items()}, + } def _run_confined( @@ -487,10 +516,6 @@ def _run_confined( remaining = plan["command_timeout_seconds"] if deadline_utc is not None: remaining = min(seconds_remaining(deadline_utc, current()), remaining) - device = ",".join(devices) - nodes = list(plan["shared_device_paths"]) - for held in devices: - nodes.extend(plan["device_paths"].get(held, [])) try: record = confined_execute( argv, @@ -499,9 +524,7 @@ def _run_confined( readable, time.monotonic() + remaining, backend=backend, - device_paths=tuple(Path(path) for path in nodes), - runtime_paths=tuple(Path(path) for path in plan["runtime_paths"]), - env={key: value.replace("{device}", device) for key, value in plan["env"].items()}, + **_device_view(plan, devices), ) except subprocess.TimeoutExpired as exc: raise ConfinementUnavailable(f"{stage} timed out") from exc @@ -509,83 +532,113 @@ def _run_confined( raise ConfinementUnavailable(f"{stage} {record.get('status')}") -def compare( - plan: dict, - candidate: Path, - incumbent: Path, - trial_dir: Path, - devices: list[str], - deadline_utc: str | None, - *, - backend, - current, -) -> tuple[list[float], list[float]]: - """Build the candidate once, then run ``repeats`` trial pairs in ABBA order. +def _confined_trial( + plan: dict, devices: list[str], deadline_utc: str, *, backend, current, failures: list[str] +): + """``execute`` for ``run_experiment``: one command in confinement, never past the lane deadline. - Raises ``ConfinementUnavailable`` or ``BudgetError`` when a command cannot run, and - ``MeasurementError`` when a finished trial left no readable metric. + The trial directory is the only writable path and the repository is read-only. + An unreaped command propagates so the caller can quarantine its devices. """ - device = ",".join(devices) - build_dir = candidate / "build-output" - build_dir.mkdir(parents=True, exist_ok=True) - trial_dir.mkdir(parents=True, exist_ok=True) - common = {"backend": backend, "current": current} - if plan["build"]: - _run_confined( - plan, - "build", - _command(plan["build"], None, device), - candidate, - (candidate, build_dir), - (), - devices, - deadline_utc, - **common, - ) - sides = {"candidate": candidate, "incumbent": incumbent} - samples: dict[str, list[float]] = {"candidate": [], "incumbent": []} - name = plan["metric"]["name"] - for index in range(plan["metric"]["repeats"]): - order = ("candidate", "incumbent") if index % 2 == 0 else ("incumbent", "candidate") - for side in order: - result = trial_dir / f"{side}-{index}.json" - _run_confined( - plan, - side, - _command(plan["trial"], result, device), - sides[side], + view = _device_view(plan, devices) + + def execute(command, cwd, trial_dir, deadline, metric_key): + trial_dir.mkdir(parents=True) + result = trial_dir / "result.json" + argv = tuple(part.replace("{result}", str(result)) for part in command) + record = { + "status": "pending", + "argv": list(argv), + "cwd": str(cwd), + "returncode": None, + "result_path": str(result), + "metric": None, + "metric_error": None, + } + try: + remaining = min( + deadline - time.monotonic(), + plan["command_timeout_seconds"], + seconds_remaining(deadline_utc, current()), + ) + if remaining <= 0: + raise BudgetError("run deadline passed") + outcome = confined_execute( + argv, + cwd, (trial_dir,), - (sides[side],), - devices, - deadline_utc, - **common, + (cwd,), + time.monotonic() + remaining, + backend=backend, + **view, ) + record.update(status=outcome.get("status"), returncode=outcome.get("returncode")) + for stream in ("stdout", "stderr"): + (trial_dir / f"{stream}.log").write_text( + outcome.get(stream) or "", encoding="utf-8" + ) + except UnreapedCommand: + raise + except BudgetError as exc: + record.update(status="timeout", error=str(exc)) + except subprocess.TimeoutExpired: + record.update(status="timeout", error="command timed out") + except ConfinementUnavailable as exc: + status = "timeout" if "timed out" in str(exc) else "failed" + record.update(status=status, error=str(exc)) + if record["status"] == "completed" and metric_key is not None: try: - samples[side].append(_metric_value(result, name)) - except (OSError, json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: - raise MeasurementError( - f"trial finished without a readable {name} result: {exc}" - ) from exc - return samples["candidate"], samples["incumbent"] - - -def summarize(plan: dict, candidate_samples: list[float], incumbent_samples: list[float]) -> dict: - """Medians and a gain where positive always means the candidate is better.""" - candidate_value = statistics.median(candidate_samples) - incumbent_value = statistics.median(incumbent_samples) - gain = incumbent_value - candidate_value - if plan["metric"]["better"] == "higher": - gain = -gain - return { - "metric": plan["metric"]["name"], - "better": plan["metric"]["better"], - "candidate_samples": candidate_samples, - "incumbent_samples": incumbent_samples, - "candidate": candidate_value, - "incumbent": incumbent_value, - "gain": gain, - "relative_gain": gain / abs(incumbent_value) if incumbent_value else None, - } + document = json.loads(result.read_text(encoding="utf-8")) + record["metric"] = read_metric(document, metric_key) + except (OSError, ValueError, TypeError) as exc: + record["metric_error"] = str(exc) + if record.get("error") or record["metric_error"]: + failures.append(record.get("error") or record["metric_error"]) + write_json(trial_dir / "execution.json", record) + return record + + return execute + + +def _run_spec( + plan: dict, + candidate: Path, + incumbent: Path, + devices: list[str], + output_dir: Path, + seconds_left: float, +) -> RunSpec: + device = ",".join(devices) + trial = _command(plan["trial"], None, device) + validation = plan.get("validation") + return RunSpec( + repo=str(incumbent), + candidate_repo=str(candidate), + baseline_command=trial, + candidate_command=trial, + validation_command=_command(validation, None, device) if validation else None, + metric_key=f"metrics.{plan['metric']['name']}", + metric_unit="metric", + direction="minimize" if plan["metric"]["better"] == "lower" else "maximize", + repetitions=plan["metric"]["repeats"], + min_improvement_pct=plan["metric"].get("min_improvement_pct", 0), + max_minutes=min(seconds_left, 7 * 24 * 3600) / 60, + device_ids=tuple(devices), + output_dir=str(output_dir), + ) + + +def _qualify(device_model: str, devices: list[str]) -> None: + """``qualify_devices`` reads the caller's reservation from ``CUDA_VISIBLE_DEVICES``.""" + previous = os.environ.get("CUDA_VISIBLE_DEVICES") + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(devices) + try: + qualify_devices(count=len(devices), device_model=device_model) + finally: + if previous is None: + os.environ.pop("CUDA_VISIBLE_DEVICES", None) + else: + os.environ["CUDA_VISIBLE_DEVICES"] = previous def hold_devices(round_dir: Path, plan: dict, count: int) -> list[tuple[str, object]] | None: @@ -669,6 +722,11 @@ def current() -> datetime: if held is None: return {"ticket": ticket, "status": "queued"} try: + if plan.get("device_model"): + try: + _qualify(plan["device_model"], [device for device, _lease in held]) + except (ValueError, OSError, subprocess.SubprocessError) as exc: + return {"ticket": ticket, "status": "queued", "reason": str(exc)} return _measure( round_dir, plan, @@ -712,7 +770,6 @@ def _measure( return _record(round_dir, ticket, status="rejected", reason=str(exc)) _save_ledger(round_dir, ledger) candidate = round_dir / "candidates" / ticket - trial_dir = round_dir / "trials" / ticket materialize(incumbent, [overlay], candidate) try: seconds_remaining(lane["deadline_utc"], current()) @@ -721,24 +778,61 @@ def _measure( return _record(round_dir, ticket, status="rejected", reason=str(exc)) ledger = advance(ledger, binding, "claimed") _save_ledger(round_dir, ledger) + failures: list[str] = [] try: - candidate_samples, incumbent_samples = compare( + if plan["build"]: + build_dir = candidate / "build-output" + build_dir.mkdir(parents=True, exist_ok=True) + _run_confined( + plan, + "build", + _command(plan["build"], None, ",".join(devices)), + candidate, + (candidate, build_dir), + (), + devices, + lane["deadline_utc"], + backend=backend, + current=current, + ) + spec = _run_spec( plan, candidate, incumbent, - trial_dir, devices, - lane["deadline_utc"], - backend=backend, - current=current, + round_dir / "runs" / ticket, + seconds_remaining(lane["deadline_utc"], current()), + ) + run_dir = run_experiment( + spec, + {"round_id": plan["round_id"], "ticket": ticket, "lane_id": lane["id"]}, + execute=_confined_trial( + plan, + devices, + lane["deadline_utc"], + backend=backend, + current=current, + failures=failures, + ), ) except UnreapedCommand as exc: leases = DeviceLease(round_dir / "leases") for device in devices: leases.quarantine(device, f"{ticket}: {exc}") return _record(round_dir, ticket, status="unsettled", reason=str(exc), devices=devices) - except (ConfinementUnavailable, BudgetError, MeasurementError) as exc: + except (ConfinementUnavailable, BudgetError, ValueError) as exc: return _record(round_dir, ticket, status="unsettled", reason=str(exc), devices=devices) + evaluation = json.loads((run_dir / "evaluation.json").read_text(encoding="utf-8")) + if evaluation["improvement_pct"] is None: + reason = failures[-1] if failures else "; ".join(evaluation["reasons"]) + return _record( + round_dir, + ticket, + status="unsettled", + reason=reason, + devices=devices, + run_dir=str(run_dir), + ) finished = advance(ledger, binding, "finished", actual_cost=0) _save_ledger(round_dir, finished) return _record( @@ -746,7 +840,16 @@ def _measure( ticket, status="measured", lane_id=lane["id"], - **summarize(plan, candidate_samples, incumbent_samples), + metric=plan["metric"]["name"], + better=plan["metric"]["better"], + verdict=evaluation["verdict"], + improvement_pct=evaluation["improvement_pct"], + reasons=evaluation["reasons"], + candidate=evaluation["candidate"]["median"], + incumbent=evaluation["baseline"]["median"], + candidate_samples=evaluation["candidate"]["values"], + incumbent_samples=evaluation["baseline"]["values"], + run_dir=str(run_dir), devices=devices, confinement=getattr(backend, "name", "unknown"), ) diff --git a/tests/test_parallel_round.py b/tests/test_parallel_round.py index 027fd6d40..ff96b7b54 100644 --- a/tests/test_parallel_round.py +++ b/tests/test_parallel_round.py @@ -208,7 +208,9 @@ def test_serve_records_a_synthetic_gain_and_a_held_device_stays_queued(tmp_path) assert measured["metric"] == "latency_ms" assert measured["candidate"] == 8 assert measured["incumbent"] == 10 - assert measured["gain"] == 2 + assert measured["improvement_pct"] == pytest.approx(20) + assert measured["verdict"] == "INCONCLUSIVE" + assert (Path(measured["run_dir"]) / "evaluation.json").exists() assert measured["devices"] == ["cpu-0"] assert measured["confinement"] == "fixture" assert (round_dir / "candidates" / ticket / "build-stamp").read_text(encoding="utf-8") == "ok" @@ -269,14 +271,25 @@ def test_prepare_rejects_bad_repeats_and_timeout(tmp_path): ({"repeats": True}, "repeats"), ({"margin": 0.03}, "metric needs"), ({"spread": 1}, "metric needs"), + ({"min_improvement_pct": -1}, "min_improvement_pct"), + ({"min_improvement_pct": float("nan")}, "min_improvement_pct"), ): with pytest.raises(RoundPlanError, match=message): prepare(_spec(metric={**base, **extra}), incumbent, tmp_path / "round-metric") for timeout in (0, -1, "30", float("inf")): with pytest.raises(RoundPlanError, match="command_timeout_seconds"): prepare(_spec(command_timeout_seconds=timeout), incumbent, tmp_path / "round-t") + for overrides, message in ( + ({"validation": "pytest"}, "validation"), + ({"validation": []}, "validation"), + ({"device_model": ""}, "device_model"), + ): + with pytest.raises(RoundPlanError, match=message): + prepare(_spec(**overrides), incumbent, tmp_path / "round-fields") plan = prepare(_spec(), incumbent, tmp_path / "round-defaults") - assert plan["metric"] == {**base, "repeats": 1} + assert plan["metric"] == {**base, "repeats": 1, "min_improvement_pct": 0} + assert plan["validation"] is None + assert plan["device_model"] is None assert plan["command_timeout_seconds"] == 30 @@ -342,7 +355,7 @@ def test_repeats_alternate_abba_and_report_medians(tmp_path): work = tmp_path / "work" _tree(work, "8") ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] - values = {"candidate": [8, 100, 9], "incumbent": [10, 10, 1000]} + values = {"candidate": [8, 100, 9], "baseline": [10, 10, 1000]} order = [] class Recording: @@ -351,21 +364,25 @@ class Recording: def run(self, argv, *args, **kwargs): del args, kwargs result = Path(argv[-1]) - order.append(result.stem) - side = result.stem.split("-")[0] + side = result.parent.parent.name + order.append(f"{side}-{result.parent.name}") body = {"metrics": {"latency_ms": values[side].pop(0)}} result.write_text(json.dumps(body), encoding="utf-8") return {"status": "completed", "stdout": "", "stderr": ""} measured = serve_ticket(round_dir, ticket, incumbent, backend=Recording(), now=WHEN) assert order == [ - "candidate-0", "incumbent-0", "incumbent-1", "candidate-1", "candidate-2", "incumbent-2" + "baseline-trial-001", + "candidate-trial-001", + "candidate-trial-002", + "baseline-trial-002", + "baseline-trial-003", + "candidate-trial-003", ] assert measured["candidate_samples"] == [8, 100, 9] assert measured["candidate"] == 9 assert measured["incumbent"] == 10 - assert measured["gain"] == 1 - assert measured["relative_gain"] == pytest.approx(0.1) + assert measured["improvement_pct"] == pytest.approx(10) def test_higher_is_better_metric_reports_gain_in_the_better_direction(tmp_path): @@ -379,7 +396,7 @@ def test_higher_is_better_metric_reports_gain_in_the_better_direction(tmp_path): measured = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) assert measured["status"] == "measured" assert measured["better"] == "higher" - assert measured["gain"] == -2 + assert measured["improvement_pct"] == pytest.approx(-20) def test_serve_uses_a_free_device_and_passes_its_identity_to_the_commands(tmp_path): @@ -445,7 +462,7 @@ def run(self, argv, *args, **kwargs): result = serve_ticket(round_dir, ticket, incumbent, backend=WritesText(), now=WHEN) assert result["status"] == "unsettled" - assert "not a finite number" in result["reason"] + assert "must be a number" in result["reason"] ledger = json.loads((round_dir / "ledgers" / "L1.json").read_text(encoding="utf-8")) assert [entry["phase"] for entry in ledger["entries"]] == ["claimed"] @@ -783,10 +800,82 @@ def run(self, *args, **kwargs): round_dir, ticket, incumbent, backend=CompletesWithoutResult(), now=WHEN ) assert result["status"] == "unsettled" - assert "readable latency_ms result" in result["reason"] + assert "result.json" in result["reason"] assert ticket_status(round_dir, ticket)["status"] == "unsettled" +def _git_tree(root: Path, knob: str) -> None: + _tree(root, knob) + for argv in ( + ["init", "-q"], + ["add", "-A"], + ["-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "incumbent"], + ): + subprocess.run(["git", "-C", str(root), *argv], check=True) + + +def test_a_git_incumbent_with_validation_gets_a_verdict(tmp_path): + incumbent = tmp_path / "incumbent" + _git_tree(incumbent, "10") + work = tmp_path / "work" + _tree(work, "8") + passing = [sys.executable, "-c", "pass"] + for threshold, verdict in ((3, "PASS"), (25, "FAIL")): + round_dir = tmp_path / f"round-{threshold}" + metric = {"name": "latency_ms", "better": "lower", "min_improvement_pct": threshold} + prepare(_spec(metric=metric, validation=passing), incumbent, round_dir) + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + measured = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + assert measured["status"] == "measured" + assert measured["verdict"] == verdict, measured["reasons"] + assert measured["improvement_pct"] == pytest.approx(20) + + +def test_a_failing_validation_is_a_measured_fail(tmp_path): + incumbent = tmp_path / "incumbent" + _git_tree(incumbent, "10") + round_dir = tmp_path / "round" + failing = [sys.executable, "-c", "raise SystemExit(1)"] + prepare(_spec(validation=failing), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + measured = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + assert measured["status"] == "measured" + assert measured["verdict"] == "FAIL" + ledger = json.loads((round_dir / "ledgers" / "L1.json").read_text(encoding="utf-8")) + assert [entry["phase"] for entry in ledger["entries"]] == ["finished"] + + +def test_device_model_checks_the_held_devices_before_reserving(tmp_path, monkeypatch): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare(_spec(device_ids=["0", "1"], device_model="L20X"), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + seen = [] + + def busy(*, count, device_model): + seen.append((count, device_model, os.environ.get("CUDA_VISIBLE_DEVICES"))) + raise ValueError("An allocated GPU already has a compute process") + + monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False) + monkeypatch.setattr(parallel_round, "qualify_devices", busy) + queued = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + assert queued["status"] == "queued" + assert "compute process" in queued["reason"] + assert seen == [(1, "L20X", "0")] + assert "CUDA_VISIBLE_DEVICES" not in os.environ + assert not (round_dir / "ledgers").exists() + DeviceLease(round_dir / "leases").acquire("0").release() + + monkeypatch.setattr(parallel_round, "qualify_devices", lambda **kwargs: []) + measured = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + assert measured["status"] == "measured" + + def test_serve_cli_runs_without_a_caller_supplied_clock(tmp_path): incumbent = tmp_path / "incumbent" _tree(incumbent, "10") From e3b4631f3ea118368b2772479606f06b8f1b4f24 Mon Sep 17 00:00:00 2001 From: NumberWan Date: Tue, 17 Nov 2026 11:07:38 +0800 Subject: [PATCH 20/23] feat(harness): joint single-rerun close for parallel rounds Add close that overlays PASS-lane outboxes in lane_id order, measures the joint candidate once with run_experiment, and writes successor/ only on PASS. This is an intentional MVP, not #249 stepwise composition. Also stop host git on candidates after a writable build (_tree_identity) and map ConfinementUnavailable during validation to unsettled instead of measured FAIL, matching the review blockers on 026f686d. Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_round.py | 259 ++++++++++++++++++++++++- tests/test_parallel_round.py | 155 +++++++++++++++ tools/parallel_round.py | 21 +- 3 files changed, 431 insertions(+), 4 deletions(-) diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py index 5d7383a89..3bcd7fc79 100644 --- a/src/inferrsi/harness/parallel_round.py +++ b/src/inferrsi/harness/parallel_round.py @@ -1,4 +1,4 @@ -"""Prepare, submit and measure one offline parallel round. +"""Prepare, submit, measure and jointly close one offline parallel round. Directory diffs stand in for git clones. A synthetic metric value is not a campaign verdict. @@ -25,6 +25,12 @@ If a timed-out command cannot be confirmed dead, its devices are quarantined: ``leases/.quarantined`` is written before the lease is released, and the device is refused until an operator removes that file. + +``close`` is a joint single-rerun MVP (not #249 stepwise composition): after every +outbox ticket has a terminal receipt, PASS lanes are overlaid in ``lane_id`` +order onto one candidate and measured once against the incumbent. A PASS joint +run writes ``successor/``; otherwise the incumbent is retained. This is not +Claude/Codex drivers, SwarmFlow, or campaign incumbent advancement. """ import hashlib @@ -64,6 +70,8 @@ from inferrsi.model.protocols.metrics import read_metric from inferrsi.runspec import RunSpec +_TERMINAL = frozenset({"measured", "rejected", "unsettled"}) + PLANES = {"harness", "model", "infra"} DIRECTIONS = {"lower", "higher"} @@ -500,6 +508,38 @@ def _device_view(plan: dict, devices: list[str]) -> dict: } +def _tree_identity(root: Path, exclude: Path | None = None) -> dict: + """Fingerprint a tree by hashing regular files. Never invokes host ``git``. + + Build runs as a writable candidate and may rewrite ``.git``; calling + ``repository_identity`` afterward would execute planted ``diff.external`` / + ``core.fsmonitor`` hooks outside confinement. This digest stays inert. + """ + root = Path(root).resolve() + identity: dict = {"path": str(root), "available": False, "tool": "directory"} + try: + hashes, blocked = _snapshot(root) + if blocked: + raise ValueError(f"unreadable paths: {sorted(blocked)}") + if exclude is not None: + excluded = Path(exclude).resolve() + hashes = { + path: digest_value + for path, digest_value in hashes.items() + if not (root / path).resolve().is_relative_to(excluded) + } + snapshot_hash = digest({"files": hashes}) + identity.update( + available=True, + snapshot_hash=snapshot_hash, + dirty=False, + files=len(hashes), + ) + except (OSError, ValueError) as exc: + identity["error"] = str(exc) + return identity + + def _run_confined( plan: dict, stage: str, @@ -584,7 +624,9 @@ def execute(command, cwd, trial_dir, deadline, metric_key): except subprocess.TimeoutExpired: record.update(status="timeout", error="command timed out") except ConfinementUnavailable as exc: - status = "timeout" if "timed out" in str(exc) else "failed" + # Infrastructure refusal is not a quality-gate failure: evaluate maps + # status "failed" to FAIL, while "error" stays incomplete / INCONCLUSIVE. + status = "timeout" if "timed out" in str(exc) else "error" record.update(status=status, error=str(exc)) if record["status"] == "completed" and metric_key is not None: try: @@ -814,6 +856,7 @@ def _measure( current=current, failures=failures, ), + snapshot=_tree_identity, ) except UnreapedCommand as exc: leases = DeviceLease(round_dir / "leases") @@ -823,7 +866,9 @@ def _measure( except (ConfinementUnavailable, BudgetError, ValueError) as exc: return _record(round_dir, ticket, status="unsettled", reason=str(exc), devices=devices) evaluation = json.loads((run_dir / "evaluation.json").read_text(encoding="utf-8")) - if evaluation["improvement_pct"] is None: + # failures include ConfinementUnavailable during validation: that must not + # settle as measured FAIL after metrics already produced an improvement_pct. + if evaluation["improvement_pct"] is None or failures: reason = failures[-1] if failures else "; ".join(evaluation["reasons"]) return _record( round_dir, @@ -860,3 +905,211 @@ def ticket_status(round_dir: Path, ticket: str) -> dict: if path.exists(): return json.loads(path.read_text(encoding="utf-8")) return {"ticket": ticket, "status": "queued"} + + +def _receipts(round_dir: Path) -> dict[str, dict]: + root = round_dir / "results" + if not root.exists(): + return {} + return {path.stem: json.loads(path.read_text(encoding="utf-8")) for path in root.glob("*.json")} + + +def close_round( + round_dir: Path, + incumbent: Path, + *, + backend, + now: datetime | None = None, + clock=None, +) -> dict: + """Joint single-rerun close: compose PASS overlays once, measure once, keep or retain. + + Refuses while any outbox ticket lacks a terminal receipt, or any receipt is + ``unsettled``. Device shortage returns ``queued`` without writing ``close.json``. + """ + del now + + def current() -> datetime: + if clock is not None: + return clock() + return datetime.now(timezone.utc) + + close_path = round_dir / "close.json" + if close_path.exists(): + return json.loads(close_path.read_text(encoding="utf-8")) + + plan = json.loads((round_dir / "round-plan.json").read_text(encoding="utf-8")) + outbox = round_dir / "outbox" + tickets = ( + sorted(path.name for path in outbox.iterdir() if path.is_dir()) if outbox.exists() else [] + ) + receipts = _receipts(round_dir) + missing = [ticket for ticket in tickets if ticket not in receipts] + if missing: + return { + "status": "refused", + "composition": "joint-single-rerun", + "reason": f"tickets without receipts: {', '.join(missing)}", + } + unsettled = [ticket for ticket in tickets if receipts[ticket].get("status") == "unsettled"] + if unsettled: + return { + "status": "refused", + "composition": "joint-single-rerun", + "reason": f"unsettled tickets: {', '.join(unsettled)}", + } + bad = [ticket for ticket in tickets if receipts[ticket].get("status") not in _TERMINAL] + if bad: + return { + "status": "refused", + "composition": "joint-single-rerun", + "reason": f"non-terminal receipts: {', '.join(bad)}", + } + + selected = sorted( + ( + receipts[ticket] + for ticket in tickets + if receipts[ticket].get("status") == "measured" + and receipts[ticket].get("verdict") == "PASS" + ), + key=lambda body: body["lane_id"], + ) + + def finish(**fields) -> dict: + body = { + "status": "closed", + "composition": "joint-single-rerun", + "accepted": False, + "retained": True, + "composed": [], + "successor": None, + **fields, + } + write_json(close_path, body) + return body + + if not selected: + return finish(reason="no PASS lanes") + + lanes = [_lane(plan, body["lane_id"]) for body in selected] + deadline_utc = min(lane["deadline_utc"] for lane in lanes) + try: + ensure_open(deadline_utc, current()) + except BudgetError as exc: + return finish(reason=str(exc), composed=[body["ticket"] for body in selected]) + + count = max(lane["device_count"] for lane in lanes) + held = hold_devices(round_dir, plan, count) + if held is None: + return { + "status": "queued", + "composition": "joint-single-rerun", + "reason": "devices unavailable", + "composed": [body["ticket"] for body in selected], + } + devices = [device for device, _lease in held] + composed = [{"ticket": body["ticket"], "lane_id": body["lane_id"]} for body in selected] + try: + if plan.get("device_model"): + try: + _qualify(plan["device_model"], devices) + except (ValueError, OSError, subprocess.SubprocessError) as exc: + return { + "status": "queued", + "composition": "joint-single-rerun", + "reason": str(exc), + "composed": [item["ticket"] for item in composed], + } + overlays = [outbox / body["ticket"] for body in selected] + joint = round_dir / "candidates" / "joint-close" + materialize(incumbent, overlays, joint) + failures: list[str] = [] + try: + if plan["build"]: + build_dir = joint / "build-output" + build_dir.mkdir(parents=True, exist_ok=True) + _run_confined( + plan, + "build", + _command(plan["build"], None, ",".join(devices)), + joint, + (joint, build_dir), + (), + devices, + deadline_utc, + backend=backend, + current=current, + ) + spec = _run_spec( + plan, + joint, + incumbent, + devices, + round_dir / "runs" / "joint-close", + seconds_remaining(deadline_utc, current()), + ) + run_dir = run_experiment( + spec, + {"round_id": plan["round_id"], "ticket": "joint-close", "lane_id": "close"}, + execute=_confined_trial( + plan, + devices, + deadline_utc, + backend=backend, + current=current, + failures=failures, + ), + snapshot=_tree_identity, + ) + except UnreapedCommand as exc: + leases = DeviceLease(round_dir / "leases") + for device in devices: + leases.quarantine(device, f"joint-close: {exc}") + return finish( + reason=str(exc), + composed=composed, + devices=devices, + ) + except (ConfinementUnavailable, BudgetError, ValueError) as exc: + return finish(reason=str(exc), composed=composed, devices=devices) + evaluation = json.loads((run_dir / "evaluation.json").read_text(encoding="utf-8")) + if evaluation["improvement_pct"] is None or failures: + return finish( + reason=failures[-1] if failures else "; ".join(evaluation["reasons"]), + composed=composed, + devices=devices, + run_dir=str(run_dir), + verdict=evaluation.get("verdict"), + improvement_pct=evaluation.get("improvement_pct"), + reasons=evaluation.get("reasons"), + ) + if evaluation["verdict"] != "PASS": + return finish( + reason="joint measurement did not PASS", + composed=composed, + devices=devices, + run_dir=str(run_dir), + verdict=evaluation["verdict"], + improvement_pct=evaluation["improvement_pct"], + reasons=evaluation["reasons"], + ) + successor = round_dir / "successor" + materialize(incumbent, overlays, successor) + return finish( + accepted=True, + retained=False, + reason="joint measurement PASS", + composed=composed, + successor=str(successor), + devices=devices, + run_dir=str(run_dir), + verdict=evaluation["verdict"], + improvement_pct=evaluation["improvement_pct"], + reasons=evaluation["reasons"], + candidate=evaluation["candidate"]["median"], + incumbent=evaluation["baseline"]["median"], + ) + finally: + for _device, lease in held: + lease.release() diff --git a/tests/test_parallel_round.py b/tests/test_parallel_round.py index ff96b7b54..a82e8ac09 100644 --- a/tests/test_parallel_round.py +++ b/tests/test_parallel_round.py @@ -14,6 +14,7 @@ from inferrsi.harness.parallel_round import ( RoundPlanError, _read_nofollow, + close_round, hold_devices, prepare, serve_ticket, @@ -911,3 +912,157 @@ def test_serve_cli_runs_without_a_caller_supplied_clock(tmp_path): else: assert body["status"] == "unsettled" assert body["reason"] == "confinement backend unavailable" + + +SUM_BENCH = ( + "import json, sys\n" + "from pathlib import Path\n" + "a = float(Path('knobs/a.txt').read_text())\n" + "b = float(Path('knobs/b.txt').read_text())\n" + "body = {'metrics': {'latency_ms': a + b}, 'synthetic': True}\n" + "Path(sys.argv[1]).write_text(json.dumps(body))\n" +) + + +def _sum_tree(root: Path, a: str, b: str) -> None: + (root / "knobs").mkdir(parents=True) + (root / "knobs" / "a.txt").write_text(a, encoding="utf-8") + (root / "knobs" / "b.txt").write_text(b, encoding="utf-8") + (root / "bench.py").write_text(SUM_BENCH, encoding="utf-8") + + +def _sum_spec(**overrides): + return _spec( + build=[], + trial=[sys.executable, "bench.py", "{result}"], + validation=[sys.executable, "-c", "pass"], + **overrides, + ) + + +def test_close_composes_pass_lanes_once_and_writes_a_successor(tmp_path): + incumbent = tmp_path / "incumbent" + _sum_tree(incumbent, "10", "10") + round_dir = tmp_path / "round" + prepare(_sum_spec(), incumbent, round_dir) + for lane_id, knob, value in (("L1", "a.txt", "8"), ("L2", "b.txt", "8")): + work = tmp_path / f"work-{lane_id}" + _sum_tree(work, "10", "10") + (work / "knobs" / knob).write_text(value, encoding="utf-8") + ticket = submit(round_dir, lane_id, incumbent, work, now=WHEN)["ticket"] + measured = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + assert measured["status"] == "measured" + assert measured["verdict"] == "PASS" + closed = close_round(round_dir, incumbent, backend=FixtureBackend(), now=WHEN) + assert closed["status"] == "closed" + assert closed["composition"] == "joint-single-rerun" + assert closed["accepted"] is True + assert closed["retained"] is False + assert [item["lane_id"] for item in closed["composed"]] == ["L1", "L2"] + successor = Path(closed["successor"]) + assert (successor / "knobs" / "a.txt").read_text(encoding="utf-8") == "8" + assert (successor / "knobs" / "b.txt").read_text(encoding="utf-8") == "8" + assert closed["improvement_pct"] == pytest.approx(20) + again = close_round(round_dir, incumbent, backend=FixtureBackend(), now=WHEN) + assert again == closed + + +def test_close_with_no_pass_lanes_retains_the_incumbent(tmp_path): + incumbent = tmp_path / "incumbent" + _sum_tree(incumbent, "10", "10") + round_dir = tmp_path / "round" + prepare(_sum_spec(), incumbent, round_dir) + work = tmp_path / "work" + _sum_tree(work, "12", "10") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + measured = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + assert measured["verdict"] == "FAIL" + closed = close_round(round_dir, incumbent, backend=FixtureBackend(), now=WHEN) + assert closed["accepted"] is False + assert closed["retained"] is True + assert closed["composed"] == [] + assert closed["successor"] is None + assert not (round_dir / "successor").exists() + + +def test_close_refuses_while_a_ticket_is_unsettled(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare(_spec(build=[]), incumbent, round_dir) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + + class Empty(FixtureBackend): + def run(self, *args, **kwargs): + del args, kwargs + return {"status": "completed", "stdout": "", "stderr": "", "returncode": 0} + + unsettled = serve_ticket(round_dir, ticket, incumbent, backend=Empty(), now=WHEN) + assert unsettled["status"] == "unsettled" + refused = close_round(round_dir, incumbent, backend=FixtureBackend(), now=WHEN) + assert refused["status"] == "refused" + assert "unsettled" in refused["reason"] + assert not (round_dir / "close.json").exists() + + +def test_build_planted_git_hooks_do_not_run_on_the_host(tmp_path): + canary = tmp_path / "hook-ran" + hook = tmp_path / "evil-hook.py" + hook.write_text( + f"from pathlib import Path\nPath({str(canary)!r}).write_text('ran')\n", + encoding="utf-8", + ) + incumbent = tmp_path / "incumbent" + _git_tree(incumbent, "10") + plant = ( + "from pathlib import Path\n" + "root = Path('.git')\n" + "root.mkdir(exist_ok=True)\n" + f"(root / 'config').write_text('[diff]\\n\\texternal = {sys.executable} {hook}\\n')\n" + ) + round_dir = tmp_path / "round" + prepare( + _spec( + build=[sys.executable, "-c", plant], + validation=[sys.executable, "-c", "pass"], + ), + incumbent, + round_dir, + ) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + measured = serve_ticket(round_dir, ticket, incumbent, backend=FixtureBackend(), now=WHEN) + assert measured["status"] == "measured", measured + assert not canary.exists() + assert "diff" in (round_dir / "candidates" / ticket / ".git" / "config").read_text( + encoding="utf-8" + ) + + +def test_validation_confinement_failure_stays_unsettled(tmp_path): + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare( + _spec(build=[], validation=[sys.executable, "-c", "pass"]), + incumbent, + round_dir, + ) + work = tmp_path / "work" + _tree(work, "8") + ticket = submit(round_dir, "L1", incumbent, work, now=WHEN)["ticket"] + + class BoomValidation(FixtureBackend): + def run(self, argv, cwd, writable, readable, deadline, **kwargs): + if writable and Path(writable[0]).name == "validation": + raise ConfinementUnavailable("device node missing") + return super().run(argv, cwd, writable, readable, deadline, **kwargs) + + result = serve_ticket(round_dir, ticket, incumbent, backend=BoomValidation(), now=WHEN) + assert result["status"] == "unsettled" + assert "device node missing" in result["reason"] + ledger = json.loads((round_dir / "ledgers" / "L1.json").read_text(encoding="utf-8")) + assert [entry["phase"] for entry in ledger["entries"]] == ["claimed"] diff --git a/tools/parallel_round.py b/tools/parallel_round.py index 40ddaff59..0d4927627 100644 --- a/tools/parallel_round.py +++ b/tools/parallel_round.py @@ -6,7 +6,13 @@ from datetime import datetime, timezone from pathlib import Path -from inferrsi.harness.parallel_round import prepare, serve_ticket, submit, ticket_status +from inferrsi.harness.parallel_round import ( + close_round, + prepare, + serve_ticket, + submit, + ticket_status, +) from inferrsi.infra.confinement import select_backend @@ -38,6 +44,13 @@ def main(argv: list[str] | None = None) -> int: status_command.add_argument("--round-dir", required=True, type=Path) status_command.add_argument("--ticket", required=True) + close_command = commands.add_parser( + "close", + help="Joint single-rerun: compose PASS lanes once, measure once, write successor or retain", + ) + close_command.add_argument("--round-dir", required=True, type=Path) + close_command.add_argument("--incumbent", required=True, type=Path) + args = parser.parse_args(argv) if args.command == "prepare": result = prepare(_load(args.spec), args.incumbent, args.round_dir) @@ -56,6 +69,12 @@ def main(argv: list[str] | None = None) -> int: args.incumbent, backend=select_backend(), ) + elif args.command == "close": + result = close_round( + args.round_dir, + args.incumbent, + backend=select_backend(), + ) else: result = ticket_status(args.round_dir, args.ticket) json.dump(result, sys.stdout, indent=2) From be9af6f579ec58c989d20d2a3d8ee08a85d98e85 Mon Sep 17 00:00:00 2001 From: NumberWan Date: Tue, 17 Nov 2026 14:07:22 +0800 Subject: [PATCH 21/23] fix(harness): adapt parallel_round to contracts and runner ports After rebasing onto main, RunSpec lives in inferrsi.contracts and run_experiment requires assess/report ports. Wire the v0 gates from benchmark.legacy_run_v0 and keep report as a no-op; measurement still reads evaluation.json. Signed-off-by: NumberWan --- src/inferrsi/harness/parallel_round.py | 41 +++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py index 3bcd7fc79..5a1309456 100644 --- a/src/inferrsi/harness/parallel_round.py +++ b/src/inferrsi/harness/parallel_round.py @@ -45,6 +45,9 @@ from fnmatch import fnmatch from pathlib import Path +from inferrsi.benchmark.legacy_run_v0 import RunRules +from inferrsi.benchmark.legacy_run_v0 import evaluate as assess_run +from inferrsi.contracts import RunSpec from inferrsi.harness.round_budget import ( Binding, BudgetError, @@ -58,7 +61,7 @@ ensure_open, seconds_remaining, ) -from inferrsi.harness.runner import run_experiment +from inferrsi.harness.runner import run_experiment as run_experiment_loop from inferrsi.infra.artifacts import digest, write_json from inferrsi.infra.confinement import ( ConfinementUnavailable, @@ -68,7 +71,6 @@ from inferrsi.infra.device_lease import DeviceLease, LeaseError from inferrsi.infra.platforms.cuda import qualify_devices from inferrsi.model.protocols.metrics import read_metric -from inferrsi.runspec import RunSpec _TERMINAL = frozenset({"measured", "rejected", "unsettled"}) @@ -670,6 +672,37 @@ def _run_spec( ) +def _run_experiment(spec: RunSpec, environment: dict, *, execute, snapshot) -> Path: + """Compose the harness loop with the v0 gates; report rendering is optional here.""" + + def assess(baseline, candidate, validation, integrity, error): + return assess_run( + RunRules( + repetitions=spec.repetitions, + required_trial_checks=spec.required_trial_checks, + metric_key=spec.metric_key, + metric_unit=spec.metric_unit, + comparison_scope=spec.comparison_scope, + direction=spec.direction, + min_improvement_pct=spec.min_improvement_pct, + ), + baseline, + candidate, + validation, + integrity, + run_error=error, + ) + + return run_experiment_loop( + spec, + environment, + execute=execute, + snapshot=snapshot, + assess=assess, + report=lambda _run_dir: None, + ) + + def _qualify(device_model: str, devices: list[str]) -> None: """``qualify_devices`` reads the caller's reservation from ``CUDA_VISIBLE_DEVICES``.""" previous = os.environ.get("CUDA_VISIBLE_DEVICES") @@ -845,7 +878,7 @@ def _measure( round_dir / "runs" / ticket, seconds_remaining(lane["deadline_utc"], current()), ) - run_dir = run_experiment( + run_dir = _run_experiment( spec, {"round_id": plan["round_id"], "ticket": ticket, "lane_id": lane["id"]}, execute=_confined_trial( @@ -1049,7 +1082,7 @@ def finish(**fields) -> dict: round_dir / "runs" / "joint-close", seconds_remaining(deadline_utc, current()), ) - run_dir = run_experiment( + run_dir = _run_experiment( spec, {"round_id": plan["round_id"], "ticket": "joint-close", "lane_id": "close"}, execute=_confined_trial( From ceb0f1f1680176bda7dd783778292da432442fbd Mon Sep 17 00:00:00 2001 From: NumberWan Date: Tue, 17 Nov 2026 15:45:41 +0800 Subject: [PATCH 22/23] fix(harness): enforce submit boundaries and venv namespace stdlib Move parallel-round assess composition into apps so harness stays clear of benchmark imports. Reject protected and cross-lane paths on concrete submit, and mount the versioned stdlib for UnshareBackend under a venv. Signed-off-by: NumberWan --- src/inferrsi/apps/parallel_round.py | 66 ++++++++++++++++++++ src/inferrsi/harness/parallel_round.py | 65 ++++++++++--------- src/inferrsi/infra/confinement.py | 22 ++++++- tests/test_confinement.py | 28 +++++++++ tests/test_knowledge_compatibility_shells.py | 22 ++++--- tests/test_parallel_round.py | 55 +++++++++++++++- tests/test_target_module_import_gate.py | 9 +-- tools/parallel_round.py | 9 +-- 8 files changed, 226 insertions(+), 50 deletions(-) create mode 100644 src/inferrsi/apps/parallel_round.py diff --git a/src/inferrsi/apps/parallel_round.py b/src/inferrsi/apps/parallel_round.py new file mode 100644 index 000000000..fcc0b7a64 --- /dev/null +++ b/src/inferrsi/apps/parallel_round.py @@ -0,0 +1,66 @@ +"""Compose Harness parallel-round orchestration with Benchmark v0 gates. + +Harness owns prepare / submit / serve / close scheduling. This package binds the +legacy assess port so ``inferrsi.harness.parallel_round`` never imports Benchmark. +""" + +from collections.abc import Callable +from datetime import datetime +from pathlib import Path + +from inferrsi.apps.legacy.evaluation import run_rules +from inferrsi.benchmark.legacy_run_v0 import evaluate as _evaluate +from inferrsi.contracts import RunSpec +from inferrsi.harness import parallel_round as harness + + +def make_assess(spec: RunSpec) -> Callable: + """Bind v0 gates for one ``RunSpec``; the returned port matches ``runner.run_experiment``.""" + rules = run_rules(spec) + + def assess(baseline, candidate, validation, integrity, error): + return _evaluate( + rules, baseline, candidate, validation, integrity, run_error=error + ) + + return assess + + +def serve_ticket( + round_dir: Path, + ticket: str, + incumbent: Path, + *, + backend, + now: datetime | None = None, + clock=None, +) -> dict: + """Serve one ticket with the v0 assess composition.""" + return harness.serve_ticket( + round_dir, + ticket, + incumbent, + backend=backend, + make_assess=make_assess, + now=now, + clock=clock, + ) + + +def close_round( + round_dir: Path, + incumbent: Path, + *, + backend, + now: datetime | None = None, + clock=None, +) -> dict: + """Close one round with the v0 assess composition.""" + return harness.close_round( + round_dir, + incumbent, + backend=backend, + make_assess=make_assess, + now=now, + clock=clock, + ) diff --git a/src/inferrsi/harness/parallel_round.py b/src/inferrsi/harness/parallel_round.py index 5a1309456..6745e96ad 100644 --- a/src/inferrsi/harness/parallel_round.py +++ b/src/inferrsi/harness/parallel_round.py @@ -11,8 +11,9 @@ ``{"metrics": {: }}`` to ``{result}``. The candidate is built once in confinement; measurement is ``run_experiment`` with a confined ``execute``, so incumbent and candidate trials alternate AB/BA -``metric.repeats`` times each, the optional ``validation`` runs once on the -candidate, and ``evaluate`` gives the verdict against ``metric.min_improvement_pct``. +``metric.repeats`` times each, and the optional ``validation`` runs once on the +candidate. The composing application supplies ``make_assess`` so Harness never +imports Benchmark; that assess gives the verdict against ``metric.min_improvement_pct``. ``{device}`` is the held device identities joined with commas. Other text, including ``{result}`` inside ``build``, is passed through unchanged. @@ -41,12 +42,11 @@ import stat import subprocess import time +from collections.abc import Callable from datetime import datetime, timezone from fnmatch import fnmatch from pathlib import Path -from inferrsi.benchmark.legacy_run_v0 import RunRules -from inferrsi.benchmark.legacy_run_v0 import evaluate as assess_run from inferrsi.contracts import RunSpec from inferrsi.harness.round_budget import ( Binding, @@ -388,6 +388,18 @@ def submit( outside = [path for path in changed if not _match(path, lane["globs"])] if outside or not changed: return _refusal("unchanged" if not changed else "outside", outside=outside) + # Concrete paths, including newly created files: prepare only saw the incumbent. + protected_hits = [path for path in changed if _match(path, plan.get("protected") or [])] + if protected_hits: + return _refusal("protected", rejected=protected_hits) + others = [other for other in plan["lanes"] if other["id"] != lane_id] + shared = [ + path + for path in changed + if any(_match(path, other["globs"]) for other in others) + ] + if shared: + return _refusal("overlap", rejected=shared) manifest = {path: after[path] for path in changed} ticket = digest( {"round_digest": plan["round_digest"], "lane_id": lane_id, "manifest": manifest} @@ -672,33 +684,21 @@ def _run_spec( ) -def _run_experiment(spec: RunSpec, environment: dict, *, execute, snapshot) -> Path: - """Compose the harness loop with the v0 gates; report rendering is optional here.""" - - def assess(baseline, candidate, validation, integrity, error): - return assess_run( - RunRules( - repetitions=spec.repetitions, - required_trial_checks=spec.required_trial_checks, - metric_key=spec.metric_key, - metric_unit=spec.metric_unit, - comparison_scope=spec.comparison_scope, - direction=spec.direction, - min_improvement_pct=spec.min_improvement_pct, - ), - baseline, - candidate, - validation, - integrity, - run_error=error, - ) - +def _run_experiment( + spec: RunSpec, + environment: dict, + *, + execute, + snapshot, + make_assess: Callable[[RunSpec], Callable], +) -> Path: + """Run the harness loop with an application-supplied assess port; report is a no-op.""" return run_experiment_loop( spec, environment, execute=execute, snapshot=snapshot, - assess=assess, + assess=make_assess(spec), report=lambda _run_dir: None, ) @@ -740,14 +740,17 @@ def serve_ticket( incumbent: Path, *, backend, + make_assess: Callable[[RunSpec], Callable], now: datetime | None = None, clock=None, ) -> dict: """Measure one ticket on the lane's ``device_count`` free devices. A busy lane or too few free devices leaves it queued and does not reserve an attempt. - ``now`` is accepted for older callers and is not a frozen deadline. Every - admission and command reads ``clock`` or the current time again. + ``make_assess`` binds Benchmark (or other) gates for one ``RunSpec``; Harness + only calls the returned assess port. ``now`` is accepted for older callers and + is not a frozen deadline. Every admission and command reads ``clock`` or the + current time again. """ del now @@ -811,6 +814,7 @@ def current() -> datetime: incumbent, [device for device, _lease in held], backend=backend, + make_assess=make_assess, current=current, reserved=entry, ) @@ -831,6 +835,7 @@ def _measure( devices: list[str], *, backend, + make_assess: Callable[[RunSpec], Callable], current, reserved=None, ) -> dict: @@ -890,6 +895,7 @@ def _measure( failures=failures, ), snapshot=_tree_identity, + make_assess=make_assess, ) except UnreapedCommand as exc: leases = DeviceLease(round_dir / "leases") @@ -952,11 +958,13 @@ def close_round( incumbent: Path, *, backend, + make_assess: Callable[[RunSpec], Callable], now: datetime | None = None, clock=None, ) -> dict: """Joint single-rerun close: compose PASS overlays once, measure once, keep or retain. + ``make_assess`` is the same application-supplied port as ``serve_ticket``. Refuses while any outbox ticket lacks a terminal receipt, or any receipt is ``unsettled``. Device shortage returns ``queued`` without writing ``close.json``. """ @@ -1094,6 +1102,7 @@ def finish(**fields) -> dict: failures=failures, ), snapshot=_tree_identity, + make_assess=make_assess, ) except UnreapedCommand as exc: leases = DeviceLease(round_dir / "leases") diff --git a/src/inferrsi/infra/confinement.py b/src/inferrsi/infra/confinement.py index c64a605e1..40c0b2355 100644 --- a/src/inferrsi/infra/confinement.py +++ b/src/inferrsi/infra/confinement.py @@ -113,11 +113,29 @@ def minimal_environment(extra: dict | None = None) -> dict: def runtime_roots() -> tuple[Path, ...]: - """Python prefix needed to start the interpreter. Never the host root.""" + """Python prefixes and stdlib needed to start the interpreter. Never the host root. + + A virtualenv's ``sys.prefix`` does not contain ``encodings``; the versioned + stdlib tree under the base install must be mounted too. Prefer that tree + over mounting a broad ``sys.base_prefix`` such as ``/usr``. + """ + import sysconfig + candidates = [Path(sys.prefix).resolve(), Path(sys.executable).resolve().parent] + stdlib = Path(sysconfig.get_path("stdlib")).resolve() + if stdlib.is_dir() and stdlib != Path("/"): + candidates.append(stdlib) + base = Path(sys.base_prefix).resolve() + prefix = Path(sys.prefix).resolve() + # Contained base installs (pyenv, standalone builds) may keep more than + # stdlib under base_prefix; skip over-broad roots like / or /usr. + if base not in (prefix, Path("/"), Path("/usr")) and base.is_dir(): + candidates.append(base) roots: list[Path] = [] for candidate in candidates: - if any(candidate == earlier or earlier in candidate.parents for earlier in roots): + if candidate == Path("/") or any( + candidate == earlier or earlier in candidate.parents for earlier in roots + ): continue roots.append(candidate) return tuple(roots) diff --git a/tests/test_confinement.py b/tests/test_confinement.py index 28d1ad6ee..648f3690b 100644 --- a/tests/test_confinement.py +++ b/tests/test_confinement.py @@ -23,6 +23,7 @@ minimal_environment, namespace_plan, namespaces_available, + runtime_roots, select_backend, supervisor_command, ) @@ -235,6 +236,33 @@ def test_namespace_plan_includes_the_loader_but_not_a_private_file(tmp_path, mon assert "SECRET_CANARY" not in plan["env"] +def test_runtime_roots_include_stdlib_without_mounting_usr(): + import sysconfig + + roots = runtime_roots() + assert Path("/") not in roots + assert Path("/usr") not in roots + stdlib = Path(sysconfig.get_path("stdlib")).resolve() + assert any(root == stdlib or stdlib.is_relative_to(root) for root in roots) + if Path(sys.prefix).resolve() != Path(sys.base_prefix).resolve(): + assert Path(sys.prefix).resolve() in roots + + +@pytest.mark.skipif(not namespaces_available(), reason="user namespaces are unavailable") +def test_unshare_from_venv_can_import_encodings(tmp_path): + work = tmp_path / "work" + work.mkdir() + record = UnshareBackend().run( + (sys.executable, "-c", "import encodings; print(encodings.__file__)"), + work, + (work,), + (), + time.monotonic() + 30, + ) + assert record["status"] == "completed" + assert "encodings" in record["stdout"] + + @pytest.mark.skipif(not namespaces_available(), reason="user namespaces are unavailable") def test_unshare_starts_python_and_hides_a_private_canary(tmp_path, monkeypatch): monkeypatch.setenv("SECRET_CANARY", "CANARY-ENV") diff --git a/tests/test_knowledge_compatibility_shells.py b/tests/test_knowledge_compatibility_shells.py index dbd6c4825..e43b50f80 100644 --- a/tests/test_knowledge_compatibility_shells.py +++ b/tests/test_knowledge_compatibility_shells.py @@ -282,20 +282,28 @@ def test_package_init_and_infra_inspection_load_no_knowledge_body(): def test_knowledge_closure_is_clean_while_overall_stays_false(): # K01-5: the Knowledge package exists and its closure is clean. R1 cleared the - # allowlist and the old-run forwarders. The parser entry due in A1, the twelve - # harness -> infra hits (R2) and the five runtime resource loaders (R3) remain, - # and the loaders keep new_core_closure_isolated false. + # allowlist and the old-run forwarders. Remaining harness -> model/infra hits + # (R2, including parallel_round) and the five runtime resource loaders (R3) + # keep overall isolation incomplete; the loaders keep new_core_closure_isolated + # false. report = first_wave_report() assert report.violations == () assert report.blockers == ( - "harness reaches model/infra: inferrsi.harness.agent_sessions -> inferrsi.infra (12 hits)", + "harness reaches model/infra: inferrsi.harness.agent_sessions -> inferrsi.infra (21 hits)", "runtime resource loaders: 5", ) - assert len(report.harness_model_infra) == 12 + assert len(report.harness_model_infra) == 21 assert {path.split(" -> ")[0] for path in report.harness_model_infra} == { f"inferrsi.harness.{name}" - for name in ("agent_sessions", "candidates", "optimization", "optimization_session", - "project", "task_protocol") + for name in ( + "agent_sessions", + "candidates", + "optimization", + "optimization_session", + "parallel_round", + "project", + "task_protocol", + ) } assert not report.new_core_closure_isolated assert not report.isolation_complete diff --git a/tests/test_parallel_round.py b/tests/test_parallel_round.py index a82e8ac09..92931c95b 100644 --- a/tests/test_parallel_round.py +++ b/tests/test_parallel_round.py @@ -2,6 +2,7 @@ import json import os +import shutil import subprocess import sys import threading @@ -10,14 +11,13 @@ import pytest +from inferrsi.apps.parallel_round import close_round, serve_ticket from inferrsi.harness import parallel_round from inferrsi.harness.parallel_round import ( RoundPlanError, _read_nofollow, - close_round, hold_devices, prepare, - serve_ticket, submit, ticket_status, ) @@ -188,6 +188,57 @@ def test_submit_rejects_paths_outside_the_lane_and_is_idempotent(tmp_path): assert first["ticket"] == second["ticket"] +def test_submit_rejects_new_protected_and_cross_lane_owned_paths(tmp_path): + """prepare only sees the incumbent; submit must police concrete new paths.""" + incumbent = tmp_path / "incumbent" + _tree(incumbent, "10") + round_dir = tmp_path / "round" + prepare( + _spec( + protected=["new/protected.txt"], + lanes=[ + { + "id": "L1", + "entry": "D-01", + "plane": "infra", + "globs": ["new/*"], + "cap": 2, + "per_attempt_worst_case": 1, + "deadline_utc": "2099-01-01T00:00:00+00:00", + }, + { + "id": "L2", + "entry": "D-02", + "plane": "harness", + "globs": ["new/*.txt"], + "cap": 2, + "per_attempt_worst_case": 1, + "deadline_utc": "2099-01-01T00:00:00+00:00", + }, + ], + ), + incumbent, + round_dir, + ) + work = tmp_path / "work" + shutil.copytree(incumbent, work) + (work / "new").mkdir() + (work / "new" / "protected.txt").write_text("x", encoding="utf-8") + for lane_id in ("L1", "L2"): + refused = submit(round_dir, lane_id, incumbent, work) + assert refused["submitted"] is False + assert refused["reason"] == "protected" + assert refused["rejected"] == ["new/protected.txt"] + (work / "new" / "protected.txt").unlink() + (work / "new" / "shared.txt").write_text("y", encoding="utf-8") + for lane_id in ("L1", "L2"): + refused = submit(round_dir, lane_id, incumbent, work) + assert refused["submitted"] is False + assert refused["reason"] == "overlap" + assert refused["rejected"] == ["new/shared.txt"] + assert not (round_dir / "outbox").exists() + + def test_serve_records_a_synthetic_gain_and_a_held_device_stays_queued(tmp_path): incumbent = tmp_path / "incumbent" _tree(incumbent, "10") diff --git a/tests/test_target_module_import_gate.py b/tests/test_target_module_import_gate.py index bafcdff94..84e11bd7f 100644 --- a/tests/test_target_module_import_gate.py +++ b/tests/test_target_module_import_gate.py @@ -148,17 +148,18 @@ def test_clean_scan_is_not_isolation(self) -> None: self.assertFalse(hasattr(report, "residuals")) self.assertNotIn("residual", kinds) self.assertEqual(report.violations, ()) - # What keeps isolation false after R1: the parser entry due in A1, the six - # Harness modules that R2 closes and the five loaders that R3 closes. + # What keeps isolation false after R1: remaining harness→model/infra + # reaches (including parallel_round confinement/lease/cuda/metric) and + # the five loaders that R3 closes. self.assertEqual( report.blockers, ( "harness reaches model/infra: inferrsi.harness.agent_sessions -> " - "inferrsi.infra (12 hits)", + "inferrsi.infra (21 hits)", "runtime resource loaders: 5", ), ) - self.assertEqual(len(report.harness_model_infra), 12) + self.assertEqual(len(report.harness_model_infra), 21) self.assertFalse(report.new_core_closure_isolated) def test_split_skill_eval_modules_reach_no_model_or_infra(self) -> None: diff --git a/tools/parallel_round.py b/tools/parallel_round.py index 0d4927627..79629515e 100644 --- a/tools/parallel_round.py +++ b/tools/parallel_round.py @@ -6,13 +6,8 @@ from datetime import datetime, timezone from pathlib import Path -from inferrsi.harness.parallel_round import ( - close_round, - prepare, - serve_ticket, - submit, - ticket_status, -) +from inferrsi.apps.parallel_round import close_round, serve_ticket +from inferrsi.harness.parallel_round import prepare, submit, ticket_status from inferrsi.infra.confinement import select_backend From 07817ac6f2c424687021f676dd3a18bbec425929 Mon Sep 17 00:00:00 2001 From: NumberWan Date: Tue, 17 Nov 2026 16:20:40 +0800 Subject: [PATCH 23/23] fix(confinement): exclude /usr from every runtime_roots candidate System interpreters use prefix=/usr; filter that broad root for all candidates and keep versioned stdlib plus the executable directory. Cover both the active interpreter and a /usr/bin/python3 subprocess. Signed-off-by: NumberWan --- src/inferrsi/infra/confinement.py | 15 ++++--- tests/test_confinement.py | 71 ++++++++++++++++++++++++++++--- 2 files changed, 74 insertions(+), 12 deletions(-) diff --git a/src/inferrsi/infra/confinement.py b/src/inferrsi/infra/confinement.py index 40c0b2355..854f1ffc2 100644 --- a/src/inferrsi/infra/confinement.py +++ b/src/inferrsi/infra/confinement.py @@ -112,28 +112,33 @@ def minimal_environment(extra: dict | None = None) -> dict: return environment +# Broad host trees must never become jail roots; use versioned stdlib instead. +_BROAD_RUNTIME_ROOTS = frozenset({Path("/"), Path("/usr")}) + + def runtime_roots() -> tuple[Path, ...]: """Python prefixes and stdlib needed to start the interpreter. Never the host root. A virtualenv's ``sys.prefix`` does not contain ``encodings``; the versioned stdlib tree under the base install must be mounted too. Prefer that tree - over mounting a broad ``sys.base_prefix`` such as ``/usr``. + over mounting a broad ``sys.prefix`` / ``sys.base_prefix`` such as ``/usr``. + Every candidate is filtered the same way, including system interpreters. """ import sysconfig candidates = [Path(sys.prefix).resolve(), Path(sys.executable).resolve().parent] stdlib = Path(sysconfig.get_path("stdlib")).resolve() - if stdlib.is_dir() and stdlib != Path("/"): + if stdlib.is_dir(): candidates.append(stdlib) base = Path(sys.base_prefix).resolve() prefix = Path(sys.prefix).resolve() # Contained base installs (pyenv, standalone builds) may keep more than - # stdlib under base_prefix; skip over-broad roots like / or /usr. - if base not in (prefix, Path("/"), Path("/usr")) and base.is_dir(): + # stdlib under base_prefix; the loop below still drops / and /usr. + if base != prefix and base.is_dir(): candidates.append(base) roots: list[Path] = [] for candidate in candidates: - if candidate == Path("/") or any( + if candidate in _BROAD_RUNTIME_ROOTS or any( candidate == earlier or earlier in candidate.parents for earlier in roots ): continue diff --git a/tests/test_confinement.py b/tests/test_confinement.py index 648f3690b..d26a84c14 100644 --- a/tests/test_confinement.py +++ b/tests/test_confinement.py @@ -236,16 +236,73 @@ def test_namespace_plan_includes_the_loader_but_not_a_private_file(tmp_path, mon assert "SECRET_CANARY" not in plan["env"] +def _assert_runtime_roots_exclude_broad_hosts(roots, *, prefix, base_prefix, stdlib): + del base_prefix + assert Path("/") not in roots + assert Path("/usr") not in roots + stdlib_path = Path(stdlib).resolve() + assert any(root == stdlib_path or stdlib_path.is_relative_to(root) for root in roots) + prefix_path = Path(prefix).resolve() + # Virtualenv prefix must stay visible; a system prefix of /usr is skipped on purpose. + if prefix_path not in {Path("/"), Path("/usr")}: + assert prefix_path in roots + + def test_runtime_roots_include_stdlib_without_mounting_usr(): import sysconfig - roots = runtime_roots() - assert Path("/") not in roots - assert Path("/usr") not in roots - stdlib = Path(sysconfig.get_path("stdlib")).resolve() - assert any(root == stdlib or stdlib.is_relative_to(root) for root in roots) - if Path(sys.prefix).resolve() != Path(sys.base_prefix).resolve(): - assert Path(sys.prefix).resolve() in roots + _assert_runtime_roots_exclude_broad_hosts( + runtime_roots(), + prefix=sys.prefix, + base_prefix=sys.base_prefix, + stdlib=sysconfig.get_path("stdlib"), + ) + + +def test_runtime_roots_under_system_python_exclude_usr(): + """System interpreters often use prefix=/usr; that tree must not be mounted whole.""" + system = Path("/usr/bin/python3") + if not system.is_file(): + pytest.skip("no /usr/bin/python3") + if Path(sys.executable).resolve() == system.resolve() and Path(sys.prefix).resolve() == Path( + "/usr" + ): + # Already covered by the current-interpreter test above. + import sysconfig + + _assert_runtime_roots_exclude_broad_hosts( + runtime_roots(), + prefix=sys.prefix, + base_prefix=sys.base_prefix, + stdlib=sysconfig.get_path("stdlib"), + ) + return + repo = Path(__file__).resolve().parents[1] + script = ( + "import json, sys, sysconfig\n" + "sys.path[:0] = %r\n" + "from inferrsi.infra.confinement import runtime_roots\n" + "print(json.dumps({\n" + " 'roots': [str(p) for p in runtime_roots()],\n" + " 'prefix': sys.prefix,\n" + " 'base_prefix': sys.base_prefix,\n" + " 'stdlib': sysconfig.get_path('stdlib'),\n" + "}))\n" % [str(repo / "src"), str(repo)] + ) + completed = subprocess.run( + [str(system), "-c", script], + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0, completed.stderr + payload = json.loads(completed.stdout) + _assert_runtime_roots_exclude_broad_hosts( + [Path(item) for item in payload["roots"]], + prefix=payload["prefix"], + base_prefix=payload["base_prefix"], + stdlib=payload["stdlib"], + ) @pytest.mark.skipif(not namespaces_available(), reason="user namespaces are unavailable")