diff --git a/README.md b/README.md index 5ada596..78ca3bc 100644 --- a/README.md +++ b/README.md @@ -119,11 +119,13 @@ connected through SSH local forwarding. env-var rename) are offline by default. `--live` skips with exit 0 when Ollama is down; that skip is not a - quality pass: + quality pass. `--require-live` writes `eval-runs/live-status.json` + `{"skipped": true}` and exits 2 so CI can fail closed: ```bash .venv/bin/python scripts/prove_multifile_refactor.py .venv/bin/python scripts/prove_multifile_refactor.py --live --model fast + .venv/bin/python scripts/prove_multifile_refactor.py --live --require-live --model fast ``` The unit suite proves deterministic client, safety, evaluation-scorer, diff --git a/docs/evaluation-protocol.md b/docs/evaluation-protocol.md index eaeba5d..b9531cb 100644 --- a/docs/evaluation-protocol.md +++ b/docs/evaluation-protocol.md @@ -198,12 +198,15 @@ PYTHONPATH=src .venv/bin/python -m unittest discover -s tests -v GitHub Actions (`.github/workflows/tests.yml`) runs the same no-GPU path on every push. Live protocol (workstation with Ollama). If the server is down, `--live` -prints `SKIP live` and exits 0. That skip is not a model-quality pass: +prints `SKIP live` and exits 0. `--require-live` writes +`eval-runs/live-status.json` `{"skipped": true}` and exits 2. That skip +is not a model-quality pass: ```bash .venv/bin/python scripts/run_eval.py --live --model fast .venv/bin/python scripts/run_eval.py --live --suite harder --model fast .venv/bin/python scripts/prove_multifile_refactor.py --live --model fast +.venv/bin/python scripts/prove_multifile_refactor.py --live --require-live --model fast .venv/bin/python scripts/run_eval.py --live --model strong ``` diff --git a/scripts/prove_multifile_refactor.py b/scripts/prove_multifile_refactor.py index 93157e9..d990f90 100644 --- a/scripts/prove_multifile_refactor.py +++ b/scripts/prove_multifile_refactor.py @@ -3,7 +3,8 @@ Default path is offline fixtures (CI / no GPU). ``--live`` calls real ``local_refactor`` through stdio MCP and skips with exit 0 when Ollama -is down. A skip is not a model-quality pass. +is down. ``--require-live`` is the same path but exits 2 on skip and +writes ``eval-runs/live-status.json``. A skip is not a model-quality pass. """ from __future__ import annotations @@ -19,6 +20,7 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--live", action="store_true") + parser.add_argument("--require-live", action="store_true") parser.add_argument("--model", choices=("fast", "strong"), default="fast") parser.add_argument("--case", dest="case_id", default=None) args = parser.parse_args() @@ -28,8 +30,10 @@ def main() -> None: "--suite", "harder", ] - if args.live: + if args.live or args.require_live: cmd.extend(["--live", "--model", args.model]) + if args.require_live: + cmd.append("--require-live") if args.case_id: cmd.extend(["--case", args.case_id]) raise SystemExit(subprocess.call(cmd, cwd=str(ROOT))) diff --git a/scripts/prove_refactor_acceptance.py b/scripts/prove_refactor_acceptance.py index a73216f..7c56ad1 100644 --- a/scripts/prove_refactor_acceptance.py +++ b/scripts/prove_refactor_acceptance.py @@ -3,7 +3,7 @@ Single-file whitespace extract only. For the harder multi-file corpus, use ``scripts/prove_multifile_refactor.py`` (offline fixtures by default; -``--live`` skips when Ollama is down). +``--live`` skips when Ollama is down; ``--require-live`` exits 2). """ from __future__ import annotations diff --git a/scripts/run_eval.py b/scripts/run_eval.py index 35efa7c..ccd82ec 100644 --- a/scripts/run_eval.py +++ b/scripts/run_eval.py @@ -5,12 +5,14 @@ import argparse import asyncio +import json import os import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[1] SRC = ROOT / "src" +LIVE_STATUS_PATH = ROOT / "eval-runs" / "live-status.json" if str(SRC) not in sys.path: sys.path.insert(0, str(SRC)) @@ -42,6 +44,14 @@ def _print_result(label: str, result: object) -> None: print(f" {layer.status:4} {layer.name}: {layer.message}") +def _write_live_status(*, skipped: bool) -> None: + LIVE_STATUS_PATH.parent.mkdir(parents=True, exist_ok=True) + LIVE_STATUS_PATH.write_text( + json.dumps({"skipped": skipped}) + "\n", + encoding="utf-8", + ) + + def _wanted(case_id: str, suite: str, only: str | None) -> bool: if only and case_id != only: return False @@ -81,18 +91,22 @@ def _run_fixtures(case_id: str | None, suite: str) -> int: return failed -async def _run_live(case_id: str | None, model: str, suite: str) -> int: +async def _run_live( + case_id: str | None, model: str, suite: str, *, require_live: bool +) -> int: from local_coding_slm.ollama_client import OllamaSettings, is_reachable from local_coding_slm.server import _load_dotenv _load_dotenv() settings = OllamaSettings.from_env() if not is_reachable(settings): + _write_live_status(skipped=True) print( f"SKIP live: Ollama unreachable at {settings.host_label()} " "(offline fixtures still pass; this is not a model-quality fail)" ) - return 0 + return 2 if require_live else 0 + _write_live_status(skipped=False) from mcp import ClientSession, StdioServerParameters from mcp.client.stdio import stdio_client @@ -143,6 +157,11 @@ async def _run_live(case_id: str | None, model: str, suite: str) -> int: def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--live", action="store_true", help="Call real Ollama via stdio MCP") + parser.add_argument( + "--require-live", + action="store_true", + help="Same as --live, but exit 2 when Ollama is down instead of skip-0.", + ) parser.add_argument("--model", choices=("fast", "strong"), default="fast") parser.add_argument("--case", dest="case_id", default=None) parser.add_argument( @@ -152,8 +171,17 @@ def main() -> None: help="Fixture / live subset. harder = Phase 3 multi-file behavior cases.", ) args = parser.parse_args() - if args.live: - raise SystemExit(asyncio.run(_run_live(args.case_id, args.model, args.suite))) + if args.live or args.require_live: + raise SystemExit( + asyncio.run( + _run_live( + args.case_id, + args.model, + args.suite, + require_live=args.require_live, + ) + ) + ) failed = _run_fixtures(args.case_id, args.suite) raise SystemExit(1 if failed else 0) diff --git a/spec.md b/spec.md index 64d0e15..0010d84 100644 --- a/spec.md +++ b/spec.md @@ -681,7 +681,7 @@ OLLAMA_BASE_URL │ ├── run_mcp.sh ← project MCP entry (loads .env) │ ├── prove_acceptance.py │ ├── prove_refactor_acceptance.py -│ ├── prove_multifile_refactor.py ← harder multi-file suite; --live skips if down +│ ├── prove_multifile_refactor.py ← harder multi-file suite; --live skip-0, --require-live skip-2 │ ├── run_eval.py ← fixture corpus; --live on the workstation │ ├── run_harness.py ← timed MCP campaign (stub or live) │ └── check_deployment_safety.py ← defensive bind / tag / git checks diff --git a/tests/test_eval_harder.py b/tests/test_eval_harder.py index 3206474..eb90171 100644 --- a/tests/test_eval_harder.py +++ b/tests/test_eval_harder.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os import subprocess import sys @@ -95,6 +96,31 @@ def test_live_flag_skips_when_ollama_is_down(self) -> None: self.assertEqual(proc.returncode, 0, proc.stderr) self.assertIn("SKIP live", proc.stdout) + def test_require_live_exits_2_when_ollama_is_down(self) -> None: + env = os.environ.copy() + env["PYTHONPATH"] = str(ROOT / "src") + os.pathsep + env.get("PYTHONPATH", "") + env["OLLAMA_BASE_URL"] = "http://127.0.0.1:1" + status_path = ROOT / "eval-runs" / "live-status.json" + if status_path.exists(): + status_path.unlink() + proc = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "prove_multifile_refactor.py"), + "--live", + "--require-live", + ], + cwd=str(ROOT), + env=env, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(proc.returncode, 2, proc.stderr) + self.assertIn("SKIP live", proc.stdout) + payload = json.loads(status_path.read_text(encoding="utf-8")) + self.assertEqual(payload, {"skipped": True}) + def test_partial_files_are_format(self) -> None: for name in ( "rename_exception_partial",