feat(code-editing): Step 1 — dependency pipeline (install-RCE control)

apps/api/pytest.ini

@@ -29,5 +29,6 @@ python_files = test_suite_v060.py test_suite_v062.py test_suite_v0610.py test_se
     test_cloud_params_upload.py
     test_params_eval.py
     test_templates_params.py
+    test_agent_deps.py
 markers =
     no_api_key: test does not require WAYFORTH_TEST_API_KEY (e.g. probes unauthenticated paths)

apps/api/scripts/deps_live_proof.py

@@ -0,0 +1,120 @@
+"""scripts/deps_live_proof.py — §6 ship-gate for the dependency pipeline (LIVE).
+
+Runs the real services.agent_deps pipeline against real E2B sandboxes + real pip.
+These are must-pass-BEFORE-user-visible (same standard as the run-token rotation
+proof) — they exercise behavior unit tests can't fake (pip's hash enforcement, a
+booted snapshot, the two egress allowlists).
+
+Gated — does nothing unless DEPS_LIVE_PROOF=1 and E2B_API_KEY is set:
+
+    DEPS_LIVE_PROOF=1 E2B_API_KEY=... [DEPS_MIRROR_URL=...] \
+        /app/.venv/bin/python -m scripts.deps_live_proof
+
+Without a private mirror yet, PyPI stands in as the index (DEPS_MIRROR_URL /
+allowed hosts default to PyPI). In prod these point at the private mirror.
+
+Proves:
+  #1 hash-mismatch → pip rejects the install (a swapped artifact is caught).
+  #3 base deps build → snapshot → boot a run sandbox (gateway-only egress) with
+     httpx + wayforth-sdk importable and PyPI unreachable (unblocks Step 2).
+  §0 two-allowlist separation: build can't reach the gateway; run can't reach the mirror.
+TODO(canary): a wheel that POSTs to an external host → blocked at run egress.
+"""
+from __future__ import annotations
+
+import os
+import sys
+
+from services.agent_deps import (
+    BASE_DEPS, _norm, build_requirements_lock, load_lockfile, pip_install_command,
+)
+
+MIRROR_URL = os.environ.get("DEPS_MIRROR_URL", "https://pypi.org/simple")
+MIRROR_HOSTS = os.environ.get("DEPS_MIRROR_HOSTS", "pypi.org,files.pythonhosted.org").split(",")
+GATEWAY_HOST = os.environ.get("WAYFORTH_GATEWAY_HOST", "gateway.wayforth.io")
+
+
+def _net(allow):
+    from e2b import SandboxNetworkOpts
+    return SandboxNetworkOpts(deny_out=["0.0.0.0/0"], allow_out=list(allow))
+
+
+def _run(sbx, cmd, timeout=150):
+    return sbx.commands.run(cmd + " ; echo EXIT=$?", timeout=timeout)
+
+
+def main() -> int:
+    if os.environ.get("DEPS_LIVE_PROOF") != "1" or not os.environ.get("E2B_API_KEY"):
+        print("deps_live_proof: skipped (set DEPS_LIVE_PROOF=1 and E2B_API_KEY).")
+        return 0
+
+    from e2b import Sandbox
+
+    lock = load_lockfile()
+    base = [(_norm(n), v, lock[_norm(n)][v]) for n, v in BASE_DEPS]
+    good_lock = build_requirements_lock(base)
+    bad_lock = build_requirements_lock([("httpx", "0.28.1", ["sha256:" + "0" * 64])])
+    pip = pip_install_command(MIRROR_URL)
+    failures = []
+
+    # #1 — hash mismatch must be rejected
+    b = Sandbox.create(timeout=180, network=_net(MIRROR_HOSTS))
+    try:
+        # §0: build sandbox cannot reach the gateway
+        g = _run(b, f'curl -sS -o /dev/null -w "%{{http_code}}" --max-time 8 https://{GATEWAY_HOST}/status')
+        if "ec=35" not in g.stdout and "000" not in g.stdout:
+            failures.append(f"#0 build reached gateway: {g.stdout!r}")
+        b.files.write("/home/user/requirements.lock", bad_lock)
+        r = _run(b, pip)
+        if "do not match" not in (r.stdout + r.stderr).lower():
+            failures.append(f"#1 hash mismatch NOT rejected: {(r.stdout + r.stderr)[-300:]!r}")
+        else:
+            print("#1 PASS — pip rejected the tampered hash")
+    finally:
+        b.kill()
+
+    # #3 — base build → snapshot → boot run sandbox, importable, PyPI blocked
+    b2 = Sandbox.create(timeout=300, network=_net(MIRROR_HOSTS))
+    sid = None
+    try:
+        b2.files.write("/home/user/requirements.lock", good_lock)
+        r2 = _run(b2, pip, timeout=240)
+        if "EXIT=0" not in r2.stdout:
+            failures.append(f"#3 base install failed: {(r2.stderr or '')[-300:]!r}")
+        snap = b2.create_snapshot(name="wf-deps-shipgate")
+        sid = getattr(snap, "snapshot_id", None) or getattr(snap, "template_id", None)
+    finally:
+        b2.kill()
+
+    if sid:
+        rn = Sandbox.create(sid, timeout=120, network=_net([GATEWAY_HOST]))
+        try:
+            imp = _run(rn, 'python3 -c "import httpx;print(httpx.__version__)" && pip show wayforth-sdk | grep -i ^version')
+            if "EXIT=0" not in imp.stdout:
+                failures.append(f"#3 base deps not importable in run sandbox: {imp.stdout!r}")
+            else:
+                print(f"#3 PASS — base deps importable in run sandbox: {imp.stdout.splitlines()[:2]}")
+            # §0: run sandbox cannot reach the mirror/PyPI
+            pp = _run(rn, f'curl -sS -o /dev/null -w "%{{http_code}}" --max-time 8 {MIRROR_URL}')
+            if "ec=35" not in pp.stdout and "000" not in pp.stdout:
+                failures.append(f"#0 run reached mirror: {pp.stdout!r}")
+            else:
+                print("#0 PASS — run sandbox blocked from the mirror; build blocked from gateway")
+        finally:
+            rn.kill()
+            try:
+                Sandbox.delete_snapshot(sid)
+            except Exception:
+                pass
+
+    if failures:
+        print("DEPS LIVE PROOF: FAIL")
+        for f in failures:
+            print("  -", f)
+        return 1
+    print("DEPS LIVE PROOF: PASS")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

apps/api/services/agent_deps.py

@@ -0,0 +1,258 @@
+"""services/agent_deps.py — agent code-editing v1, Step 1: dependency pipeline.
+
+THE install-time-RCE control. A user's requirements are validated against a curated,
+pinned, hashed ALLOWLIST (agent_deps_lock.json — the only packages the private mirror
+serves), then installed WHEELS-ONLY (no setup.py execution) in an isolated build
+sandbox whose egress is restricted to the mirror only, then snapshotted into a
+per-version image that run sandboxes boot from. Nothing here trusts user input.
+
+Defense in depth (each layer independently reduces blast radius):
+  • allowlist + private mirror           → only vetted packages can install at all
+  • pinned (==) + --require-hashes        → exact, reproducible artifacts
+  • --only-binary=:all:                   → no setup.py / build hooks run at install
+  • --no-deps + locked closure            → no surprise transitive pulls
+  • --index-url=<mirror>                  → resolver can't reach public PyPI
+  • build-sandbox egress = mirror-only    → real network isolation at build (proven)
+  • (Step 2) run-sandbox egress = gateway → no exfiltration at run
+
+This module is pure logic + an injectable build orchestration; importing it is inert.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import shlex
+
+logger_name = "wayforth"
+
+# ── caps (anti-DoS / anti-bloat) ────────────────────────────────────────────────
+MAX_DIRECT_DEPS = 20
+BUILD_TIMEOUT_S = 180
+MAX_IMAGE_DELTA_MB = 500
+
+# Base closure baked into EVERY agent image. Gateway-only run egress (Step 2) makes
+# run-time pip impossible, so the SDK + http client (and httpx's pinned closure) live
+# in the image. All present in the lockfile.
+BASE_DEPS = [
+    ("wayforth-sdk", "0.9.0"),
+    ("httpx", "0.28.1"),
+    ("anyio", "4.14.1"),
+    ("sniffio", "1.3.1"),
+    ("h11", "0.16.0"),
+    ("certifi", "2026.6.17"),
+    ("idna", "3.18"),
+]
+
+_REQS_PATH = "/home/user/requirements.lock"
+_LOCK_PATH = os.path.join(os.path.dirname(__file__), "agent_deps_lock.json")
+_LOCK_CACHE: dict | None = None
+
+# A requirement must be EXACTLY name==version — nothing else (no ranges, markers,
+# extras, urls, options, editable installs).
+_PIN_RE = re.compile(r"^([A-Za-z0-9][A-Za-z0-9._-]*)==([A-Za-z0-9][A-Za-z0-9.+!_-]*)$")
+_SAFE_PATH_RE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_./-]*$")
+
+
+class DepsError(Exception):
+    """Validation/build failure. `.errors` is a list of {field, code, message}."""
+
+    def __init__(self, errors: list):
+        self.errors = errors
+        super().__init__("; ".join(e.get("message", "") for e in errors) or "deps error")
+
+
+def _norm(name: str) -> str:
+    """PEP 503 normalization for package names."""
+    return re.sub(r"[-_.]+", "-", name.strip().lower())
+
+
+def load_lockfile() -> dict:
+    """The curated allowlist: {normalized_name: {version: [sha256 hashes]}}."""
+    global _LOCK_CACHE
+    if _LOCK_CACHE is None:
+        with open(_LOCK_PATH) as f:
+            raw = json.load(f)
+        _LOCK_CACHE = {_norm(k): v for k, v in raw.items()}
+    return _LOCK_CACHE
+
+
+# ── the gatekeeper ──────────────────────────────────────────────────────────────
+
+def validate_requirements(text: str, lockfile: dict | None = None):
+    """Validate a requirements.txt body against the allowlist. Returns (pins, errors).
+
+    Each pin is (name, version, [hashes]). A non-empty `errors` list ⇒ reject (422).
+    Fail-closed: anything not exactly `name==version` from the allowlist is rejected.
+    """
+    lock = lockfile if lockfile is not None else load_lockfile()
+    pins: list = []
+    errors: list = []
+    seen: set = set()
+
+    lines = [ln.strip() for ln in (text or "").splitlines()]
+    lines = [ln for ln in lines if ln and not ln.startswith("#")]
+
+    if len(lines) > MAX_DIRECT_DEPS:
+        errors.append({"field": None, "code": "too_many",
+                       "message": f"too many dependencies ({len(lines)} > {MAX_DIRECT_DEPS})"})
+
+    for ln in lines:
+        if ln.startswith("-") or "://" in ln or " @ " in ln:
+            errors.append({"field": ln, "code": "unsupported",
+                           "message": f"unsupported requirement line: {ln!r} "
+                                      "(only 'name==version' is allowed)"})
+            continue
+        m = _PIN_RE.match(ln)
+        if not m:
+            errors.append({"field": ln, "code": "unpinned",
+                           "message": f"requirement must be pinned 'name==version': {ln!r}"})
+            continue
+        name, version = _norm(m.group(1)), m.group(2)
+        if name in seen:
+            errors.append({"field": name, "code": "duplicate",
+                           "message": f"duplicate requirement '{name}'"})
+            continue
+        seen.add(name)
+        if name not in lock:
+            errors.append({"field": name, "code": "not_allowed",
+                           "message": f"'{name}' is not in the allowlist (request it for review)"})
+            continue
+        if version not in lock[name]:
+            errors.append({"field": name, "code": "version_not_allowed",
+                           "message": f"'{name}=={version}' is not an allowed version "
+                                      f"(allowed: {sorted(lock[name])})"})
+            continue
+        pins.append((name, version, list(lock[name][version])))
+
+    return pins, errors
+
+
+def resolve_install_set(requirements_text: str, lockfile: dict | None = None):
+    """Base closure + validated user pins, deduped. Returns (install_set, errors).
+
+    A user pin that names a base dep at a different version is a conflict (base wins).
+    """
+    lock = lockfile if lockfile is not None else load_lockfile()
+    user_pins, errors = validate_requirements(requirements_text, lock)
+
+    base = {}
+    for name, version in BASE_DEPS:
+        n = _norm(name)
+        base[n] = (n, version, list(lock[n][version]))
+
+    out = dict(base)
+    for name, version, hashes in user_pins:
+        if name in base and base[name][1] != version:
+            errors.append({"field": name, "code": "base_dep_conflict",
+                           "message": f"'{name}' is a base dependency pinned to "
+                                      f"{base[name][1]}; cannot override with {version}"})
+            continue
+        out[name] = (name, version, hashes)
+
+    if errors:
+        return [], errors
+    return list(out.values()), []
+
+
+# ── install command + egress (two-allowlist) ────────────────────────────────────
+
+def build_requirements_lock(install_set) -> str:
+    """`name==version --hash=sha256:… ` lines for --require-hashes."""
+    lines = []
+    for name, version, hashes in install_set:
+        hs = " ".join(f"--hash={h}" for h in hashes)
+        lines.append(f"{name}=={version} {hs}")
+    return "\n".join(lines) + "\n"
+
+
+def pip_install_command(mirror_url: str, reqs_path: str = _REQS_PATH) -> str:
+    """The wheels-only, hashed, mirror-pinned, no-deps install — the install-RCE control."""
+    return (
+        "pip install --only-binary=:all: --require-hashes --no-deps --no-input -q "
+        f"--index-url {shlex.quote(mirror_url)} -r {shlex.quote(reqs_path)} "
+        "--break-system-packages"
+    )
+
+
+def build_egress(mirror_host: str) -> dict:
+    """BUILD sandbox egress: deny all, allow ONLY the mirror (never the gateway)."""
+    return {"deny_out": ["0.0.0.0/0"], "allow_out": [mirror_host]}
+
+
+def run_egress(gateway_host: str = "gateway.wayforth.io") -> dict:
+    """RUN sandbox egress: deny all, allow ONLY the gateway (never the mirror)."""
+    return {"deny_out": ["0.0.0.0/0"], "allow_out": [gateway_host]}
+
+
+# ── build orchestration (injectable sandbox factory for tests) ──────────────────
+
+def _safe_rel_path(path: str) -> str:
+    p = (path or "").strip().lstrip("/")
+    if not p or ".." in p.split("/") or not _SAFE_PATH_RE.match(p):
+        raise DepsError([{"field": path, "code": "bad_path",
+                          "message": f"unsafe file path: {path!r}"}])
+    return p
+
+
+def _default_sandbox_factory(egress: dict, timeout_s: int):
+    from e2b import Sandbox, SandboxNetworkOpts
+    return Sandbox.create(
+        timeout=timeout_s,
+        network=SandboxNetworkOpts(deny_out=egress["deny_out"], allow_out=egress["allow_out"]),
+    )
+
+
+def build_agent_image(
+    agent_id: str,
+    version: int,
+    files: dict,
+    requirements_text: str,
+    *,
+    mirror_url: str,
+    mirror_host: str,
+    sandbox_factory=_default_sandbox_factory,
+) -> str:
+    """Validate → isolated mirror-only build → wheels-only hashed install → snapshot.
+
+    Returns the per-version image ref (snapshot id). Raises DepsError on validation or
+    install failure. The build sandbox holds NO Wayforth secrets and can reach only the
+    mirror; it is always killed.
+    """
+    install_set, errors = resolve_install_set(requirements_text)
+    if errors:
+        raise DepsError(errors)
+
+    reqs_lock = build_requirements_lock(install_set)
+    cmd = pip_install_command(mirror_url)
+
+    sbx = sandbox_factory(build_egress(mirror_host), BUILD_TIMEOUT_S)
+    try:
+        for path, content in (files or {}).items():
+            sbx.files.write(f"/home/user/{_safe_rel_path(path)}", content)
+        sbx.files.write(_REQS_PATH, reqs_lock)
+
+        res = sbx.commands.run(cmd, timeout=float(BUILD_TIMEOUT_S))
+        if getattr(res, "exit_code", 1) != 0:
+            raise DepsError([{"field": None, "code": "install_failed",
+                              "message": (getattr(res, "stderr", "") or "")[:500]}])
+
+        # Image-size ceiling (approximate; site-packages growth).
+        du = sbx.commands.run(
+            "du -sm /usr/lib/python3*/site-packages /home/user 2>/dev/null "
+            "| awk '{s+=$1} END{print s+0}'")
+        if int((getattr(du, "stdout", "0") or "0").strip() or 0) > MAX_IMAGE_DELTA_MB:
+            raise DepsError([{"field": None, "code": "image_too_large",
+                              "message": f"image exceeds {MAX_IMAGE_DELTA_MB} MB cap"}])
+
+        snap = sbx.create_snapshot(name=f"agent-{agent_id}-v{version}")
+        image_ref = getattr(snap, "snapshot_id", None) or getattr(snap, "template_id", None)
+        if not image_ref:
+            raise DepsError([{"field": None, "code": "snapshot_failed",
+                              "message": "no image ref returned"}])
+        return image_ref
+    finally:
+        try:
+            sbx.kill()
+        except Exception:
+            pass