From 9df387ec6e22ec0edf120635a0c0d4021e8bdc4a Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:40:43 +0300 Subject: [PATCH 01/26] Add Docker-backed application runtime --- contracts/python_runtime_gap_report.json | 6 +- contracts/runtime_feature_matrix.json | 3 +- docs/reference/configuration.md | 59 ++ docs/reference/important-flags.md | 10 + docs/user/common-workflows.md | 27 + .../planning/worktree_setup_coordinator.py | 3 - python/envctl_engine/runtime/cli.py | 2 + .../runtime/docker_service_runtime.py | 666 ++++++++++++++++++ .../runtime/engine_runtime_artifacts.py | 22 +- .../runtime/engine_runtime_commands.py | 7 +- .../runtime/engine_runtime_hooks.py | 4 + python/envctl_engine/runtime/help_general.py | 1 + .../runtime/help_topic_lifecycle.py | 2 + .../runtime/lifecycle_blast_docker.py | 5 +- .../runtime/lifecycle_cleanup_orchestrator.py | 16 + .../runtime/lifecycle_dependency_stop.py | 50 ++ .../runtime/lifecycle_service_termination.py | 24 + .../runtime/lifecycle_worktree_containers.py | 15 + .../runtime/service_status_truth.py | 27 + .../runtime/startup_inspection_support.py | 3 + .../startup/finalization_run_state.py | 6 + .../startup/restart_prestop_support.py | 4 + .../startup/run_reuse_decision.py | 29 +- .../startup/run_reuse_fresh_start.py | 64 +- .../startup/run_reuse_identity.py | 10 +- .../envctl_engine/startup/run_reuse_resume.py | 4 + .../startup/service_attach_execution.py | 177 ++++- .../startup/service_project_execution.py | 46 +- python/envctl_engine/state/__init__.py | 8 + .../state/action_command_support.py | 57 +- .../state/action_health_support.py | 4 + .../envctl_engine/state/action_log_support.py | 2 + .../state/action_orchestrator.py | 19 +- python/envctl_engine/state/models.py | 4 + python/envctl_engine/state/runtime_map.py | 4 + .../runtime/test_docker_service_runtime.py | 307 ++++++++ .../runtime/test_engine_runtime_artifacts.py | 35 +- .../runtime/test_engine_runtime_commands.py | 52 ++ .../test_engine_runtime_startup_support.py | 35 + .../runtime/test_lifecycle_dependency_stop.py | 42 ++ .../test_lifecycle_worktree_cleanup.py | 37 + tests/python/runtime/test_prereq_policy.py | 31 + .../startup/test_restart_prestop_support.py | 21 + .../python/startup/test_run_reuse_identity.py | 16 + .../python/startup/test_run_reuse_support.py | 92 +++ .../startup/test_service_attach_execution.py | 73 ++ .../test_state_action_orchestrator_logs.py | 46 ++ tests/python/state/test_state_roundtrip.py | 14 + 48 files changed, 2099 insertions(+), 92 deletions(-) create mode 100644 python/envctl_engine/runtime/docker_service_runtime.py create mode 100644 tests/python/runtime/test_docker_service_runtime.py diff --git a/contracts/python_runtime_gap_report.json b/contracts/python_runtime_gap_report.json index b135b3f8e..8c7270b80 100644 --- a/contracts/python_runtime_gap_report.json +++ b/contracts/python_runtime_gap_report.json @@ -1,8 +1,8 @@ { "version": 1, - "generated_at": "2026-07-06T17:49:55+00:00", - "matrix_generated_at": "2026-07-06T17:49:55+00:00", - "matrix_sha256": "19b3bb085a8392aa9e6f99856983555c06da3606d5e81a6d1f2be7d20022899d", + "generated_at": "2026-07-10T21:33:31+00:00", + "matrix_generated_at": "2026-07-10T21:30:49+00:00", + "matrix_sha256": "9968ef658488512f15c0ebda73d08c82da359ab389b1ff880b585dd47d5bb935", "summary": { "feature_count": 59, "gap_count": 0, diff --git a/contracts/runtime_feature_matrix.json b/contracts/runtime_feature_matrix.json index a29441403..44926b96c 100644 --- a/contracts/runtime_feature_matrix.json +++ b/contracts/runtime_feature_matrix.json @@ -1,6 +1,6 @@ { "version": 1, - "generated_at": "2026-07-06T17:49:55+00:00", + "generated_at": "2026-07-10T21:30:49+00:00", "inventory_sources": { "python_supported_commands": [ "blast-all", @@ -327,6 +327,7 @@ "--deps", "--deps-parallel", "--deps-sequential", + "--docker", "--doctor", "--entire-system", "--failed", diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 2e6bb6613..aff9e7939 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -365,6 +365,65 @@ The wizard saves accepted backend/frontend test suggestions to `ENVCTL_BACKEND_T | `ENVCTL_WORKTREE_CGC_DATABASE` | `kuzudb` | Backend passed as `--database ` to `cgc context create` for generated worktree contexts. | ## Execution Policy + +### Docker application runtime + +`envctl ... --docker` preserves normal project selection, port allocation, dependency startup, +service ordering, state, dashboard, logs, health, restart, and stop behavior, but launches every +selected app service in a managed container instead of as a host process. Managed dependencies such +as Postgres, Redis, Supabase, and n8n continue to use their existing Docker adapters. + +Settings resolve service-specific keys before global fallbacks. Additional service names are +uppercased and non-alphanumeric characters become underscores, so `voice-runtime` accepts the +standard `ENVCTL_SERVICE_VOICE_RUNTIME_*` prefix (and the shorter `ENVCTL_VOICE_RUNTIME_*` alias). + +| Variable | Default | Purpose | +| --- | --- | --- | +| `ENVCTL__DOCKER_IMAGE` / `ENVCTL_DOCKER_IMAGE` | unset | Existing image to run. Without a Dockerfile setting, no build runs. | +| `ENVCTL__DOCKERFILE` / `ENVCTL_DOCKERFILE` | discovered | Dockerfile to build; discovery checks the service directory and then project root. | +| `ENVCTL__DOCKER_CONTEXT` / `ENVCTL_DOCKER_CONTEXT` | Dockerfile directory | Build context relative to the project root. | +| `ENVCTL__DOCKER_TARGET` / `ENVCTL_DOCKER_TARGET` | unset | Optional multi-stage build target. | +| `ENVCTL__DOCKER_BUILD_ARGS` / `ENVCTL_DOCKER_BUILD_ARGS` | public frontend env keys | Comma-separated build-arg names or `NAME=value` entries. Named values come from the service env and are not placed on the Docker command line. | +| `ENVCTL__DOCKER_COMMAND` / `ENVCTL_DOCKER_COMMAND` | image command | Explicit command executed inside the container. | +| `ENVCTL__DOCKER_COMMAND_MODE` / `ENVCTL_DOCKER_COMMAND_MODE` | `image` | `image` keeps the image CMD/ENTRYPOINT; `service` translates and runs envctl's normal service command. | +| `ENVCTL__DOCKER_WORKDIR` / `ENVCTL_DOCKER_WORKDIR` | image default | Container working directory. | +| `ENVCTL__DOCKER_PORT` / `ENVCTL_DOCKER_PORT` | single exposed image port or allocated host port | Container-side listener port. | +| `ENVCTL__DOCKER_BUILD_POLICY` / `ENVCTL_DOCKER_BUILD_POLICY` | `cached` | `cached`, `missing`, or `never`. | +| `ENVCTL_DOCKER_BUILD_TIMEOUT_SECONDS` | `900` | Image build timeout. | +| `ENVCTL_DOCKER_RUN_TIMEOUT_SECONDS` | `120` | Container creation timeout. | + +The default `cached` policy invokes `docker build` with BuildKit enabled on every start. Docker +evaluates the context and reuses unchanged layers, while changed source cannot accidentally reuse a +stale whole image. Generated tags are stable for one checkout/worktree, preventing parallel trees +from racing on one tag. Use `missing` for immutable/prebuilt local tags and `never` when image +lifecycle is owned outside envctl. + +Containers publish only to `127.0.0.1`, carry `io.envctl.*` ownership labels, and persist their ID, +name, and image in runtime state. Internal URLs and host settings pointing at `localhost` map to +`host.docker.internal`; browser-facing `VITE_*`, `NEXT_PUBLIC_*`, `PUBLIC_*`, CORS, and frontend URL +values remain host-facing. Service env passes through a short-lived mode-`0600` file and is not +rendered into Docker command diagnostics. Normal logs are snapshotted from Docker on demand; +`logs --follow` creates followers only for that command and tears them down on exit, so application +runtime state does not depend on long-lived host helper processes. + +```dotenv +ENVCTL_BACKEND_DOCKER_IMAGE=acme/api-dev:latest +ENVCTL_BACKEND_DOCKER_COMMAND=uvicorn app.main:app --host 0.0.0.0 --port 8000 +ENVCTL_BACKEND_DOCKER_PORT=8000 + +ENVCTL_FRONTEND_DOCKERFILE=frontend/Dockerfile +ENVCTL_FRONTEND_DOCKER_CONTEXT=frontend +ENVCTL_FRONTEND_DOCKER_PORT=5173 +``` + +```bash +envctl start --trees --entire-system --docker --headless +envctl health --json +envctl logs --project feature-a-1 --service backend +envctl restart --project feature-a-1 --service backend --docker --headless +envctl stop --project feature-a-1 --entire-system +``` + | Variable | Default | Purpose | | --- | --- | --- | | `ENVCTL_SERVICE_ATTACH_PARALLEL` | `true` | Run backend+frontend service attach in parallel when both are selected. | diff --git a/docs/reference/important-flags.md b/docs/reference/important-flags.md index 4441f34f9..0a5f853ed 100644 --- a/docs/reference/important-flags.md +++ b/docs/reference/important-flags.md @@ -89,12 +89,22 @@ Plan-agent handoff note: if `--plan --tmux/--omx --headless` starts the implemen | `--parallel-trees` | Enable parallel tree startup workers. | | `--parallel-trees-max ` | Max parallel tree startup workers. | | `--service-parallel` / `--service-sequential` | Run backend+frontend startup attach in parallel or sequential mode (default: parallel). | +| `--docker` | Build or reuse images and run backend, frontend, and additional app services as managed Docker containers. | | `--parallel` / `--sequential` | For `envctl test`, run backend/frontend test suites in parallel or sequential mode (default: parallel for multiple suites, with sequential fallback). Legacy aliases: `--test-parallel` / `--test-sequential`. | | `--test-parallel-max ` | Cap concurrent test suites in parallel mode and pytest-xdist workers when pytest-xdist is explicitly enabled (default suite cap: CPU-aware, max `4`; default pytest workers: current free CPU cores). | | `--clear-port-state` | Clear saved port reservations/state. | | `--force` | Free configured ports if needed. | | `ENVCTL_DOCKER_PREWARM=0\|1` | Enable/disable one-shot Docker daemon prewarm before requirements startup (default: `1`). | | `ENVCTL_DOCKER_PREWARM_TIMEOUT_SECONDS=` | Timeout for Docker prewarm command (default: `10`). | +| `ENVCTL__DOCKER_IMAGE=` | Use an existing image for one app service. `ENVCTL_DOCKER_IMAGE` is the global fallback. | +| `ENVCTL__DOCKERFILE=` | Build an app service image from a Dockerfile. `ENVCTL_DOCKERFILE` is the global fallback. | +| `ENVCTL__DOCKER_CONTEXT=` | Override the Docker build context; defaults to the Dockerfile directory. | +| `ENVCTL__DOCKER_BUILD_ARGS=` | Add comma-separated Docker build-arg names (or `NAME=value` entries); public frontend env keys are passed automatically. | +| `ENVCTL__DOCKER_COMMAND=` | Override the image command inside the container. | +| `ENVCTL__DOCKER_COMMAND_MODE=image\|service` | Keep image CMD/ENTRYPOINT (default) or run envctl's normal service command. | +| `ENVCTL__DOCKER_PORT=` | Override the container-side port while preserving envctl's allocated host port. | +| `ENVCTL__DOCKER_WORKDIR=` | Override the image working directory for the service command. | +| `ENVCTL__DOCKER_BUILD_POLICY=cached\|missing\|never` | `cached` (default) reuses Docker layers; `missing` builds only when absent; `never` requires an existing image. | | `ENVCTL_REQUIREMENTS_PORT_MISMATCH_POLICY=adopt_existing\|recreate` | On existing container port mismatch, adopt existing mapping (default) or force recreate. | ## Logs and Debugging diff --git a/docs/user/common-workflows.md b/docs/user/common-workflows.md index f9de37747..0b478e40e 100644 --- a/docs/user/common-workflows.md +++ b/docs/user/common-workflows.md @@ -212,6 +212,33 @@ This is especially useful: - before CI or agent automation - when debugging resume surprises +## Run the Same System in Docker + +Use `--docker` to preserve envctl's main/trees, selective-service, dependency, dashboard, health, +logs, restart, and stop workflows while replacing app host processes with managed containers: + +```bash +envctl start --main --entire-system --docker --headless +envctl start --trees --entire-system --isolated-deps --docker --headless +``` + +If a service directory contains a `Dockerfile`, envctl builds it with Docker's layer cache. Otherwise, +configure an existing image with `ENVCTL_BACKEND_DOCKER_IMAGE`, `ENVCTL_FRONTEND_DOCKER_IMAGE`, or +the corresponding prefix for an additional app service. Production-oriented images commonly need +`ENVCTL__DOCKER_COMMAND` and `ENVCTL__DOCKER_PORT` so the service listens on +`0.0.0.0` at the expected container port. + +```bash +envctl endpoints --json +envctl health --json +envctl logs --project Main --service backend +envctl restart --project Main --service backend --docker --headless +envctl stop --project Main --entire-system +``` + +See [Configuration: Docker application runtime](../reference/configuration.md#docker-application-runtime) +for image, Dockerfile, command, port, workdir, target, and cache-policy settings. + ## Debug a Bad Interactive Session Use this when the issue is in the dashboard, selector, spinner, or input flow. diff --git a/python/envctl_engine/planning/worktree_setup_coordinator.py b/python/envctl_engine/planning/worktree_setup_coordinator.py index 21bc724fb..92d2292f2 100644 --- a/python/envctl_engine/planning/worktree_setup_coordinator.py +++ b/python/envctl_engine/planning/worktree_setup_coordinator.py @@ -28,9 +28,6 @@ def apply_setup_worktree_selection( ) -> list[ProjectContextLike]: if not setup_worktree_requested(route): return project_contexts - if bool(route.flags.get("docker")): - raise RuntimeError("setup-worktrees is not supported in Docker mode.") - multi_entries = coerce_setup_entries( route=route, flag_name="setup_worktrees", diff --git a/python/envctl_engine/runtime/cli.py b/python/envctl_engine/runtime/cli.py index 9ee0d43af..92831ea00 100644 --- a/python/envctl_engine/runtime/cli.py +++ b/python/envctl_engine/runtime/cli.py @@ -80,6 +80,8 @@ def _effective_prereq_mode(route: Route) -> str: def _requires_docker(mode: str, config: EngineConfig, *, route: Route | None = None) -> bool: + if route is not None and bool(route.flags.get("docker")): + return True if route is not None and route.flags.get("launch_dependencies") is False: return False if effective_dependency_scope(route, mode) == "shared" and mode == "trees": diff --git a/python/envctl_engine/runtime/docker_service_runtime.py b/python/envctl_engine/runtime/docker_service_runtime.py new file mode 100644 index 000000000..bc14c3542 --- /dev/null +++ b/python/envctl_engine/runtime/docker_service_runtime.py @@ -0,0 +1,666 @@ +from __future__ import annotations + +import hashlib +import json +import os +import re +import shlex +import threading +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Sequence + +from envctl_engine.shared.parsing import parse_bool + + +_HOST_ENV_KEYS = { + "HOME", + "OLDPWD", + "PATH", + "PWD", + "PYTHONHOME", + "PYTHONPATH", + "SHELL", + "SHLVL", + "TMPDIR", + "VIRTUAL_ENV", + "VIRTUAL_ENV_PROMPT", + "ZDOTDIR", + "_", +} +_PUBLIC_ENV_PREFIXES = ("VITE_", "NEXT_PUBLIC_", "PUBLIC_") +_PUBLIC_ENV_KEYS = { + "BACKEND_CORS_ORIGINS", + "CORS_ORIGINS", + "FRONTEND_URL", + "PUBLIC_URL", +} +_DOCKER_INFO_LOCK = threading.Lock() +_RUNTIME_SCOPE_LABEL = "io.envctl.runtime-scope" + + +def _docker_name(value: str, *, limit: int) -> str: + normalized = re.sub(r"[^a-z0-9_.-]+", "-", value.strip().lower()).strip("-._") + return (normalized or "service")[:limit] + + +def docker_service_container_name(*, project_name: str, project_root: Path, service_name: str) -> str: + digest = hashlib.sha256(str(project_root.resolve()).encode()).hexdigest()[:10] + prefix = _docker_name(f"envctl-app-{project_name}-{service_name}", limit=52) + return f"{prefix}-{digest}" + + +@dataclass(frozen=True, slots=True) +class DockerServiceLaunch: + container_id: str + container_name: str + image: str + log_path: str + + +def docker_service_mode_enabled(runtime: Any) -> bool: + env = getattr(runtime, "env", {}) + return parse_bool(str(env.get("DOCKER_MODE", "")), False) + + +def docker_service_uses_image_command( + runtime: Any, + service_name: str, + *, + docker_mode: bool | None = None, +) -> bool: + if not (docker_service_mode_enabled(runtime) if docker_mode is None else docker_mode): + return False + env = getattr(runtime, "env", {}) + raw = getattr(getattr(runtime, "config", None), "raw", {}) + suffix = re.sub(r"[^A-Z0-9]+", "_", service_name.upper()).strip("_") or "SERVICE" + + def setting(name: str) -> str: + for key in (f"ENVCTL_SERVICE_{suffix}_{name}", f"ENVCTL_{suffix}_{name}", f"ENVCTL_{name}"): + value = str(env.get(key) or raw.get(key) or "").strip() + if value: + return value + return "" + + if setting("DOCKER_COMMAND"): + return False + return (setting("DOCKER_COMMAND_MODE") or "image").lower() in {"image", "default"} + + +def state_uses_docker_services(state: object) -> bool: + metadata = getattr(state, "metadata", {}) + if isinstance(metadata, dict) and str(metadata.get("application_runtime", "")).lower() == "docker": + return True + services = getattr(state, "services", {}) + return isinstance(services, dict) and any( + str(getattr(service, "runtime_kind", "process") or "process").lower() == "docker" + for service in services.values() + ) + + +class DockerServiceRuntime: + """Build and run application services as managed Docker containers.""" + + def __init__(self, runtime: Any, process_runtime: Any, *, refresh_cache: bool = False) -> None: + self.runtime = runtime + self.process_runtime = process_runtime + self.refresh_cache = refresh_cache + + def launch( + self, + *, + project_name: str, + project_root: Path, + service_name: str, + cwd: Path, + command: Sequence[str], + env: Mapping[str, str], + host_port: int, + container_port: int, + listener_expected: bool, + log_path: str, + ) -> DockerServiceLaunch: + self._require_docker() + image = self._resolve_image( + project_root=project_root, + service_name=service_name, + cwd=cwd, + build_env=env, + ) + container_name = self._container_name( + project_name=project_name, + project_root=project_root, + service_name=service_name, + ) + self._prepare_container_name(container_name) + effective_container_port = self._container_port(service_name, container_port, image=image) + container_env = self._container_env(env) + if listener_expected and effective_container_port > 0: + container_env["PORT"] = str(effective_container_port) + env_file = self._write_env_file( + project_name=project_name, + service_name=service_name, + values=container_env, + ) + try: + docker_command = [ + "docker", + "run", + "--detach", + "--name", + container_name, + "--label", + "io.envctl.managed=true", + "--label", + f"io.envctl.project={project_name}", + "--label", + f"io.envctl.service={service_name}", + "--label", + f"{_RUNTIME_SCOPE_LABEL}={self._runtime_scope_identity()}", + "--add-host", + "host.docker.internal:host-gateway", + "--env-file", + str(env_file), + ] + if listener_expected and host_port > 0: + docker_command.extend(["--publish", f"127.0.0.1:{host_port}:{effective_container_port}"]) + workdir = self._setting(service_name, "DOCKER_WORKDIR") + if workdir: + docker_command.extend(["--workdir", workdir]) + docker_command.append(image) + docker_command.extend( + self._container_command( + service_name, + command, + project_root=project_root, + cwd=cwd, + host_port=host_port, + container_port=effective_container_port, + ) + ) + result = self.process_runtime.run( + docker_command, + cwd=project_root, + env={**os.environ, "DOCKER_BUILDKIT": "1"}, + timeout=self._timeout("ENVCTL_DOCKER_RUN_TIMEOUT_SECONDS", 120.0), + ) + finally: + env_file.unlink(missing_ok=True) + if result.returncode != 0: + detail = str(result.stderr or result.stdout or "docker run failed").strip() + raise RuntimeError(detail) + container_id = str(result.stdout or "").strip().splitlines()[-1] + if not container_id: + raise RuntimeError(f"docker run returned no container id for {project_name} {service_name}") + launch = DockerServiceLaunch( + container_id=container_id, + container_name=container_name, + image=image, + log_path=log_path, + ) + self.runtime._emit( + "service.container.start", + project=project_name, + service=service_name, + container_name=container_name, + image=image, + port=host_port if listener_expected else None, + ) + return launch + + def wait_until_ready(self, launch: DockerServiceLaunch, *, port: int, listener_expected: bool) -> bool: + if not self.container_running(launch.container_id): + self.snapshot_logs(launch.container_id, launch.log_path, tail=200) + return False + if not listener_expected: + return True + ready = bool( + self.process_runtime.wait_for_port( + port, + timeout=self._timeout("ENVCTL_SERVICE_LISTENER_TIMEOUT_SECONDS", 30.0), + ) + ) + if not ready: + self.snapshot_logs(launch.container_id, launch.log_path, tail=200) + return ready + + def snapshot_logs(self, container: str, log_path: str, *, tail: int) -> bool: + command = ["docker", "logs"] + since_marker = Path(f"{log_path}.docker-since") + if since_marker.is_file(): + try: + command.extend(["--since", str(since_marker.stat().st_mtime)]) + except OSError: + pass + command.extend(["--tail", str(max(tail, 0)), container]) + result = self.process_runtime.run( + command, + timeout=30.0, + ) + if result.returncode != 0: + return False + stdout = str(result.stdout or "") + stderr = str(result.stderr or "") + content = stdout + (("\n" if stdout and stderr and not stdout.endswith("\n") else "") + stderr) + path = Path(log_path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return True + + def follow_logs(self, container: str, log_path: str) -> int | None: + process = self.process_runtime.start_background( + ["docker", "logs", "--follow", "--since", "0s", container], + stdout_path=log_path, + stderr_path=log_path, + ) + pid = getattr(process, "pid", None) + return pid if isinstance(pid, int) and pid > 0 else None + + def stop(self, container: str, *, remove: bool = True) -> bool: + if not container: + return True + command = ["docker", "rm", "--force", container] if remove else ["docker", "stop", container] + result = self.process_runtime.run(command, timeout=30.0) + return result.returncode == 0 or "No such container" in str(result.stderr or "") + + def container_running(self, container: str) -> bool: + if not container: + return False + result = self.process_runtime.run( + ["docker", "inspect", "--format", "{{.State.Running}}", container], + timeout=10.0, + ) + return result.returncode == 0 and str(result.stdout or "").strip().lower() == "true" + + def _resolve_image( + self, + *, + project_root: Path, + service_name: str, + cwd: Path, + build_env: Mapping[str, str], + ) -> str: + configured_image = self._setting(service_name, "DOCKER_IMAGE") + dockerfile_setting = self._setting(service_name, "DOCKERFILE") + dockerfile = self._resolve_path(dockerfile_setting, project_root) if dockerfile_setting else None + if dockerfile is None and not configured_image: + dockerfile = next( + (path for path in (cwd / "Dockerfile", project_root / "Dockerfile") if path.is_file()), + None, + ) + if dockerfile is None: + if configured_image: + return configured_image + raise RuntimeError( + f"Docker mode needs ENVCTL_{self._service_key(service_name)}_DOCKER_IMAGE, " + "ENVCTL_DOCKER_IMAGE, or a Dockerfile" + ) + if not dockerfile.is_file(): + raise RuntimeError(f"Configured Dockerfile does not exist: {dockerfile}") + context_setting = self._setting(service_name, "DOCKER_CONTEXT") + context = self._resolve_path(context_setting, project_root) if context_setting else dockerfile.parent + if context is None or not context.is_dir(): + raise RuntimeError(f"Configured Docker build context does not exist: {context}") + image = configured_image or self._generated_image_name(project_root, service_name) + self._build_image( + image=image, + dockerfile=dockerfile, + context=context, + service_name=service_name, + build_env=build_env, + ) + return image + + def _build_image( + self, + *, + image: str, + dockerfile: Path, + context: Path, + service_name: str, + build_env: Mapping[str, str], + ) -> None: + policy = (self._setting(service_name, "DOCKER_BUILD_POLICY") or "cached").strip().lower() + if policy in {"never", "image", "existing"}: + if not self._image_exists(image): + raise RuntimeError(f"Docker image {image!r} does not exist and build policy is {policy!r}") + return + if policy == "missing" and self._image_exists(image): + self.runtime._emit("service.container.build", service=service_name, image=image, cache="image") + return + command = ["docker", "build", "--file", str(dockerfile), "--tag", image] + if self.refresh_cache or parse_bool( + str(getattr(self.runtime, "env", {}).get("RUN_SH_REFRESH_CACHE", "")), + False, + ): + command.append("--pull") + target = self._setting(service_name, "DOCKER_TARGET") + if target: + command.extend(["--target", target]) + build_arguments = self._docker_build_arguments(service_name, build_env) + for key in sorted(build_arguments): + command.extend(["--build-arg", key]) + command.append(str(context)) + self.runtime._emit( + "service.container.build", + service=service_name, + image=image, + dockerfile=str(dockerfile), + context=str(context), + cache="docker_layer_cache", + ) + result = self.process_runtime.run( + command, + cwd=context, + env={**os.environ, **build_arguments, "DOCKER_BUILDKIT": "1"}, + timeout=self._timeout("ENVCTL_DOCKER_BUILD_TIMEOUT_SECONDS", 900.0), + ) + if result.returncode != 0: + detail = str(result.stderr or result.stdout or "docker build failed").strip() + raise RuntimeError(detail) + + def _image_exists(self, image: str) -> bool: + return self.process_runtime.run(["docker", "image", "inspect", image], timeout=10.0).returncode == 0 + + def _prepare_container_name(self, name: str) -> None: + inspect = self.process_runtime.run( + [ + "docker", + "container", + "inspect", + "--format", + f"{{{{.State.Running}}}}|{{{{index .Config.Labels {json.dumps(_RUNTIME_SCOPE_LABEL)}}}}}", + name, + ], + timeout=10.0, + ) + if inspect.returncode != 0: + if "No such" in str(inspect.stderr or "") or "not found" in str(inspect.stderr or "").lower(): + return + raise RuntimeError(str(inspect.stderr or inspect.stdout or f"failed to inspect {name}").strip()) + running_text, _separator, owner = str(inspect.stdout or "").strip().partition("|") + expected_owner = self._runtime_scope_identity() + if owner and owner != expected_owner: + raise RuntimeError( + f"Docker container {name!r} is managed by another envctl runtime scope; " + "stop it from that runtime before starting this one" + ) + if running_text.lower() == "true": + raise RuntimeError( + f"Docker container {name!r} is already running but was not resumed from current state; " + "run envctl stop for this project before starting it again" + ) + self._remove_container(name) + + def _remove_container(self, name: str) -> None: + result = self.process_runtime.run(["docker", "rm", "--force", name], timeout=30.0) + if result.returncode != 0 and "No such container" not in str(result.stderr or ""): + raise RuntimeError(str(result.stderr or result.stdout or f"failed to remove {name}").strip()) + + def _runtime_scope_identity(self) -> str: + config = getattr(self.runtime, "config", None) + scope = getattr(config, "runtime_scope_dir", None) + if scope is None: + scope = Path(self.runtime._run_dir_path("docker-scope")).parent + return hashlib.sha256(str(Path(scope).resolve()).encode()).hexdigest()[:16] + + def _require_docker(self) -> None: + command_exists = getattr(self.runtime, "_command_exists", None) + if callable(command_exists) and not command_exists("docker"): + raise RuntimeError("Docker mode requires the docker CLI") + with _DOCKER_INFO_LOCK: + if bool(getattr(self.runtime, "_docker_application_daemon_checked", False)): + return + result = self.process_runtime.run( + ["docker", "info", "--format", "{{.ServerVersion}}"], + timeout=15.0, + ) + if result.returncode != 0: + raise RuntimeError(str(result.stderr or "Docker daemon is unavailable").strip()) + setattr(self.runtime, "_docker_application_daemon_checked", True) + + def _container_command( + self, + service_name: str, + command: Sequence[str], + *, + project_root: Path, + cwd: Path, + host_port: int, + container_port: int, + ) -> list[str]: + override = self._setting(service_name, "DOCKER_COMMAND") + command_mode = (self._setting(service_name, "DOCKER_COMMAND_MODE") or "image").lower() + if override: + resolved = shlex.split(override) + elif command_mode in {"service", "envctl", "host"}: + resolved = [str(part) for part in command] + elif command_mode in {"image", "default"}: + return [] + else: + raise RuntimeError( + f"Invalid Docker command mode for {service_name}: {command_mode!r}; expected image or service" + ) + if not resolved: + return [] + executable = Path(resolved[0]) + if executable.is_absolute(): + try: + resolved[0] = str(executable.relative_to(cwd)) + except ValueError: + try: + resolved[0] = str(executable.relative_to(project_root)) + except ValueError: + pass + normalized: list[str] = [] + for part in resolved: + if part in {"127.0.0.1", "localhost"}: + normalized.append("0.0.0.0") + continue + if host_port > 0 and container_port > 0: + if part == str(host_port): + normalized.append(str(container_port)) + continue + part = re.sub( + rf"^(--port=){re.escape(str(host_port))}$", + rf"\g<1>{container_port}", + part, + ) + part = re.sub(r"^(--host=)(localhost|127\.0\.0\.1)$", r"\g<1>0.0.0.0", part) + normalized.append(part) + return normalized + + def _docker_build_arguments( + self, + service_name: str, + build_env: Mapping[str, str], + ) -> dict[str, str]: + arguments = { + str(key): str(value) + for key, value in build_env.items() + if str(key).startswith(_PUBLIC_ENV_PREFIXES) + } + configured = self._setting(service_name, "DOCKER_BUILD_ARGS") + for token in configured.split(","): + item = token.strip() + if not item: + continue + key, separator, configured_value = item.partition("=") + key = key.strip() + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + raise RuntimeError(f"Invalid Docker build argument name: {key!r}") + if separator: + arguments[key] = configured_value + elif key in build_env: + arguments[key] = str(build_env[key]) + else: + raise RuntimeError(f"Docker build argument {key!r} is not present in the service environment") + return arguments + + def _container_env(self, values: Mapping[str, str]) -> dict[str, str]: + result: dict[str, str] = {} + for raw_key, raw_value in values.items(): + key = str(raw_key) + if key in _HOST_ENV_KEYS or key.startswith("TERM_") or key.startswith("LC_"): + continue + value = str(raw_value) + if key in {"HOST", "BIND_HOST", "LISTEN_HOST", "SERVER_HOST"} and value in { + "localhost", + "127.0.0.1", + }: + result[key] = "0.0.0.0" + continue + if key not in _PUBLIC_ENV_KEYS and not key.startswith(_PUBLIC_ENV_PREFIXES): + value = re.sub( + r"(://(?:[^/@\s]+@)?)(localhost|127\.0\.0\.1)(?=[:/]|$)", + r"\1host.docker.internal", + value, + ) + if key.endswith("_HOST") and value in {"localhost", "127.0.0.1"}: + value = "host.docker.internal" + result[key] = value + result["ENVCTL_CONTAINER_RUNTIME"] = "docker" + result["ENVCTL_HOST_GATEWAY"] = "host.docker.internal" + return result + + def _write_env_file(self, *, project_name: str, service_name: str, values: Mapping[str, str]) -> Path: + run_dir = Path(self.runtime._run_dir_path(getattr(self.runtime, "_active_run_id", None) or "docker")) + run_dir.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256(f"{project_name}\0{service_name}".encode()).hexdigest()[:12] + path = run_dir / f".docker-env-{digest}" + lines = [f"{key}={str(value).replace(chr(10), '')}" for key, value in sorted(values.items())] + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + path.chmod(0o600) + return path + + def _setting(self, service_name: str, suffix: str) -> str: + env = getattr(self.runtime, "env", {}) + raw = getattr(getattr(self.runtime, "config", None), "raw", {}) + service_key = f"ENVCTL_{self._service_key(service_name)}_{suffix}" + additional_service_key = f"ENVCTL_SERVICE_{self._service_key(service_name)}_{suffix}" + global_key = f"ENVCTL_{suffix}" + for key in (additional_service_key, service_key, global_key): + value = str(env.get(key) or raw.get(key) or "").strip() + if value: + return value + return "" + + def _container_port(self, service_name: str, host_port: int, *, image: str) -> int: + value = self._setting(service_name, "DOCKER_PORT") + if not value: + exposed = self._single_exposed_port(image) + return exposed if exposed is not None else host_port + try: + port = int(value) + except ValueError as exc: + raise RuntimeError(f"Invalid Docker container port for {service_name}: {value!r}") from exc + if port <= 0 or port > 65535: + raise RuntimeError(f"Invalid Docker container port for {service_name}: {value!r}") + return port + + def _single_exposed_port(self, image: str) -> int | None: + result = self.process_runtime.run( + ["docker", "image", "inspect", "--format", "{{json .Config.ExposedPorts}}", image], + timeout=10.0, + ) + if result.returncode != 0: + return None + try: + payload = json.loads(str(result.stdout or "null")) + except json.JSONDecodeError: + return None + if not isinstance(payload, dict) or len(payload) != 1: + return None + raw_port = str(next(iter(payload))).partition("/")[0] + try: + port = int(raw_port) + except ValueError: + return None + return port if 0 < port <= 65535 else None + + def _container_name(self, *, project_name: str, project_root: Path, service_name: str) -> str: + return docker_service_container_name( + project_name=project_name, + project_root=project_root, + service_name=service_name, + ) + + def _generated_image_name(self, project_root: Path, service_name: str) -> str: + identity = self._repository_identity(project_root) + digest = hashlib.sha256(str(project_root.resolve()).encode()).hexdigest()[:10] + repo = _docker_name(identity.name, limit=32) + service = _docker_name(service_name, limit=24) + return f"envctl/{repo}-{service}:{digest}" + + @staticmethod + def _repository_identity(project_root: Path) -> Path: + dot_git = project_root / ".git" + if dot_git.is_file(): + try: + gitdir = dot_git.read_text(encoding="utf-8").strip().removeprefix("gitdir:").strip() + resolved = (project_root / gitdir).resolve() + if resolved.parent.name == "worktrees": + return resolved.parent.parent.parent + except OSError: + pass + return project_root.resolve() + + @staticmethod + def _service_key(service_name: str) -> str: + return re.sub(r"[^A-Z0-9]+", "_", service_name.upper()).strip("_") or "SERVICE" + + @staticmethod + def _resolve_path(value: str, project_root: Path) -> Path | None: + if not value: + return None + path = Path(value).expanduser() + return path.resolve() if path.is_absolute() else (project_root / path).resolve() + + def _timeout(self, key: str, default: float) -> float: + value = str(getattr(self.runtime, "env", {}).get(key, "") or "").strip() + try: + return max(float(value), 1.0) if value else default + except ValueError: + return default + + +def refresh_container_log_snapshots( + runtime: Any, + state: object, + *, + tail: int, + follow: bool, +) -> list[int]: + docker_services = [ + service + for service in getattr(state, "services", {}).values() + if str(getattr(service, "runtime_kind", "process") or "process").lower() == "docker" + ] + if not docker_services: + return [] + process_runner = getattr(runtime, "process_runner", None) + if process_runner is None: + return [] + docker_runtime = DockerServiceRuntime(runtime, process_runner) + follower_pids: list[int] = [] + for service in docker_services: + container = str( + getattr(service, "container_id", "") or getattr(service, "container_name", "") or "" + ).strip() + log_path = str(getattr(service, "log_path", "") or "").strip() + if not container or not log_path: + continue + docker_runtime.snapshot_logs(container, log_path, tail=tail) + if follow: + pid = docker_runtime.follow_logs(container, log_path) + if pid is not None: + follower_pids.append(pid) + return follower_pids + + +def stop_container_log_followers(runtime: Any, pids: Sequence[int]) -> None: + for pid in pids: + try: + runtime.process_runner.terminate_process_group(pid, term_timeout=1.0, kill_timeout=1.0) + except Exception: # noqa: BLE001 + continue diff --git a/python/envctl_engine/runtime/engine_runtime_artifacts.py b/python/envctl_engine/runtime/engine_runtime_artifacts.py index aaf7d9310..2f50a73f5 100644 --- a/python/envctl_engine/runtime/engine_runtime_artifacts.py +++ b/python/envctl_engine/runtime/engine_runtime_artifacts.py @@ -221,12 +221,28 @@ def _runtime_readiness_async_enabled(runtime: Any) -> bool: def print_summary(_runtime: Any, state: object, contexts: Sequence[_SummaryContext]) -> None: - _ = state print("envctl Python engine run summary") for context in contexts: - backend = context.ports["backend"].final - frontend = context.ports["frontend"].final + backend = _summary_service_port(state, context.name, "backend") or context.ports["backend"].final + frontend = _summary_service_port(state, context.name, "frontend") or context.ports["frontend"].final db = context.ports["db"].final redis = context.ports["redis"].final n8n = context.ports["n8n"].final print(f"- {context.name}: backend={backend} frontend={frontend} db={db} redis={redis} n8n={n8n}") + + +def _summary_service_port(state: object, project_name: str, service_type: str) -> int | None: + for service_name, service in getattr(state, "services", {}).items(): + project = str(getattr(service, "project", "") or "").strip() + if not project: + suffix = f" {service_type}".lower() + rendered_name = str(service_name) + project = rendered_name[: -len(suffix)] if rendered_name.lower().endswith(suffix) else "" + if project.casefold() != project_name.casefold(): + continue + if str(getattr(service, "type", "") or "").casefold() != service_type.casefold(): + continue + port = getattr(service, "actual_port", None) or getattr(service, "requested_port", None) + if isinstance(port, int) and port > 0: + return port + return None diff --git a/python/envctl_engine/runtime/engine_runtime_commands.py b/python/envctl_engine/runtime/engine_runtime_commands.py index 49fbddf58..39c5a5c95 100644 --- a/python/envctl_engine/runtime/engine_runtime_commands.py +++ b/python/envctl_engine/runtime/engine_runtime_commands.py @@ -7,6 +7,7 @@ from typing import Any, Mapping from envctl_engine.runtime.command_resolution import resolve_requirement_start_command, resolve_service_start_command +from envctl_engine.runtime.docker_service_runtime import docker_service_uses_image_command SERVICE_LAUNCH_ENV_REMOVALS = { "ACTIONS_CACHE_URL", @@ -105,13 +106,17 @@ def service_start_command_resolved( project_root: Path | None = None, port: int = 0, ) -> tuple[list[str], str]: + command_exists = runtime._command_exists + if docker_service_uses_image_command(runtime, service_name): + def command_exists(_executable: str) -> bool: + return True result = resolve_service_start_command( service_name=service_name, project_root=(project_root or runtime.config.base_dir), port=port, env=runtime.env, config_raw=runtime.config.raw, - command_exists=runtime._command_exists, + command_exists=command_exists, ) return result.command, result.source diff --git a/python/envctl_engine/runtime/engine_runtime_hooks.py b/python/envctl_engine/runtime/engine_runtime_hooks.py index 2085ec347..4ba8394ad 100644 --- a/python/envctl_engine/runtime/engine_runtime_hooks.py +++ b/python/envctl_engine/runtime/engine_runtime_hooks.py @@ -148,6 +148,10 @@ def services_from_hook_payload( actual_port=(actual_port if actual_port > 0 else None), log_path=(str(log_path) if log_path else None), status=status, + runtime_kind=str(raw.get("runtime_kind") or "process"), + container_id=str(raw.get("container_id") or "") or None, + container_name=str(raw.get("container_name") or "") or None, + container_image=str(raw.get("container_image") or "") or None, ) records[name] = record if service_type == "backend" and actual_port > 0: diff --git a/python/envctl_engine/runtime/help_general.py b/python/envctl_engine/runtime/help_general.py index 0e7b00251..7451029a7 100644 --- a/python/envctl_engine/runtime/help_general.py +++ b/python/envctl_engine/runtime/help_general.py @@ -87,6 +87,7 @@ def render_general_help() -> str: " --only-backend launch only backend; skip frontend and dependencies/prep", " --no-deps skip managed dependencies and plan-agent dependency prep", " --no-infra skip backend, frontend, managed dependencies, and plan-agent prep", + " --docker run configured app services as managed Docker containers", "", "Important global flags:", " --headless / --batch do not prompt; use deterministic automation-friendly output", diff --git a/python/envctl_engine/runtime/help_topic_lifecycle.py b/python/envctl_engine/runtime/help_topic_lifecycle.py index 4de800366..c80ba9975 100644 --- a/python/envctl_engine/runtime/help_topic_lifecycle.py +++ b/python/envctl_engine/runtime/help_topic_lifecycle.py @@ -26,6 +26,7 @@ "--deps-sequential|--sequential-deps force managed dependencies to start one at a time", "--project limit startup to selected project/worktree when supported", "--no-resume skip automatic resume of a compatible saved run", + "--docker build/use images and run app services in managed containers", ), examples=( "envctl", @@ -33,6 +34,7 @@ "envctl start --trees --entire-system --headless", "envctl start --project feature-a-1 --backend --headless", "envctl --trees --only-backend --headless", + "envctl start --trees --entire-system --docker --headless", ), related=("restart", "resume", "dashboard", "stop", "health"), ), diff --git a/python/envctl_engine/runtime/lifecycle_blast_docker.py b/python/envctl_engine/runtime/lifecycle_blast_docker.py index 73997ac0a..ab62412c1 100644 --- a/python/envctl_engine/runtime/lifecycle_blast_docker.py +++ b/python/envctl_engine/runtime/lifecycle_blast_docker.py @@ -10,7 +10,10 @@ def matches_blast_container(*, image: str, name: str) -> bool: image_l = image.lower() name_l = name.lower() - return any(token in name_l or token in image_l for token in ("supabase", "n8n", "redis", "postgres")) + return any( + token in name_l or token in image_l + for token in ("envctl-app-", "supabase", "n8n", "redis", "postgres") + ) class BlastDockerCleanupSupport: diff --git a/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py b/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py index b2680e985..f145dc768 100644 --- a/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py +++ b/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py @@ -3,6 +3,7 @@ from collections.abc import Iterable from typing import Any, Callable, Mapping, cast +from envctl_engine.requirements.core import dependency_definitions from envctl_engine.runtime.lifecycle_blast_support import LifecycleBlastCleanupSupport from envctl_engine.runtime.engine_runtime_lifecycle_support import release_requirement_ports from envctl_engine.runtime.lifecycle_dependency_stop import ( @@ -10,6 +11,7 @@ release_selected_dependency_components, requirements_have_enabled_components, select_dependency_components_for_stop, + stop_requirement_component_containers, ) from envctl_engine.runtime.lifecycle_service_stop_selection import ( select_services_for_stop, @@ -223,6 +225,11 @@ def operation() -> int: return 0 for project in list(state.requirements): + requirements = state.requirements[project] + for definition in dependency_definitions(): + component = requirements.component(definition.id) + if bool(component.get("enabled", False)): + self._stop_requirement_component_containers(component) self._release_requirement_ports(state.requirements[project]) state.requirements.pop(project, None) @@ -253,8 +260,12 @@ def _release_selected_dependency_components( state, selected_dependencies, release_component_ports_fn=self._release_requirement_component_ports, + stop_component_fn=self._stop_requirement_component_containers, ) + def _stop_requirement_component_containers(self, component: Mapping[str, object]) -> None: + stop_requirement_component_containers(self.runtime, component) + def _release_requirement_component_ports(self, component: Mapping[str, object]) -> None: port_allocator = self._port_allocator(self.runtime) release_requirement_component_ports(component, release_port_fn=port_allocator.release) @@ -312,6 +323,11 @@ def clear_runtime_state(self, *, command: str, aggressive: bool = False, route: aggressive=aggressive, verify_ownership=not aggressive, ) + for requirements in state.requirements.values(): + for definition in dependency_definitions(): + component = requirements.component(definition.id) + if bool(component.get("enabled", False)): + self._stop_requirement_component_containers(component) if aggressive and self.blast_all_ecosystem_enabled(): self.blast_all_ecosystem_cleanup(route=route) diff --git a/python/envctl_engine/runtime/lifecycle_dependency_stop.py b/python/envctl_engine/runtime/lifecycle_dependency_stop.py index aedc97432..c18c7ba02 100644 --- a/python/envctl_engine/runtime/lifecycle_dependency_stop.py +++ b/python/envctl_engine/runtime/lifecycle_dependency_stop.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable, Mapping +from typing import Any from envctl_engine.requirements.core import dependency_definitions from envctl_engine.runtime.command_router import Route @@ -9,6 +10,17 @@ def select_dependency_components_for_stop(state: RunState, route: Route) -> dict[str, set[str]]: + if route.flags.get("runtime_scope") == "entire-system": + requested_projects = {str(project).strip().casefold() for project in route.projects if str(project).strip()} + return { + project_name: { + definition.id + for definition in dependency_definitions() + if bool(requirements.component(definition.id).get("enabled", False)) + } + for project_name, requirements in state.requirements.items() + if not requested_projects or str(project_name).strip().casefold() in requested_projects + } raw_components = route.flags.get("stop_dependency_components") if not isinstance(raw_components, list): return {} @@ -38,6 +50,7 @@ def release_selected_dependency_components( selected_dependencies: dict[str, set[str]], *, release_component_ports_fn: Callable[[Mapping[str, object]], None], + stop_component_fn: Callable[[Mapping[str, object]], None] | None = None, ) -> None: for project_name, dependency_ids in selected_dependencies.items(): requirements = state.requirements.get(project_name) @@ -48,6 +61,8 @@ def release_selected_dependency_components( if not bool(component.get("enabled", False)): continue if not bool(component.get("external")): + if stop_component_fn is not None: + stop_component_fn(component) release_component_ports_fn(component) requirements.components[dependency_id] = {} if not requirements_have_enabled_components(requirements): @@ -72,3 +87,38 @@ def requirements_have_enabled_components(requirements: object) -> bool: for component in components.values() if isinstance(component, Mapping) ) + + +def stop_requirement_component_containers(runtime: Any, component: Mapping[str, object]) -> None: + if bool(component.get("external")) or bool(component.get("simulated")): + return + container_name = str(component.get("container_name") or "").strip() + if not container_name: + return + process_runner = getattr(runtime, "process_runner", None) + if process_runner is None: + return + if str(component.get("id") or "").strip().lower() == "supabase": + listed = process_runner.run( + [ + "docker", + "ps", + "--all", + "--quiet", + "--filter", + f"label=com.docker.compose.project={container_name}", + ], + timeout=30.0, + ) + if listed.returncode != 0: + raise RuntimeError(str(listed.stderr or listed.stdout or "failed listing Supabase containers").strip()) + container_ids = [line.strip() for line in str(listed.stdout or "").splitlines() if line.strip()] + if container_ids: + removed = process_runner.run(["docker", "rm", "--force", *container_ids], timeout=60.0) + if removed.returncode != 0: + detail = removed.stderr or removed.stdout or "failed stopping Supabase containers" + raise RuntimeError(str(detail).strip()) + return + removed = process_runner.run(["docker", "rm", "--force", container_name], timeout=30.0) + if removed.returncode != 0 and "No such container" not in str(removed.stderr or ""): + raise RuntimeError(str(removed.stderr or removed.stdout or f"failed stopping {container_name}").strip()) diff --git a/python/envctl_engine/runtime/lifecycle_service_termination.py b/python/envctl_engine/runtime/lifecycle_service_termination.py index 0c8ff9841..88ca18af7 100644 --- a/python/envctl_engine/runtime/lifecycle_service_termination.py +++ b/python/envctl_engine/runtime/lifecycle_service_termination.py @@ -45,6 +45,30 @@ def terminate_one(item: tuple[str, object]) -> tuple[str, bool, int | None]: def terminate_service_record(runtime: Any, service: object, *, aggressive: bool, verify_ownership: bool) -> bool: + if str(getattr(service, "runtime_kind", "process") or "process").lower() == "docker": + from envctl_engine.runtime.docker_service_runtime import DockerServiceRuntime + + container = str( + getattr(service, "container_id", "") or getattr(service, "container_name", "") or "" + ).strip() + stopped = DockerServiceRuntime(runtime, runtime.process_runner).stop(container) + pid = getattr(service, "pid", None) + if isinstance(pid, int) and pid > 0: + try: + runtime.process_runner.terminate_process_group( + pid, + term_timeout=0.5 if aggressive else 2.0, + kill_timeout=1.0, + ) + except Exception: # noqa: BLE001 + pass + runtime._emit( + "service.container.stop", + service=getattr(service, "name", "unknown"), + container_name=getattr(service, "container_name", None), + stopped=stopped, + ) + return stopped pid = getattr(service, "pid", None) if not isinstance(pid, int) or pid <= 0: return True diff --git a/python/envctl_engine/runtime/lifecycle_worktree_containers.py b/python/envctl_engine/runtime/lifecycle_worktree_containers.py index 55717aab7..6f796743b 100644 --- a/python/envctl_engine/runtime/lifecycle_worktree_containers.py +++ b/python/envctl_engine/runtime/lifecycle_worktree_containers.py @@ -7,6 +7,7 @@ from envctl_engine.requirements.common_contracts import build_container_name from envctl_engine.requirements.docker_runtime import run_docker, run_result_error from envctl_engine.requirements.supabase import build_supabase_project_name +from envctl_engine.runtime.docker_service_runtime import docker_service_container_name def legacy_container_name(*, prefix: str, project_name: str) -> str: @@ -36,6 +37,20 @@ def remove_tree_containers( legacy_container_name(prefix="envctl-redis", project_name=project_name): "redis", legacy_container_name(prefix="envctl-n8n", project_name=project_name): "n8n", } + app_service_names = ["backend", "frontend"] + app_service_names.extend( + str(getattr(service, "name", "") or "").strip() + for service in getattr(getattr(runtime, "config", None), "additional_services", ()) + if str(getattr(service, "name", "") or "").strip() + ) + for service_name in app_service_names: + exact_names[ + docker_service_container_name( + project_name=project_name, + project_root=resolved_root, + service_name=service_name, + ) + ] = service_name supabase_prefixes: set[str] = set() if include_supabase: supabase_prefixes = { diff --git a/python/envctl_engine/runtime/service_status_truth.py b/python/envctl_engine/runtime/service_status_truth.py index c5001a5a3..e2335c245 100644 --- a/python/envctl_engine/runtime/service_status_truth.py +++ b/python/envctl_engine/runtime/service_status_truth.py @@ -6,6 +6,33 @@ def service_truth_status(runtime: Any, service: object) -> str: + if str(getattr(service, "runtime_kind", "process") or "process").lower() == "docker": + from envctl_engine.runtime.docker_service_runtime import DockerServiceRuntime + + container = str( + getattr(service, "container_id", "") or getattr(service, "container_name", "") or "" + ).strip() + docker_runtime = DockerServiceRuntime(runtime, runtime.process_runner) + if not docker_runtime.container_running(container): + status = "stale" + elif not bool(getattr(service, "listener_expected", True)): + status = "running" + else: + port = getattr(service, "actual_port", None) or getattr(service, "requested_port", None) + try: + reachable = isinstance(port, int) and port > 0 and bool( + runtime.process_runner.wait_for_port(port, timeout=runtime._service_truth_timeout()) + ) + except Exception: # noqa: BLE001 + reachable = False + status = "running" if reachable else "unreachable" + runtime._emit( + "service.truth.check", + service=service_display_name(service), + status=status, + reason_code=f"container_truth_status_{status}", + ) + return status backend = getattr(runtime.process_probe, "backend", None) if isinstance(backend, ShellProbeBackend): backend.process_runner = runtime.process_runner diff --git a/python/envctl_engine/runtime/startup_inspection_support.py b/python/envctl_engine/runtime/startup_inspection_support.py index 58ec1b20e..1b2cb1f22 100644 --- a/python/envctl_engine/runtime/startup_inspection_support.py +++ b/python/envctl_engine/runtime/startup_inspection_support.py @@ -84,6 +84,9 @@ def build(self) -> dict[str, Any]: payload = { "command": getattr(self.startup_route, "command", "start"), "mode": self.runtime_mode, + "application_runtime": ( + "docker" if bool(getattr(self.startup_route, "flags", {}).get("docker")) else "process" + ), "headless": self.batch or not self.can_tty, "interactive_tty": self.can_tty, "selection": selection_info, diff --git a/python/envctl_engine/startup/finalization_run_state.py b/python/envctl_engine/startup/finalization_run_state.py index 37a34cc90..423e4377a 100644 --- a/python/envctl_engine/startup/finalization_run_state.py +++ b/python/envctl_engine/startup/finalization_run_state.py @@ -54,6 +54,9 @@ def build_planning_dashboard_state( ), } ) + metadata["application_runtime"] = ( + "docker" if bool(route.flags.get("docker")) else "process" + ) run_state = RunState( run_id=run_id, mode=runtime_mode, @@ -80,6 +83,9 @@ def _build_run_state(runtime: StartupRuntime, session: StartupSession, *, failed { "command": session.effective_route.command, "repo_scope_id": runtime.config.runtime_scope_id, + "application_runtime": ( + "docker" if bool(session.effective_route.flags.get("docker")) else "process" + ), } ) dependency_mode = effective_dependency_scope(session.effective_route, session.runtime_mode) diff --git a/python/envctl_engine/startup/restart_prestop_support.py b/python/envctl_engine/startup/restart_prestop_support.py index aa0d1e981..72c2ea6d8 100644 --- a/python/envctl_engine/startup/restart_prestop_support.py +++ b/python/envctl_engine/startup/restart_prestop_support.py @@ -5,6 +5,7 @@ from typing import Any, Callable, Iterable from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.docker_service_runtime import state_uses_docker_services from envctl_engine.runtime.runtime_context import resolve_port_allocator, resolve_process_runtime from envctl_engine.shared.services import service_project_name, service_slug_from_record from envctl_engine.startup.startup_selection_support import ( @@ -77,6 +78,9 @@ def restart_prestop_state(*, route: Route, runtime: Any) -> RestartPrestopState: run_id=resumed.run_id, ) resumed = None + if resumed is not None and state_uses_docker_services(resumed): + route.flags["docker"] = True + runtime.env["DOCKER_MODE"] = "true" fallback_route = None if resumed is None: fallback_route = restart_fallback_start_route(route, restart_lookup_mode=restart_lookup_mode) diff --git a/python/envctl_engine/startup/run_reuse_decision.py b/python/envctl_engine/startup/run_reuse_decision.py index c97bcfc65..4c0a9228e 100644 --- a/python/envctl_engine/startup/run_reuse_decision.py +++ b/python/envctl_engine/startup/run_reuse_decision.py @@ -7,6 +7,7 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.state.models import RunState from envctl_engine.startup.run_reuse_identity import ( + ProjectIdentity, _auto_resume_start_enabled, _identity_keys, _root_mismatches, @@ -57,6 +58,21 @@ def evaluate(self) -> RunReuseDecision: startup_enabled=startup_enabled, ) if not _auto_resume_start_enabled(self.route): + if bool(self.route.flags.get("no_resume")): + candidate = self.runtime._try_load_existing_state( + mode=self.runtime_mode, + strict_mode_match=True, + ) + if candidate is not None: + state_identities = project_identities_from_state(self.runtime, candidate) + return RunReuseDecision( + candidate_state=candidate, + decision_kind="fresh_run", + reason="explicit_no_resume", + selected_projects=selected_payload, + state_projects=identities_to_payload(state_identities), + startup_enabled=startup_enabled, + ) return self._fresh_without_state( reason="auto_resume_disabled", selected_payload=selected_payload, @@ -97,7 +113,7 @@ def _evaluate_candidate( self, *, candidate: RunState, - selected_identities: list[object], + selected_identities: list[ProjectIdentity], selected_payload: list[dict[str, str | None]], startup_enabled: bool, ) -> RunReuseDecision: @@ -200,7 +216,11 @@ def _initial_candidate_rejection( ) return None - def _startup_identity_mismatch(self, candidate: RunState, state_identities: list[object]) -> dict[str, object]: + def _startup_identity_mismatch( + self, + candidate: RunState, + state_identities: list[ProjectIdentity], + ) -> dict[str, object]: previous_identity = candidate.metadata.get("startup_identity") if not isinstance(previous_identity, dict): return {} @@ -316,10 +336,7 @@ def _candidate_fresh( def mark_run_reused(metadata: Mapping[str, object] | None, *, reason: str) -> dict[str, object]: updated = dict(metadata or {}) raw_count = updated.get("run_reuse_count", 0) - try: - count = int(raw_count) - except (TypeError, ValueError): - count = 0 + count = int(raw_count) if isinstance(raw_count, (bool, float, int, str)) else 0 updated["run_reuse_count"] = max(0, count) + 1 updated["last_reopened_at"] = datetime.now(tz=UTC).isoformat() updated["last_reuse_reason"] = reason diff --git a/python/envctl_engine/startup/run_reuse_fresh_start.py b/python/envctl_engine/startup/run_reuse_fresh_start.py index 2cb75fc60..e1b350e21 100644 --- a/python/envctl_engine/startup/run_reuse_fresh_start.py +++ b/python/envctl_engine/startup/run_reuse_fresh_start.py @@ -18,26 +18,35 @@ def fresh_start_replacement_services( additional_services: tuple[object, ...], project_name_from_service: Callable[[str], str], ) -> set[str]: - target_projects = {str(context.name).strip().lower() for context in selected_contexts} + target_projects = {str(getattr(context, "name", "")).strip().lower() for context in selected_contexts} target_projects.discard("") if not target_projects: return set() selected_by_project = { - str(context.name).strip().lower(): _restart_service_types_for_project( + str(getattr(context, "name", "")).strip().lower(): _restart_service_types_for_project( route=route, - project_name=str(context.name), + project_name=str(getattr(context, "name", "")), default_service_types=configured_service_types, additional_services=additional_services, ) for context in selected_contexts - if str(context.name).strip() + if str(getattr(context, "name", "")).strip() } + runtime_scope = str(route.flags.get("runtime_scope") or "").strip().lower() + replace_all_for_explicit_fresh_start = ( + bool(route.flags.get("no_resume")) + and not bool(route.flags.get("services")) + and runtime_scope not in {"backend", "frontend", "dependencies", "fullstack"} + ) selected: set[str] = set() for service_name, service in getattr(candidate_state, "services", {}).items(): project = service_project_name(service) or project_name_from_service(service_name) project_key = str(project).strip().lower() if project_key not in target_projects: continue + if replace_all_for_explicit_fresh_start: + selected.add(service_name) + continue service_type = service_slug_from_record(service) if service_type and service_type in selected_by_project.get(project_key, set()): selected.add(service_name) @@ -56,7 +65,7 @@ class FreshStartServiceReplacer: terminate_restart_orphan_listeners: Callable[..., None] def replace(self) -> None: - if self.reason != "startup_fingerprint_mismatch": + if self.reason not in {"explicit_no_resume", "startup_fingerprint_mismatch"}: return route = self.session.effective_route if route.flags.get("runtime_scope") == "dependencies": @@ -65,10 +74,12 @@ def replace(self) -> None: if not selected_services: return self.announce_session_identifiers(self.session) - self.report_progress( - route, - f"Startup selection changed; replacing {len(selected_services)} existing service(s)...", + message = ( + f"Fresh start requested; replacing {len(selected_services)} existing service(s)..." + if self.reason == "explicit_no_resume" + else f"Startup selection changed; replacing {len(selected_services)} existing service(s)..." ) + self.report_progress(route, message) self.runtime._emit( "state.run_reuse.replace_existing_services", run_id=self.candidate_state.run_id, @@ -87,6 +98,43 @@ def replace(self) -> None: selected_services=selected_services, aggressive=True, ) + self._rebind_replacement_ports(selected_services) + + def _rebind_replacement_ports(self, selected_services: set[str]) -> None: + port_planner = getattr(self.runtime, "port_planner", None) + release = getattr(port_planner, "release", None) + if not callable(release): + return + contexts = { + str(getattr(context, "name", "")).casefold(): context + for context in getattr(self.session, "selected_contexts", []) + } + for name in sorted(selected_services): + service = getattr(self.candidate_state, "services", {}).get(name) + if service is None or not bool(getattr(service, "listener_expected", True)): + continue + project = service_project_name(service) or self.runtime._project_name_from_service(name) + service_type = service_slug_from_record(service) + context = contexts.get(str(project).casefold()) + plans = getattr(context, "ports", {}) if context is not None else {} + plan = plans.get(service_type) if service_type else None + old_port = getattr(service, "actual_port", None) or getattr(service, "requested_port", None) + current_port = getattr(plan, "final", None) + if plan is None or not isinstance(old_port, int) or old_port <= 0: + continue + owner = f"{getattr(context, 'name', project)}:{service_type}" + if isinstance(current_port, int) and current_port > 0: + release(current_port, owner=owner) + plan.assigned = old_port + plan.final = old_port + plan.source = "fresh_start_replacement" + self.runtime._emit( + "port.rebound.after_fresh_start_cleanup", + project=project, + service=service_type, + previous_port=old_port, + port=old_port, + ) def replace_existing_project_services_for_fresh_start( diff --git a/python/envctl_engine/startup/run_reuse_identity.py b/python/envctl_engine/startup/run_reuse_identity.py index bd8e66f43..e0245561a 100644 --- a/python/envctl_engine/startup/run_reuse_identity.py +++ b/python/envctl_engine/startup/run_reuse_identity.py @@ -65,7 +65,9 @@ def _startup_identity_mismatch(previous: Mapping[str, object], current: Mapping[ def _startup_identity_comparison_payload(identity: Mapping[str, object]) -> dict[str, object]: - return {str(key): value for key, value in identity.items() if str(key) != "fingerprint"} + payload = {str(key): value for key, value in identity.items() if str(key) != "fingerprint"} + payload.setdefault("application_runtime", "process") + return payload def project_identities_from_contexts(contexts: list[object]) -> list[ProjectIdentity]: @@ -166,6 +168,12 @@ def _startup_identity_payload( } payload = { "mode": runtime_mode, + "application_runtime": ( + "docker" + if bool(route is not None and route.flags.get("docker")) + or str(getattr(runtime, "env", {}).get("DOCKER_MODE", "")).strip().lower() == "true" + else "process" + ), "projects": identities_to_payload(project_identities_from_contexts(project_contexts)), "startup_enabled": startup_enabled, "services": services, diff --git a/python/envctl_engine/startup/run_reuse_resume.py b/python/envctl_engine/startup/run_reuse_resume.py index bed5d1840..331da5562 100644 --- a/python/envctl_engine/startup/run_reuse_resume.py +++ b/python/envctl_engine/startup/run_reuse_resume.py @@ -5,6 +5,7 @@ from typing import Any, cast from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.docker_service_runtime import state_uses_docker_services from envctl_engine.runtime.engine_runtime_startup_support import mark_run_reused from envctl_engine.runtime.runtime_context import resolve_state_repository from envctl_engine.state.runtime_map import build_runtime_map @@ -122,6 +123,9 @@ def _resume_route(self) -> Route: } if self._attach_plan_agent_after_resume(): resume_flags["batch"] = True + if state_uses_docker_services(self.candidate_state): + resume_flags["docker"] = True + self.runtime.env["DOCKER_MODE"] = "true" return Route( command="resume", mode=self.runtime_mode, diff --git a/python/envctl_engine/startup/service_attach_execution.py b/python/envctl_engine/startup/service_attach_execution.py index 1421c745b..686e44664 100644 --- a/python/envctl_engine/startup/service_attach_execution.py +++ b/python/envctl_engine/startup/service_attach_execution.py @@ -1,13 +1,19 @@ from __future__ import annotations from collections.abc import Callable, Mapping -from dataclasses import dataclass +from dataclasses import dataclass, field import os from pathlib import Path import time from typing import Any, cast from envctl_engine.runtime.service_manager import ServiceStartDescriptor +from envctl_engine.runtime.docker_service_runtime import ( + DockerServiceLaunch, + DockerServiceRuntime, + docker_service_mode_enabled, + docker_service_uses_image_command, +) from envctl_engine.startup.service_execution_policy import ordered_service_layers from envctl_engine.startup.service_execution_records import PreparedServiceLaunch from envctl_engine.state.models import PortPlan, ServiceRecord @@ -34,6 +40,9 @@ class ServiceAttachRunner: additional_services: tuple[Any, ...] backend_listener_expected: bool rebound_delta: int + docker_mode: bool = False + refresh_cache: bool = False + _container_launches: dict[str, DockerServiceLaunch] = field(default_factory=dict, init=False) def start( self, @@ -42,20 +51,27 @@ def start( on_service_retry: Callable[[str, int, int, int, str | None], None], ) -> dict[str, ServiceRecord]: descriptors = self._service_descriptors() - generic_attach = getattr(self.runtime.services, "start_services_with_attach", None) - if callable(generic_attach): - return self._start_with_generic_attach( - descriptors=descriptors, - attach_parallel=attach_parallel, - on_service_retry=on_service_retry, - generic_attach=cast(Callable[..., dict[str, ServiceRecord]], generic_attach), - ) - if self.selected_service_types <= {"backend", "frontend"}: - return self._start_core_services_with_legacy_attach( - attach_parallel=attach_parallel, - on_service_retry=on_service_retry, - ) - raise RuntimeError("Service manager does not support additional app services") + try: + generic_attach = getattr(self.runtime.services, "start_services_with_attach", None) + if callable(generic_attach): + records = self._start_with_generic_attach( + descriptors=descriptors, + attach_parallel=attach_parallel, + on_service_retry=on_service_retry, + generic_attach=cast(Callable[..., dict[str, ServiceRecord]], generic_attach), + ) + elif self.selected_service_types <= {"backend", "frontend"}: + records = self._start_core_services_with_legacy_attach( + attach_parallel=attach_parallel, + on_service_retry=on_service_retry, + ) + else: + raise RuntimeError("Service manager does not support additional app services") + except Exception: + self._cleanup_container_launches() + raise + self._annotate_container_records(records) + return records def reserve_next(self, port: int) -> int: return self.port_allocator.reserve_next(port, owner=f"{self.project_name}:services") @@ -200,13 +216,41 @@ def _launch_prepared_process( pid_fallback: int, ) -> tuple[bool, str | None, int | None]: launch_started = time.monotonic() - process = self._start_process( - command, - cwd=str(launch.cwd), - env=self.command_env_builder(port=launch_port, extra=env_extra), - log_path=launch.log_path, - ) - self._emit_attach_phase(service_name, "process_launch", launch_started) + service_env = self.command_env_builder(port=launch_port, extra=env_extra) + if self.docker_mode or docker_service_mode_enabled(self.runtime): + try: + container_launch = DockerServiceRuntime( + self.runtime, + self.process_runtime, + refresh_cache=self.refresh_cache, + ).launch( + project_name=self.project_name, + project_root=self.project_root, + service_name=service_name, + cwd=launch.cwd, + command=command, + env=service_env, + host_port=event_port, + container_port=launch_port, + listener_expected=launch.listener_expected, + log_path=launch.log_path, + ) + except RuntimeError as exc: + self._emit_attach_phase(service_name, "container_launch", launch_started) + return False, str(exc), None + self._container_launches[service_name] = container_launch + process_pid = None + phase = "container_launch" + else: + process = self._start_process( + command, + cwd=str(launch.cwd), + env=service_env, + log_path=launch.log_path, + ) + process_pid = getattr(process, "pid", pid_fallback) + phase = "process_launch" + self._emit_attach_phase(service_name, phase, launch_started) self.runtime._emit( "service.start", project=self.project_name, @@ -214,7 +258,52 @@ def _launch_prepared_process( port=event_port, retry=False, ) - return True, None, getattr(process, "pid", pid_fallback) + return True, None, process_pid + + def _container_actual_port( + self, + service_name: str, + requested: int, + *, + listener_expected: bool = True, + ) -> int | None: + launch = self._container_launches.get(service_name) + if launch is None: + return None + ready = DockerServiceRuntime(self.runtime, self.process_runtime).wait_until_ready( + launch, + port=requested, + listener_expected=listener_expected, + ) + if ready: + return requested if listener_expected else None + detail = f"container {launch.container_name} exited or did not become ready" + self.runtime._emit( + "service.failure", + project=self.project_name, + service=service_name, + failure_class="container_not_ready", + requested_port=requested if listener_expected else None, + detail=detail, + ) + raise RuntimeError(detail) + + def _annotate_container_records(self, records: dict[str, ServiceRecord]) -> None: + for record in records.values(): + service_name = str(record.service_slug or record.type) + launch = self._container_launches.get(service_name) + if launch is None: + continue + record.runtime_kind = "docker" + record.container_id = launch.container_id + record.container_name = launch.container_name + record.container_image = launch.image + + def _cleanup_container_launches(self) -> None: + docker_runtime = DockerServiceRuntime(self.runtime, self.process_runtime) + for launch in tuple(self._container_launches.values()): + docker_runtime.stop(launch.container_id) + self._container_launches.clear() def start_backend(self, port: int) -> tuple[bool, str | None, int | None]: conflict_result = self._conflicting_start_result("backend", port) @@ -273,16 +362,23 @@ def start_additional_service(self, service_name: str, port: int) -> tuple[bool, if service is None: return False, f"unknown additional service {service_name}", None command_resolve_started = time.monotonic() - command = self.runtime._split_command( - service.start_cmd, - port=port, - replacements={ - "project_root": str(self.project_root), - "service_dir": str(launch.cwd), - "service_name": service_name, - }, - cwd=launch.cwd, - ) + if docker_service_uses_image_command( + self.runtime, + service_name, + docker_mode=self.docker_mode or docker_service_mode_enabled(self.runtime), + ): + command = [] + else: + command = self.runtime._split_command( + service.start_cmd, + port=port, + replacements={ + "project_root": str(self.project_root), + "service_dir": str(launch.cwd), + "service_name": service_name, + }, + cwd=launch.cwd, + ) self._emit_attach_phase(service_name, "command_resolution", command_resolve_started) return self._launch_prepared_process( service_name=service_name, @@ -295,6 +391,12 @@ def start_additional_service(self, service_name: str, port: int) -> tuple[bool, ) def detect_backend_actual(self, pid: int | None, requested: int) -> int | None: + if "backend" in self._container_launches: + return self._container_actual_port( + "backend", + requested, + listener_expected=self.backend_listener_expected, + ) if not self.backend_listener_expected: self.runtime._emit( "service.bind.skipped", @@ -334,6 +436,9 @@ def detect_backend_actual(self, pid: int | None, requested: int) -> int | None: return actual def detect_frontend_actual(self, pid: int | None, requested: int) -> int | None: + container_actual = self._container_actual_port("frontend", requested) + if "frontend" in self._container_launches: + return container_actual self.runtime._emit( "service.bind.requested", project=self.project_name, @@ -374,6 +479,12 @@ def detect_frontend_actual(self, pid: int | None, requested: int) -> int | None: def detect_additional_actual(self, service_name: str, pid: int | None, requested: int) -> int | None: launch = self.prepared_launches[service_name] + if service_name in self._container_launches: + return self._container_actual_port( + service_name, + requested, + listener_expected=launch.listener_expected, + ) if not launch.listener_expected: self.runtime._emit( "service.bind.skipped", diff --git a/python/envctl_engine/startup/service_project_execution.py b/python/envctl_engine/startup/service_project_execution.py index 5602ed65c..3a2c04d30 100644 --- a/python/envctl_engine/startup/service_project_execution.py +++ b/python/envctl_engine/startup/service_project_execution.py @@ -6,6 +6,7 @@ from typing import cast from envctl_engine.runtime.command_resolution import CommandExists, suggest_service_start_command +from envctl_engine.runtime.docker_service_runtime import docker_service_mode_enabled from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.runtime_context import resolve_process_runtime from envctl_engine.startup.startup_selection_support import _restart_service_types_for_project @@ -302,6 +303,7 @@ def project_env_for_service(service_name: str) -> dict[str, str]: return {} service_started = time.monotonic() + use_docker_services = bool(route is not None and route.flags.get("docker")) or docker_service_mode_enabled(rt) prepare_backend_duration_ms = 0.0 prepare_frontend_duration_ms = 0.0 prepare_backend = "backend" in selected_service_types @@ -320,28 +322,34 @@ def project_env_for_service(service_name: str) -> dict[str, str]: def prepare_backend_runtime() -> float: started = time.monotonic() - rt._prepare_backend_runtime( - context=context, - backend_cwd=backend_cwd, - backend_log_path=backend_log_path, - project_env_base=project_env_internal, - route=route, - backend_env_file=backend_env_file, - backend_env_is_default=backend_env_is_default, - ) + if use_docker_services: + rt._emit("service.prepare.skipped", project=context.name, service="backend", reason="docker_image") + else: + rt._prepare_backend_runtime( + context=context, + backend_cwd=backend_cwd, + backend_log_path=backend_log_path, + project_env_base=project_env_internal, + route=route, + backend_env_file=backend_env_file, + backend_env_is_default=backend_env_is_default, + ) return round((time.monotonic() - started) * 1000.0, 2) def prepare_frontend_runtime() -> float: started = time.monotonic() - rt._prepare_frontend_runtime( - context=context, - frontend_cwd=frontend_cwd, - frontend_log_path=frontend_log_path, - project_env_base=frontend_project_env_base, - frontend_env_file=frontend_env_file, - backend_port=backend_plan.final, - route=route, - ) + if use_docker_services: + rt._emit("service.prepare.skipped", project=context.name, service="frontend", reason="docker_image") + else: + rt._prepare_frontend_runtime( + context=context, + frontend_cwd=frontend_cwd, + frontend_log_path=frontend_log_path, + project_env_base=frontend_project_env_base, + frontend_env_file=frontend_env_file, + backend_port=backend_plan.final, + route=route, + ) return round((time.monotonic() - started) * 1000.0, 2) if prepare_backend and prepare_frontend and prep_parallel: @@ -501,6 +509,8 @@ def prepare_frontend_runtime() -> float: additional_services=tuple(additional_services), backend_listener_expected=backend_listener_expected, rebound_delta=rebound_delta, + docker_mode=use_docker_services, + refresh_cache=bool(route is not None and route.flags.get("refresh_cache")), ).start(attach_parallel=attach_parallel, on_service_retry=on_service_retry) attach_duration_ms = round((time.monotonic() - attach_started) * 1000.0, 2) diff --git a/python/envctl_engine/state/__init__.py b/python/envctl_engine/state/__init__.py index 37b34ba27..33cc97981 100644 --- a/python/envctl_engine/state/__init__.py +++ b/python/envctl_engine/state/__init__.py @@ -43,6 +43,10 @@ def load_state(path: str, *, allowed_root: str | None = None) -> RunState: failure_detail=str(svc.get("failure_detail") or "") or None, critical=parse_bool(svc.get("critical"), True), degraded=parse_bool(svc.get("degraded"), False), + runtime_kind=str(svc.get("runtime_kind") or "process"), + container_id=str(svc.get("container_id") or "") or None, + container_name=str(svc.get("container_name") or "") or None, + container_image=str(svc.get("container_image") or "") or None, ) for name, svc in data.get("services", {}).items() } @@ -182,6 +186,10 @@ def state_to_dict(state: RunState) -> dict[str, object]: "failure_detail": svc.failure_detail, "critical": svc.critical, "degraded": svc.degraded, + "runtime_kind": svc.runtime_kind, + "container_id": svc.container_id, + "container_name": svc.container_name, + "container_image": svc.container_image, } for name, svc in state.services.items() }, diff --git a/python/envctl_engine/state/action_command_support.py b/python/envctl_engine/state/action_command_support.py index c5e0708d4..690f28570 100644 --- a/python/envctl_engine/state/action_command_support.py +++ b/python/envctl_engine/state/action_command_support.py @@ -2,10 +2,17 @@ from dataclasses import dataclass, field from typing import Any, cast +from contextlib import contextmanager import json +import signal +import threading from envctl_engine.runtime.browser_diagnostics import build_runtime_diagnostics from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.docker_service_runtime import ( + refresh_container_log_snapshots, + stop_container_log_followers, +) from envctl_engine.shared.parsing import parse_float_or_none, parse_int from envctl_engine.state.action_log_support import DEFAULT_LOG_ISSUE_LIMIT from envctl_engine.state.models import RunState @@ -25,6 +32,26 @@ class StateActionReconciliation: reconciled: bool = False +@contextmanager +def _interrupt_log_follow_on_termination(enabled: bool): + if not enabled or threading.current_thread() is not threading.main_thread(): + yield + return + previous: dict[signal.Signals, Any] = {} + + def interrupt(_signum: int, _frame: object) -> None: + raise KeyboardInterrupt + + for signal_name in (signal.SIGHUP, signal.SIGTERM): + previous[signal_name] = signal.getsignal(signal_name) + signal.signal(signal_name, interrupt) + try: + yield + finally: + for signal_name, handler in previous.items(): + signal.signal(signal_name, handler) + + class StateActionCommandRunner: def __init__(self, orchestrator: Any) -> None: self.orchestrator = orchestrator @@ -200,6 +227,12 @@ def _run_logs(self, route: Route) -> int: if duration_seconds is not None and duration_seconds < 0: duration_seconds = 0.0 no_color = bool(route.flags.get("logs_no_color")) or self.orchestrator._human_output_no_color(route) + follower_pids = refresh_container_log_snapshots( + self.runtime.raw_runtime, + self.state, + tail=max(tail, 0), + follow=follow and not bool(route.flags.get("json")), + ) if bool(route.flags.get("json")): print( json.dumps( @@ -215,13 +248,17 @@ def _run_logs(self, route: Route) -> int: ) ) return self._finish_action(0) - self.runtime.print_logs( - self.state, - tail=max(tail, 0), - follow=follow, - duration_seconds=duration_seconds, - no_color=no_color, - ) + try: + with _interrupt_log_follow_on_termination(bool(follower_pids)): + self.runtime.print_logs( + self.state, + tail=max(tail, 0), + follow=follow, + duration_seconds=duration_seconds, + no_color=no_color, + ) + finally: + stop_container_log_followers(self.runtime.raw_runtime, follower_pids) return self._finish_action(0) def _run_clear_logs(self, route: Route) -> int: @@ -450,6 +487,12 @@ def _run_errors(self, route: Route) -> int: parse_int(str(route.flags.get("logs_tail", DEFAULT_LOG_ISSUE_LIMIT)), DEFAULT_LOG_ISSUE_LIMIT), 0, ) + refresh_container_log_snapshots( + self.runtime.raw_runtime, + self.state, + tail=log_issue_limit, + follow=False, + ) log_issues = self.orchestrator._service_log_issues(self.state, max_matches=log_issue_limit) has_errors = bool(failed or reconciliation.requirement_issues or reconciliation.recent_failures or log_issues) if bool(route.flags.get("json")): diff --git a/python/envctl_engine/state/action_health_support.py b/python/envctl_engine/state/action_health_support.py index d61667e3f..fae8e9f05 100644 --- a/python/envctl_engine/state/action_health_support.py +++ b/python/envctl_engine/state/action_health_support.py @@ -205,6 +205,10 @@ def _service_row(service: ServiceRecord) -> dict[str, object]: "failure_detail": getattr(service, "failure_detail", None), "critical": getattr(service, "critical", True), "degraded": getattr(service, "degraded", False), + "runtime_kind": getattr(service, "runtime_kind", "process"), + "container_id": getattr(service, "container_id", None), + "container_name": getattr(service, "container_name", None), + "container_image": getattr(service, "container_image", None), } rows = self._parallel_service_map( diff --git a/python/envctl_engine/state/action_log_support.py b/python/envctl_engine/state/action_log_support.py index 851e6646e..b73a67e63 100644 --- a/python/envctl_engine/state/action_log_support.py +++ b/python/envctl_engine/state/action_log_support.py @@ -219,6 +219,8 @@ def clear_one(service: ServiceRecord) -> tuple[int, int, int, int, str | None]: try: with log_path.open("w", encoding="utf-8"): pass + if str(getattr(service, "runtime_kind", "process") or "process").lower() == "docker": + Path(f"{log_path}.docker-since").touch() line = f"{service.name}: log cleared at {log_path}" return 1, 0, 0, 0, ( None diff --git a/python/envctl_engine/state/action_orchestrator.py b/python/envctl_engine/state/action_orchestrator.py index a0989e80c..1c65e2753 100644 --- a/python/envctl_engine/state/action_orchestrator.py +++ b/python/envctl_engine/state/action_orchestrator.py @@ -216,24 +216,31 @@ def _resolve_selected_services(self, route: Route, state: RunState) -> set[str] services_flag = route.flags.get("services") project_filters = {name.lower() for name in route.projects} project_filters.update(self.runtime.selectors_from_passthrough(route.passthrough_args)) - selected: set[str] = set() + service_selected: set[str] | None = None + project_selected: set[str] | None = None if isinstance(services_flag, list): + service_selected = set() for raw in services_flag: target = str(raw).strip() if not target: continue for name, service in state.services.items(): if service_matches_selector(service, target): - selected.add(name) + service_selected.add(name) if project_filters: + project_selected = set() for name in state.services: project = self.runtime.project_name_from_service(name).lower() if project and project in project_filters: - selected.add(name) - - if selected: - return selected + project_selected.add(name) + + if service_selected is not None and project_selected is not None: + return service_selected.intersection(project_selected) + if service_selected is not None: + return service_selected + if project_selected is not None: + return project_selected if project_filters or (isinstance(services_flag, list) and services_flag): return set() return None diff --git a/python/envctl_engine/state/models.py b/python/envctl_engine/state/models.py index cfcab4405..7bd715f94 100644 --- a/python/envctl_engine/state/models.py +++ b/python/envctl_engine/state/models.py @@ -37,6 +37,10 @@ class ServiceRecord: failure_detail: str | None = None critical: bool = True degraded: bool = False + runtime_kind: str = "process" + container_id: str | None = None + container_name: str | None = None + container_image: str | None = None @dataclass(slots=True) diff --git a/python/envctl_engine/state/runtime_map.py b/python/envctl_engine/state/runtime_map.py index a3eac6dad..218974297 100644 --- a/python/envctl_engine/state/runtime_map.py +++ b/python/envctl_engine/state/runtime_map.py @@ -86,6 +86,10 @@ def build_runtime_map_without_projection(state: RunState, *, host: str = "localh "failure_detail": getattr(service, "failure_detail", None), "critical": getattr(service, "critical", True), "degraded": getattr(service, "degraded", False), + "runtime_kind": getattr(service, "runtime_kind", "process"), + "container_id": getattr(service, "container_id", None), + "container_name": getattr(service, "container_name", None), + "container_image": getattr(service, "container_image", None), } if service.type == "backend": project_entry["backend_port"] = port diff --git a/tests/python/runtime/test_docker_service_runtime.py b/tests/python/runtime/test_docker_service_runtime.py new file mode 100644 index 000000000..0e01dea3d --- /dev/null +++ b/tests/python/runtime/test_docker_service_runtime.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +from envctl_engine.runtime.docker_service_runtime import ( + DockerServiceRuntime, + docker_service_container_name, + refresh_container_log_snapshots, + stop_container_log_followers, +) +from envctl_engine.runtime.lifecycle_service_termination import terminate_service_record +from envctl_engine.runtime.service_status_truth import service_truth_status +from envctl_engine.state.models import ServiceRecord + + +class _Runner: + def __init__(self) -> None: + self.calls: list[list[str]] = [] + self.env_file_payload = "" + self.running = True + self.port_ready = True + self.terminated: list[int] = [] + self.build_env: dict[str, str] = {} + self.existing_container: tuple[bool, str] | None = None + + def run(self, command, **_kwargs): + argv = [str(part) for part in command] + self.calls.append(argv) + if argv[:2] == ["docker", "build"]: + self.build_env = dict(_kwargs.get("env", {})) + if argv[:2] == ["docker", "run"]: + env_file = Path(argv[argv.index("--env-file") + 1]) + self.env_file_payload = env_file.read_text(encoding="utf-8") + return SimpleNamespace(returncode=0, stdout="container-id\n", stderr="") + if argv[:3] == ["docker", "container", "inspect"]: + if self.existing_container is None: + return SimpleNamespace(returncode=1, stdout="", stderr="No such container") + running, owner = self.existing_container + return SimpleNamespace(returncode=0, stdout=f"{str(running).lower()}|{owner}\n", stderr="") + if argv[:3] == ["docker", "inspect", "--format"]: + return SimpleNamespace(returncode=0 if self.running else 1, stdout="true\n" if self.running else "", stderr="") + if argv[:3] == ["docker", "image", "inspect"]: + return SimpleNamespace(returncode=1, stdout="", stderr="missing") + if argv[:3] == ["docker", "rm", "--force"]: + return SimpleNamespace(returncode=0, stdout="", stderr="") + if argv[:2] == ["docker", "logs"]: + return SimpleNamespace(returncode=0, stdout="container log\n", stderr="") + return SimpleNamespace(returncode=0, stdout="ok\n", stderr="") + + def start_background(self, command, **_kwargs): + self.calls.append([str(part) for part in command]) + return SimpleNamespace(pid=4321) + + def wait_for_port(self, _port, **_kwargs): + return self.port_ready + + def terminate_process_group(self, pid, **_kwargs): + self.terminated.append(pid) + return True + + +def _runtime(tmp_path: Path, runner: _Runner, *, env: dict[str, str] | None = None): + events: list[tuple[str, dict[str, object]]] = [] + runtime = SimpleNamespace( + env={"DOCKER_MODE": "true", **(env or {})}, + config=SimpleNamespace(raw={}, runtime_scope_dir=tmp_path / "runtime-scope"), + process_runner=runner, + _command_exists=lambda command: command == "docker", + _run_dir_path=lambda run_id: tmp_path / "runs" / str(run_id), + _emit=lambda event, **payload: events.append((event, payload)), + _service_truth_timeout=lambda: 0.01, + ) + return runtime, events + + +def test_container_names_preserve_root_identity_for_long_project_names(tmp_path: Path) -> None: + first = docker_service_container_name( + project_name="feature-" + ("very-long-name-" * 8), + project_root=tmp_path / "one", + service_name="backend", + ) + second = docker_service_container_name( + project_name="feature-" + ("very-long-name-" * 8), + project_root=tmp_path / "two", + service_name="backend", + ) + + assert len(first) <= 63 + assert len(second) <= 63 + assert first != second + + +def test_builds_with_docker_layer_cache_and_runs_managed_container(tmp_path: Path) -> None: + runner = _Runner() + runtime, events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_COMMAND_MODE": "service"}) + service_root = tmp_path / "backend" + service_root.mkdir() + (service_root / "Dockerfile").write_text("FROM python:3.12-slim\nWORKDIR /app\n", encoding="utf-8") + + launch = DockerServiceRuntime(runtime, runner).launch( + project_name="Feature One", + project_root=tmp_path, + service_name="backend", + cwd=service_root, + command=["python", "-m", "uvicorn", "app:api", "--host", "127.0.0.1", "--port", "8123"], + env={ + "PATH": "/host/bin", + "DATABASE_URL": "postgres://u:p@127.0.0.1:5432/db", + "PORT": "8123", + "VITE_API_URL": "http://localhost:8123/api/v1", + }, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + + build_index = next(index for index, call in enumerate(runner.calls) if call[:2] == ["docker", "build"]) + run_index = next(index for index, call in enumerate(runner.calls) if call[:2] == ["docker", "run"]) + assert build_index < run_index + docker_build = runner.calls[build_index] + assert docker_build[docker_build.index("--build-arg") + 1] == "VITE_API_URL" + assert "http://localhost:8123/api/v1" not in docker_build + assert runner.build_env["VITE_API_URL"] == "http://localhost:8123/api/v1" + docker_run = runner.calls[run_index] + publish_index = docker_run.index("--publish") + assert docker_run[publish_index : publish_index + 2] == ["--publish", "127.0.0.1:8123:8123"] + assert "0.0.0.0" in docker_run + runtime_label_index = docker_run.index("io.envctl.service=backend") + 2 + assert docker_run[runtime_label_index].startswith("io.envctl.runtime-scope=") + assert "PATH=" not in runner.env_file_payload + assert "DATABASE_URL=postgres://u:p@host.docker.internal:5432/db" in runner.env_file_payload + assert launch.container_id == "container-id" + assert any(event == "service.container.build" and payload["cache"] == "docker_layer_cache" for event, payload in events) + + +def test_running_untracked_container_is_not_replaced_or_removed(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + docker_runtime = DockerServiceRuntime(runtime, runner) + runner.existing_container = (True, docker_runtime._runtime_scope_identity()) + + try: + docker_runtime.launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except RuntimeError as exc: + assert "already running" in str(exc) + else: + raise AssertionError("expected the untracked running container to block startup") + + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + assert not any(call[:2] == ["docker", "run"] for call in runner.calls) + + +def test_container_owned_by_another_runtime_scope_is_not_replaced(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + runner.existing_container = (False, "another-runtime") + + try: + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except RuntimeError as exc: + assert "another envctl runtime scope" in str(exc) + else: + raise AssertionError("expected the foreign runtime scope to block startup") + + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + + +def test_stopped_container_from_same_runtime_scope_is_removed(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + docker_runtime = DockerServiceRuntime(runtime, runner) + runner.existing_container = (False, docker_runtime._runtime_scope_identity()) + + docker_runtime.launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + + inspect_index = next(index for index, call in enumerate(runner.calls) if call[:3] == ["docker", "container", "inspect"]) + remove_index = next(index for index, call in enumerate(runner.calls) if call[:3] == ["docker", "rm", "--force"]) + run_index = next(index for index, call in enumerate(runner.calls) if call[:2] == ["docker", "run"]) + assert inspect_index < remove_index < run_index + + +def test_configured_image_runs_without_build_and_preserves_browser_public_url(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime( + tmp_path, + runner, + env={ + "ENVCTL_FRONTEND_DOCKER_IMAGE": "example/frontend:dev", + "ENVCTL_FRONTEND_DOCKER_COMMAND": "npm run dev -- --host 0.0.0.0 --port 9000", + "ENVCTL_FRONTEND_DOCKER_PORT": "9000", + }, + ) + + launch = DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="frontend", + cwd=tmp_path, + command=["host-only-command"], + env={ + "VITE_API_URL": "http://localhost:8000/api/v1", + "REDIS_URL": "redis://localhost:6379", + "PORT": "9100", + "HOST": "127.0.0.1", + }, + host_port=9100, + container_port=9100, + listener_expected=True, + log_path=str(tmp_path / "frontend.log"), + ) + + assert not any(call[:2] == ["docker", "build"] for call in runner.calls) + docker_run = next(call for call in runner.calls if call[:2] == ["docker", "run"]) + assert "example/frontend:dev" in docker_run + assert "127.0.0.1:9100:9000" in docker_run + assert docker_run[-8:] == ["npm", "run", "dev", "--", "--host", "0.0.0.0", "--port", "9000"] + assert "VITE_API_URL=http://localhost:8000/api/v1" in runner.env_file_payload + assert "REDIS_URL=redis://host.docker.internal:6379" in runner.env_file_payload + assert "PORT=9000" in runner.env_file_payload + assert "HOST=0.0.0.0" in runner.env_file_payload + assert launch.image == "example/frontend:dev" + + +def test_container_state_drives_health_and_stop(tmp_path: Path) -> None: + runner = _Runner() + runtime, events = _runtime(tmp_path, runner) + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd=str(tmp_path), + pid=4321, + requested_port=8123, + actual_port=8123, + status="running", + runtime_kind="docker", + container_id="container-id", + container_name="envctl-app-main-backend", + container_image="example/backend:dev", + ) + + assert service_truth_status(runtime, service) == "running" + runner.port_ready = False + assert service_truth_status(runtime, service) == "unreachable" + assert terminate_service_record(runtime, service, aggressive=False, verify_ownership=True) + assert ["docker", "rm", "--force", "container-id"] in runner.calls + assert runner.terminated == [4321] + assert any(event == "service.container.stop" for event, _payload in events) + + +def test_container_logs_are_snapshotted_and_followed_only_on_demand(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner) + log_path = tmp_path / "backend.log" + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd=str(tmp_path), + log_path=str(log_path), + runtime_kind="docker", + container_id="container-id", + ) + state = SimpleNamespace(services={service.name: service}) + + assert refresh_container_log_snapshots(runtime, state, tail=20, follow=False) == [] + assert log_path.read_text(encoding="utf-8") == "container log\n" + assert not any(call[:3] == ["docker", "logs", "--follow"] for call in runner.calls) + + followers = refresh_container_log_snapshots(runtime, state, tail=20, follow=True) + assert followers == [4321] + assert any(call[:3] == ["docker", "logs", "--follow"] for call in runner.calls) + stop_container_log_followers(runtime, followers) + assert runner.terminated == [4321] diff --git a/tests/python/runtime/test_engine_runtime_artifacts.py b/tests/python/runtime/test_engine_runtime_artifacts.py index cf3973fee..879bf3771 100644 --- a/tests/python/runtime/test_engine_runtime_artifacts.py +++ b/tests/python/runtime/test_engine_runtime_artifacts.py @@ -21,7 +21,7 @@ ) from envctl_engine.runtime.engine_runtime_event_support import persist_events_snapshot # noqa: E402 from envctl_engine.runtime.runtime_readiness import RuntimeReadinessResult # noqa: E402 -from envctl_engine.state.models import PortPlan, RunState # noqa: E402 +from envctl_engine.state.models import PortPlan, RunState, ServiceRecord # noqa: E402 class EngineRuntimeArtifactsTests(unittest.TestCase): @@ -59,6 +59,39 @@ def test_print_summary_includes_project_ports(self) -> None: self.assertIn("envctl Python engine run summary", rendered) self.assertIn("- Main: backend=8000 frontend=9000 db=5432 redis=6379 n8n=5678", rendered) + def test_print_summary_prefers_preserved_service_truth_over_unused_planned_port(self) -> None: + context = SimpleNamespace( + name="Main", + ports={ + name: PortPlan(project="Main", requested=port, assigned=port, final=port, source="fixed") + for name, port in { + "backend": 8001, + "frontend": 9001, + "db": 5432, + "redis": 6379, + "n8n": 5678, + }.items() + }, + ) + state = RunState( + run_id="run-restart", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", type="backend", cwd="/repo/backend", actual_port=8001 + ), + "Main Frontend": ServiceRecord( + name="Main Frontend", type="frontend", cwd="/repo/frontend", actual_port=9000 + ), + }, + ) + + buffer = io.StringIO() + with redirect_stdout(buffer): + print_summary(SimpleNamespace(), state, [context]) + + self.assertIn("- Main: backend=8001 frontend=9000", buffer.getvalue()) + def test_write_artifacts_forwards_expected_state_repository_payload(self) -> None: captured: dict[str, object] = {} diff --git a/tests/python/runtime/test_engine_runtime_commands.py b/tests/python/runtime/test_engine_runtime_commands.py index 9b1af759b..6730825d6 100644 --- a/tests/python/runtime/test_engine_runtime_commands.py +++ b/tests/python/runtime/test_engine_runtime_commands.py @@ -14,11 +14,63 @@ command_env, command_override_value, default_python_executable, + service_start_command_resolved, split_command, ) class EngineRuntimeCommandsTests(unittest.TestCase): + def test_docker_image_command_does_not_require_container_executable_on_host(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + (root / "frontend").mkdir() + runtime = SimpleNamespace( + env={ + "DOCKER_MODE": "true", + "ENVCTL_FRONTEND_START_CMD": "container-only-server --port {port}", + }, + config=SimpleNamespace( + base_dir=root, + raw={"FRONTEND_DIR": "frontend"}, + ), + _command_exists=lambda _executable: False, + ) + + command, source = service_start_command_resolved( + runtime, + service_name="frontend", + project_root=root, + port=8010, + ) + + self.assertEqual(command, ["container-only-server", "--port", "8010"]) + self.assertEqual(source, "configured") + + def test_docker_service_command_still_requires_executable_on_host(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + (root / "frontend").mkdir() + runtime = SimpleNamespace( + env={ + "DOCKER_MODE": "true", + "ENVCTL_FRONTEND_DOCKER_COMMAND_MODE": "service", + "ENVCTL_FRONTEND_START_CMD": "missing-command --port {port}", + }, + config=SimpleNamespace( + base_dir=root, + raw={"FRONTEND_DIR": "frontend"}, + ), + _command_exists=lambda _executable: False, + ) + + with self.assertRaises(RuntimeError): + service_start_command_resolved( + runtime, + service_name="frontend", + project_root=root, + port=8010, + ) + def test_command_override_value_prefers_runtime_env(self) -> None: runtime = SimpleNamespace( env={"DB_HOST": "env-db"}, diff --git a/tests/python/runtime/test_engine_runtime_startup_support.py b/tests/python/runtime/test_engine_runtime_startup_support.py index ca8a73908..b86206be0 100644 --- a/tests/python/runtime/test_engine_runtime_startup_support.py +++ b/tests/python/runtime/test_engine_runtime_startup_support.py @@ -319,6 +319,41 @@ def test_run_reuse_rejects_startup_fingerprint_mismatch(self) -> None: self.assertEqual(decision.decision_kind, "fresh_run") self.assertEqual(decision.reason, "startup_fingerprint_mismatch") + def test_explicit_no_resume_retains_candidate_for_safe_replacement(self) -> None: + context = ProjectContext(name="Main", root=Path("/repo"), ports={}) + route = Route( + command="start", + mode="main", + raw_args=["start", "--no-resume"], + passthrough_args=[], + projects=[], + flags={"no_resume": True}, + ) + state = RunState( + run_id="run-existing", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + ) + }, + metadata={"project_roots": {"Main": "/repo"}}, + ) + runtime = self._reuse_runtime(state=state) + + decision = startup_support.evaluate_run_reuse( + runtime, + runtime_mode="main", + route=route, + contexts=[context], + ) + + self.assertEqual(decision.decision_kind, "fresh_run") + self.assertEqual(decision.reason, "explicit_no_resume") + self.assertIs(decision.candidate_state, state) + def test_run_reuse_rejects_dependency_external_mode_mismatch(self) -> None: context = ProjectContext(name="Main", root=Path("/repo"), ports={}) route = Route(command="start", mode="main", raw_args=[], passthrough_args=[], projects=[], flags={}) diff --git a/tests/python/runtime/test_lifecycle_dependency_stop.py b/tests/python/runtime/test_lifecycle_dependency_stop.py index 59d176d1e..9f0ab968d 100644 --- a/tests/python/runtime/test_lifecycle_dependency_stop.py +++ b/tests/python/runtime/test_lifecycle_dependency_stop.py @@ -1,6 +1,7 @@ from __future__ import annotations import unittest +from types import SimpleNamespace from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.lifecycle_dependency_stop import ( @@ -8,11 +9,52 @@ release_selected_dependency_components, requirements_have_enabled_components, select_dependency_components_for_stop, + stop_requirement_component_containers, ) from envctl_engine.state.models import RequirementsResult, RunState class LifecycleDependencyStopTests(unittest.TestCase): + def test_entire_system_selects_all_enabled_dependencies_for_requested_project(self) -> None: + state = RunState( + run_id="run-entire", + mode="trees", + requirements={ + "Alpha": RequirementsResult(project="Alpha", redis={"enabled": True}, n8n={"enabled": True}), + "Beta": RequirementsResult(project="Beta", redis={"enabled": True}), + }, + ) + route = Route(command="stop", mode="trees", projects=["Alpha"], flags={"runtime_scope": "entire-system"}) + + self.assertEqual(select_dependency_components_for_stop(state, route), {"Alpha": {"redis", "n8n"}}) + + def test_stop_requirement_component_removes_direct_and_compose_containers(self) -> None: + calls: list[list[str]] = [] + + def run(command, **_kwargs): # noqa: ANN001 + argv = [str(part) for part in command] + calls.append(argv) + if argv[:3] == ["docker", "ps", "--all"]: + return SimpleNamespace(returncode=0, stdout="one\ntwo\n", stderr="") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + runtime = SimpleNamespace(process_runner=SimpleNamespace(run=run)) + stop_requirement_component_containers( + runtime, + {"id": "redis", "enabled": True, "container_name": "envctl-redis-alpha"}, + ) + stop_requirement_component_containers( + runtime, + {"id": "supabase", "enabled": True, "container_name": "envctl-supabase-alpha"}, + ) + + self.assertIn(["docker", "rm", "--force", "envctl-redis-alpha"], calls) + self.assertIn( + ["docker", "ps", "--all", "--quiet", "--filter", "label=com.docker.compose.project=envctl-supabase-alpha"], + calls, + ) + self.assertIn(["docker", "rm", "--force", "one", "two"], calls) + def test_select_dependency_components_filters_unknown_projects_dependencies_and_disabled_components(self) -> None: state = RunState( run_id="run-1", diff --git a/tests/python/runtime/test_lifecycle_worktree_cleanup.py b/tests/python/runtime/test_lifecycle_worktree_cleanup.py index 79773b828..450579fe3 100644 --- a/tests/python/runtime/test_lifecycle_worktree_cleanup.py +++ b/tests/python/runtime/test_lifecycle_worktree_cleanup.py @@ -4,6 +4,7 @@ from pathlib import Path from types import SimpleNamespace +from envctl_engine.runtime.docker_service_runtime import docker_service_container_name from envctl_engine.runtime.lifecycle_worktree_containers import legacy_container_name, remove_tree_containers from envctl_engine.runtime.lifecycle_worktree_cleanup import WorktreeCleanupDependencies, blast_worktree_before_delete from envctl_engine.runtime.lifecycle_worktree_metadata import cleanup_artifact_paths, prune_project_metadata @@ -12,6 +13,42 @@ class LifecycleWorktreeCleanupTests(unittest.TestCase): + def test_remove_tree_containers_removes_managed_application_container(self) -> None: + calls: list[list[str]] = [] + project_root = Path(".").resolve() + container_name = docker_service_container_name( + project_name="Feature", + project_root=project_root, + service_name="backend", + ) + + class Runner: + def run(self, command: list[str], **kwargs: object) -> SimpleNamespace: + calls.append(command) + if command[1:4] == ["ps", "-a", "--format"]: + return SimpleNamespace(returncode=0, stdout=f"app-id|{container_name}\n", stderr="") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + runtime = SimpleNamespace( + process_runner=Runner(), + env={}, + config=SimpleNamespace(additional_services=()), + _emit=lambda event, **payload: None, + ) + warnings: list[str] = [] + + remove_tree_containers( + runtime, + project_name="Feature", + project_root=project_root, + include_supabase=False, + remove_named_volumes=False, + warnings=warnings, + ) + + self.assertEqual(warnings, []) + self.assertIn(["docker", "rm", "-f", "-v", "app-id"], calls) + def test_blast_worktree_before_delete_updates_matching_project_state(self) -> None: events: list[tuple[str, dict[str, object]]] = [] saved_states: list[RunState] = [] diff --git a/tests/python/runtime/test_prereq_policy.py b/tests/python/runtime/test_prereq_policy.py index eeab8e36a..e9bbab344 100644 --- a/tests/python/runtime/test_prereq_policy.py +++ b/tests/python/runtime/test_prereq_policy.py @@ -13,6 +13,37 @@ class PrereqPolicyTests(unittest.TestCase): + def test_docker_application_mode_requires_docker_without_dependencies(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + repo = Path(tmpdir) / "repo" + runtime = Path(tmpdir) / "runtime" + (repo / ".git").mkdir(parents=True, exist_ok=True) + config = load_config( + { + "RUN_REPO_ROOT": str(repo), + "RUN_SH_RUNTIME_DIR": str(runtime), + "POSTGRES_MAIN_ENABLE": "false", + "REDIS_ENABLE": "false", + "SUPABASE_MAIN_ENABLE": "false", + "N8N_ENABLE": "false", + } + ) + route = parse_route(["start", "--main", "--docker"], env={}) + + def fake_which(binary: str) -> str | None: + if binary == "docker": + return None + return f"/usr/bin/{binary}" + + with ( + patch("envctl_engine.runtime.cli.shutil.which", side_effect=fake_which), + patch("envctl_engine.runtime.cli._python_dependency_available", return_value=True), + ): + ok, reason = cli.check_prereqs(route, config) + + self.assertFalse(ok) + self.assertIn("docker", str(reason).lower()) + def test_main_start_without_requirements_does_not_require_docker(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo = Path(tmpdir) / "repo" diff --git a/tests/python/startup/test_restart_prestop_support.py b/tests/python/startup/test_restart_prestop_support.py index 49bc46997..aca2119bb 100644 --- a/tests/python/startup/test_restart_prestop_support.py +++ b/tests/python/startup/test_restart_prestop_support.py @@ -218,6 +218,27 @@ def test_restart_prestop_state_rejects_mode_mismatch_and_builds_fallback_route(s ], ) + def test_restart_prestop_state_inherits_docker_runtime_from_saved_state(self) -> None: + route = Route(command="restart", mode="main", raw_args=["restart"], flags={}) + state = SimpleNamespace( + mode="main", + run_id="run-docker", + metadata={"application_runtime": "docker"}, + services={}, + ) + runtime = SimpleNamespace( + env={}, + _effective_start_mode=lambda route: "main", + _try_load_existing_state=lambda *, mode: state, + _emit=lambda event, **payload: None, + ) + + result = restart_prestop_state(route=route, runtime=runtime) + + self.assertIs(result.state, state) + self.assertTrue(route.flags["docker"]) + self.assertEqual(runtime.env["DOCKER_MODE"], "true") + def test_restart_prestop_preservation_splits_services_and_requirements(self) -> None: state = SimpleNamespace( services={ diff --git a/tests/python/startup/test_run_reuse_identity.py b/tests/python/startup/test_run_reuse_identity.py index 15cd8531d..56e9998d1 100644 --- a/tests/python/startup/test_run_reuse_identity.py +++ b/tests/python/startup/test_run_reuse_identity.py @@ -45,6 +45,7 @@ def test_build_startup_identity_metadata_records_roots_and_fingerprint_payload(s self.assertEqual(metadata["project_roots"], {"Main": normalize_project_root("/tmp/repo")}) identity = metadata["startup_identity"] self.assertEqual(identity["mode"], "main") + self.assertEqual(identity["application_runtime"], "process") self.assertEqual(identity["projects"], [{"name": "Main", "root": normalize_project_root("/tmp/repo")}]) self.assertEqual(identity["services"], {"backend": True, "frontend": False}) self.assertIn("fingerprint", identity) @@ -94,6 +95,21 @@ def test_startup_identity_mismatch_ignores_matching_fingerprints_and_reports_cha self.assertEqual(mismatch, {"fields": ["mode"]}) + self.assertEqual( + _startup_identity_mismatch( + {"mode": "main", "services": {"backend": True}}, + {"mode": "main", "services": {"backend": True}, "application_runtime": "process"}, + ), + {}, + ) + self.assertEqual( + _startup_identity_mismatch( + {"mode": "main", "application_runtime": "process"}, + {"mode": "main", "application_runtime": "docker"}, + ), + {"fields": ["application_runtime"]}, + ) + def test_root_mismatches_compare_only_strong_roots(self) -> None: selected = [ProjectIdentity("Main", "/repo"), ProjectIdentity("Weak", None)] state = [ProjectIdentity("Main", "/other"), ProjectIdentity("Weak", "/weak")] diff --git a/tests/python/startup/test_run_reuse_support.py b/tests/python/startup/test_run_reuse_support.py index 566c0b45e..4f80f1735 100644 --- a/tests/python/startup/test_run_reuse_support.py +++ b/tests/python/startup/test_run_reuse_support.py @@ -194,6 +194,31 @@ def test_fresh_start_replacement_services_selects_configured_service_types_for_t {"Main Backend"}, ) + def test_unfiltered_explicit_no_resume_replaces_tracked_removed_additional_services(self) -> None: + route = Route(command="start", mode="main", raw_args=[], flags={"no_resume": True}) + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace(name="Main Backend", project="Main", type="backend"), + "Main Old Worker": SimpleNamespace( + name="Main Old Worker", + project="Main", + type="old-worker", + service_slug="old-worker", + ), + } + ) + + selected = fresh_start_replacement_services( + route=route, + selected_contexts=[SimpleNamespace(name="Main")], + candidate_state=state, + configured_service_types={"backend"}, + additional_services=(), + project_name_from_service=lambda _name: "Main", + ) + + self.assertEqual(selected, {"Main Backend", "Main Old Worker"}) + def test_replace_existing_project_services_for_fresh_start_terminates_selected_services_and_orphans(self) -> None: route = Route(command="start", mode="trees", raw_args=["start"], flags={}) session = SimpleNamespace(effective_route=route, runtime_mode="trees") @@ -242,6 +267,73 @@ def test_replace_existing_project_services_for_fresh_start_terminates_selected_s self.assertEqual(terminated, [(candidate_state, {"Main Backend", "Main Frontend"}, False, True)]) self.assertEqual(orphaned, [(candidate_state, {"Main Backend", "Main Frontend"}, True)]) + def test_explicit_no_resume_terminates_tracked_services_before_fresh_start(self) -> None: + route = Route(command="start", mode="main", raw_args=["start", "--no-resume"], flags={"no_resume": True}) + session = SimpleNamespace(effective_route=route, runtime_mode="main") + candidate_state = SimpleNamespace(run_id="run-existing") + progress: list[str] = [] + terminated: list[set[str]] = [] + + replace_existing_project_services_for_fresh_start( + runtime=SimpleNamespace( + _emit=lambda *args, **kwargs: None, + _terminate_services_from_state=lambda _state, *, selected_services, **_kwargs: terminated.append( + set(selected_services) + ), + ), + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + fresh_start_replacement_services=lambda _session, *, candidate_state: {"Main Backend"}, + announce_session_identifiers=lambda _session: None, + report_progress=lambda _route, message: progress.append(message), + terminate_restart_orphan_listeners=lambda **_kwargs: None, + ) + + self.assertEqual(terminated, [{"Main Backend"}]) + self.assertEqual(progress, ["Fresh start requested; replacing 1 existing service(s)..."]) + + def test_explicit_no_resume_rebinds_reserved_plan_to_released_service_port(self) -> None: + route = Route(command="start", mode="main", raw_args=["start", "--no-resume"], flags={"no_resume": True}) + backend_plan = SimpleNamespace(assigned=8001, final=8001, source="retry") + context = SimpleNamespace(name="Main", ports={"backend": backend_plan}) + session = SimpleNamespace(effective_route=route, runtime_mode="main", selected_contexts=[context]) + service = SimpleNamespace( + name="Main Backend", + project="Main", + type="backend", + service_slug="backend", + listener_expected=True, + actual_port=8000, + requested_port=8000, + ) + candidate_state = SimpleNamespace(run_id="run-existing", services={"Main Backend": service}) + calls: list[tuple[str, int, str]] = [] + planner = SimpleNamespace( + release=lambda port, *, owner: calls.append(("release", port, owner)), + ) + runtime = SimpleNamespace( + port_planner=planner, + _emit=lambda *args, **kwargs: None, + _project_name_from_service=lambda _name: "Main", + _terminate_services_from_state=lambda *args, **kwargs: None, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + fresh_start_replacement_services=lambda _session, *, candidate_state: {"Main Backend"}, + announce_session_identifiers=lambda _session: None, + report_progress=lambda _route, _message: None, + terminate_restart_orphan_listeners=lambda **_kwargs: None, + ) + + self.assertEqual(calls, [("release", 8001, "Main:backend")]) + self.assertEqual((backend_plan.assigned, backend_plan.final), (8000, 8000)) + self.assertEqual(backend_plan.source, "fresh_start_replacement") + def test_replace_existing_project_services_for_fresh_start_skips_non_mismatch_or_dependencies_scope(self) -> None: runtime = SimpleNamespace(_emit=lambda *args, **kwargs: self.fail("replace should not emit")) candidate_state = SimpleNamespace(run_id="run-existing") diff --git a/tests/python/startup/test_service_attach_execution.py b/tests/python/startup/test_service_attach_execution.py index 0c61b31f4..c6001af36 100644 --- a/tests/python/startup/test_service_attach_execution.py +++ b/tests/python/startup/test_service_attach_execution.py @@ -5,7 +5,10 @@ from types import SimpleNamespace from typing import cast import unittest +from unittest.mock import patch +from envctl_engine.runtime.docker_service_runtime import DockerServiceLaunch +from envctl_engine.runtime.service_manager import ServiceManager from envctl_engine.runtime.service_manager import ServiceStartDescriptor from envctl_engine.startup.service_attach_execution import ServiceAttachRunner from envctl_engine.startup.service_execution_records import PreparedServiceLaunch @@ -17,6 +20,76 @@ def port_plan(port: int) -> PortPlan: class ServiceAttachExecutionTests(unittest.TestCase): + def test_explicit_docker_mode_launches_and_annotates_container_service(self) -> None: + project_root = Path("/tmp/envctl-project") + runtime = SimpleNamespace( + env={}, + config=SimpleNamespace(raw={}), + services=ServiceManager(), + _conflict_remaining={}, + _emit=lambda event, **payload: None, + _service_start_command_resolved=lambda service_name, project_root, port: ( + ["python", "app.py", "--port", str(port)], + "configured", + ), + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="Main", + project_root=project_root, + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=project_root / "backend", + frontend_cwd=project_root / "frontend", + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={"PORT": "8000"}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: {**extra, "PORT": str(port)}, + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=project_root / "backend", + log_path="/logs/backend.txt", + requested_port=8000, + env={"PORT": "8000"}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + docker_mode=True, + ) + launch = DockerServiceLaunch( + container_id="container-id", + container_name="envctl-app-main-backend", + image="example/backend:dev", + log_path="/logs/backend.txt", + ) + + with ( + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.launch", + return_value=launch, + ) as docker_launch, + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.wait_until_ready", + return_value=True, + ), + ): + records = runner.start(attach_parallel=False, on_service_retry=lambda *args: None) + + record = records["Main Backend"] + self.assertEqual(record.runtime_kind, "docker") + self.assertEqual(record.container_id, "container-id") + self.assertEqual(record.container_name, "envctl-app-main-backend") + self.assertEqual(record.container_image, "example/backend:dev") + docker_launch.assert_called_once() + def test_runner_builds_layered_descriptors_and_preserves_additional_urls(self) -> None: project_root = Path("/tmp/envctl-project") events: list[tuple[str, dict[str, object]]] = [] diff --git a/tests/python/state/test_state_action_orchestrator_logs.py b/tests/python/state/test_state_action_orchestrator_logs.py index fb6056b21..1da01021b 100644 --- a/tests/python/state/test_state_action_orchestrator_logs.py +++ b/tests/python/state/test_state_action_orchestrator_logs.py @@ -3,6 +3,7 @@ from contextlib import redirect_stdout from io import StringIO import json +import signal import tempfile import unittest from types import SimpleNamespace @@ -15,6 +16,7 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_runtime_support import normalize_log_line from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord +from envctl_engine.state.action_command_support import _interrupt_log_follow_on_termination from envctl_engine.state.action_orchestrator import StateActionOrchestrator from envctl_engine.test_output.parser_base import strip_ansi from envctl_engine.ui.target_selector import TargetSelection @@ -75,6 +77,17 @@ def _emit(self, event: str, **payload: object) -> None: class StateActionOrchestratorLogsTests(unittest.TestCase): + def test_log_follow_termination_signal_interrupts_for_finally_cleanup(self) -> None: + previous = signal.getsignal(signal.SIGTERM) + + with self.assertRaises(KeyboardInterrupt): + with _interrupt_log_follow_on_termination(True): + handler = signal.getsignal(signal.SIGTERM) + self.assertTrue(callable(handler)) + handler(signal.SIGTERM, None) # type: ignore[operator] + + self.assertIs(signal.getsignal(signal.SIGTERM), previous) + def test_runtime_facade_routes_state_dependencies(self) -> None: state = RunState(run_id="run-0", mode="main") runtime = _RuntimeStub(state) @@ -146,6 +159,39 @@ def test_logs_service_filter_accepts_additional_service_slug_and_display_forms(s assert runtime.seen_logs_state is not None self.assertEqual(set(runtime.seen_logs_state.services), {"Main Voice Runtime"}) + def test_logs_combined_project_and_service_filters_intersect(self) -> None: + state = RunState( + run_id="run-combined-filter", + mode="trees", + services={ + "Alpha Backend": ServiceRecord( + name="Alpha Backend", type="backend", cwd="/tmp/alpha", project="Alpha" + ), + "Alpha Frontend": ServiceRecord( + name="Alpha Frontend", type="frontend", cwd="/tmp/alpha", project="Alpha" + ), + "Beta Backend": ServiceRecord( + name="Beta Backend", type="backend", cwd="/tmp/beta", project="Beta" + ), + }, + ) + runtime = _RuntimeStub(state) + orchestrator = StateActionOrchestrator(runtime) + route = Route( + command="logs", + mode="trees", + projects=["Alpha"], + flags={"services": ["backend"], "json": True}, + ) + + output = StringIO() + with redirect_stdout(output): + code = orchestrator.execute(route) + + payload = json.loads(output.getvalue()) + self.assertEqual(code, 0) + self.assertEqual([service["name"] for service in payload["services"]], ["Alpha Backend"]) + def test_health_prints_enabled_dependency_statuses(self) -> None: state = RunState( run_id="run-1", diff --git a/tests/python/state/test_state_roundtrip.py b/tests/python/state/test_state_roundtrip.py index 781aad674..c7d87d14a 100644 --- a/tests/python/state/test_state_roundtrip.py +++ b/tests/python/state/test_state_roundtrip.py @@ -28,6 +28,10 @@ def test_json_state_roundtrip_preserves_services(self) -> None: listener_pids=[12345, 12346], status="running", started_at=1234.5, + runtime_kind="docker", + container_id="sha256:abc", + container_name="envctl-app-tree-alpha-backend", + container_image="envctl/example-backend:dev", ) }, metadata={"source": "python"}, @@ -40,6 +44,16 @@ def test_json_state_roundtrip_preserves_services(self) -> None: self.assertEqual(loaded.services["Tree Alpha Backend"].actual_port, 8001) self.assertEqual(loaded.services["Tree Alpha Backend"].listener_pids, [12345, 12346]) self.assertEqual(loaded.services["Tree Alpha Backend"].started_at, 1234.5) + self.assertEqual(loaded.services["Tree Alpha Backend"].runtime_kind, "docker") + self.assertEqual(loaded.services["Tree Alpha Backend"].container_id, "sha256:abc") + self.assertEqual( + loaded.services["Tree Alpha Backend"].container_name, + "envctl-app-tree-alpha-backend", + ) + self.assertEqual( + loaded.services["Tree Alpha Backend"].container_image, + "envctl/example-backend:dev", + ) self.assertEqual(loaded.metadata["source"], "python") def test_legacy_shell_state_is_loaded_without_source_execution(self) -> None: From 876dda8701c92dd103502931d02806b845297c87 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:45:28 +0300 Subject: [PATCH 02/26] Prefer dev for pull request bases --- docs/reference/commands.md | 4 +++ .../actions/action_git_state_support.py | 26 +++++++++++++++++++ .../actions/project_action_domain.py | 7 ++++- .../ui/dashboard/pr_selection_support.py | 4 +-- .../actions/test_action_git_state_support.py | 22 ++++++++++++++++ tests/python/actions/test_actions_cli_pr.py | 22 ++++++++-------- tests/python/actions/test_actions_cli_ship.py | 2 +- 7 files changed, 72 insertions(+), 15 deletions(-) diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 1ee514fb2..11d5ffb78 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -285,6 +285,10 @@ High-value command families: - `plan` - `import` - `test` + +When `envctl pr` or `envctl ship` opens a pull request without an explicit +`ENVCTL_PR_BASE`, it selects the first branch that exists in this order: +`dev`, `main`, then `master`. Both remote branches and local refs are checked. - `logs` - `health` - `errors` diff --git a/python/envctl_engine/actions/action_git_state_support.py b/python/envctl_engine/actions/action_git_state_support.py index d99a73ab5..2e189b67a 100644 --- a/python/envctl_engine/actions/action_git_state_support.py +++ b/python/envctl_engine/actions/action_git_state_support.py @@ -80,6 +80,32 @@ def detect_default_branch(git_root: Path, *, git_output: Callable[[Path, list[st return "main" +def detect_pr_base_branch(git_root: Path, *, git_output: Callable[[Path, list[str]], str]) -> str: + remote_heads = git_output( + git_root, + [ + "ls-remote", + "--heads", + "origin", + "refs/heads/dev", + "refs/heads/main", + "refs/heads/master", + ], + ) + remote_branches = { + line.rsplit("refs/heads/", 1)[1].strip() + for line in remote_heads.splitlines() + if "refs/heads/" in line + } + for candidate in ("dev", "main", "master"): + if candidate in remote_branches: + return candidate + for ref in (f"origin/{candidate}", candidate): + if git_output(git_root, ["rev-parse", "--verify", ref]).strip(): + return candidate + return "master" + + def existing_pr_url( git_root: Path, branch: str, diff --git a/python/envctl_engine/actions/project_action_domain.py b/python/envctl_engine/actions/project_action_domain.py index 316f75a1f..813047190 100644 --- a/python/envctl_engine/actions/project_action_domain.py +++ b/python/envctl_engine/actions/project_action_domain.py @@ -70,6 +70,7 @@ "_unmerged_stage_entries", "_write_pr_body_file", "detect_default_branch", + "detect_pr_base_branch", "existing_pr_url", "probe_dirty_worktree", "resolve_git_root", @@ -296,6 +297,10 @@ def detect_default_branch(git_root: Path) -> str: return git_state_support.detect_default_branch(git_root, git_output=_git_output) +def detect_pr_base_branch(git_root: Path) -> str: + return git_state_support.detect_pr_base_branch(git_root, git_output=_git_output) + + def existing_pr_url(git_root: Path, branch: str) -> str: return git_state_support.existing_pr_url( git_root, @@ -335,7 +340,7 @@ def _resolve_pr_base_branch(context: ActionProjectContext, git_root: Path) -> st explicit = str(context.env.get("ENVCTL_PR_BASE", "")).strip() if explicit: return explicit - return detect_default_branch(git_root) + return detect_pr_base_branch(git_root) _resolve_analyze_mode = review_iteration_support.resolve_analyze_mode diff --git a/python/envctl_engine/ui/dashboard/pr_selection_support.py b/python/envctl_engine/ui/dashboard/pr_selection_support.py index cc6fa4c45..bfbc06fdd 100644 --- a/python/envctl_engine/ui/dashboard/pr_selection_support.py +++ b/python/envctl_engine/ui/dashboard/pr_selection_support.py @@ -4,7 +4,7 @@ import subprocess from typing import Any, Callable, cast -from envctl_engine.actions.project_action_domain import detect_default_branch +from envctl_engine.actions.project_action_domain import detect_pr_base_branch from envctl_engine.runtime.command_router import Route from envctl_engine.state.models import RunState from envctl_engine.ui.dashboard.pr_flow import run_pr_flow @@ -83,7 +83,7 @@ def default_pr_base_branch( owner: Any, runtime: Any, *, - detect_default_branch_fn: Callable[[Path], str] = detect_default_branch, + detect_default_branch_fn: Callable[[Path], str] = detect_pr_base_branch, pr_git_root_fn: Callable[[Any, Any], Path] = pr_git_root, ) -> str: git_root = pr_git_root_fn(owner, runtime) diff --git a/tests/python/actions/test_action_git_state_support.py b/tests/python/actions/test_action_git_state_support.py index 30acf4c4b..be6a7c674 100644 --- a/tests/python/actions/test_action_git_state_support.py +++ b/tests/python/actions/test_action_git_state_support.py @@ -42,6 +42,28 @@ def local_main(_git_root: Path, args: list[str]) -> str: self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=local_main), "main") + def test_detect_pr_base_branch_prefers_remote_or_local_dev_then_main_then_master(self) -> None: + def resolve(existing_refs: set[str], remote_heads: set[str] | None = None) -> str: + def fake_git_output(_git_root: Path, args: list[str]) -> str: + if args[:2] == ["ls-remote", "--heads"]: + return "".join( + f"abc123\trefs/heads/{branch}\n" for branch in sorted(remote_heads or set()) + ) + self.assertEqual(args[:2], ["rev-parse", "--verify"]) + return "abc123\n" if args[2] in existing_refs else "" + + return git_state.detect_pr_base_branch(Path("/repo"), git_output=fake_git_output) + + self.assertEqual(resolve({"origin/dev", "origin/main", "origin/master"}), "dev") + self.assertEqual(resolve({"dev", "origin/main"}), "dev") + self.assertEqual(resolve({"origin/main", "origin/master"}), "main") + self.assertEqual(resolve({"main", "origin/master"}), "main") + self.assertEqual(resolve({"origin/master"}), "master") + self.assertEqual(resolve({"master"}), "master") + self.assertEqual(resolve(set()), "master") + self.assertEqual(resolve({"origin/main"}, {"dev", "main"}), "dev") + self.assertEqual(resolve({"dev"}, {"main"}), "dev") + def test_existing_pr_url_uses_gh_pr_list_for_open_branch(self) -> None: calls: list[tuple[list[str], str]] = [] diff --git a/tests/python/actions/test_actions_cli_pr.py b/tests/python/actions/test_actions_cli_pr.py index e92c9dd72..0a85ace1f 100644 --- a/tests/python/actions/test_actions_cli_pr.py +++ b/tests/python/actions/test_actions_cli_pr.py @@ -31,7 +31,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch( "envctl_engine.actions.project_action_domain.existing_pr_url", @@ -93,7 +93,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.probe_dirty_worktree") as dirty_probe, @@ -144,7 +144,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.probe_dirty_worktree") as dirty_probe, @@ -221,7 +221,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -300,7 +300,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -367,7 +367,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -429,7 +429,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch.dict(os.environ, {"ENVCTL_PR_BODY": "Typed PR body\n\nWith details."}, clear=False), - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -493,7 +493,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 patch.dict(os.environ, {"ENVCTL_ACTION_INTERACTIVE": "1"}, clear=False), patch("sys.stdin.isatty", return_value=True), patch("builtins.input", side_effect=AssertionError("input() should not be called for pr action")), - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -551,7 +551,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -600,7 +600,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -657,7 +657,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), diff --git a/tests/python/actions/test_actions_cli_ship.py b/tests/python/actions/test_actions_cli_ship.py index 569e6da2a..5fc02c739 100644 --- a/tests/python/actions/test_actions_cli_ship.py +++ b/tests/python/actions/test_actions_cli_ship.py @@ -236,7 +236,7 @@ def fake_run(git_root: Path, args: list[str]): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.shutil.which", return_value="/usr/bin/git"), - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain._run_git", side_effect=fake_run), patch("envctl_engine.actions.project_action_domain.run_commit_action", return_value=0) as commit_action, From 9ba77115dbe43380b4fa1e02ba8fdb0f1d2298a0 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 01:02:25 +0300 Subject: [PATCH 03/26] fix: serialize startup and guard bulk actions --- .../action_command_execution_support.py | 17 ++++++ .../actions/project_action_domain.py | 7 +-- python/envctl_engine/debug/debug_utils.py | 5 -- .../runtime/help_topic_actions.py | 13 +++-- python/envctl_engine/startup/lifecycle.py | 20 +++++++ .../ui/dashboard/pr_selection_support.py | 18 +++++- python/envctl_engine/ui/selection_support.py | 4 +- .../test_action_command_execution_support.py | 54 +++++++++++++++++ tests/python/actions/test_actions_cli_pr.py | 22 +++---- tests/python/actions/test_actions_cli_ship.py | 2 +- .../actions/test_actions_migrate_parity.py | 3 +- .../debug/test_debug_utils_file_lock.py | 14 +++++ .../startup/test_startup_lifecycle_lock.py | 58 +++++++++++++++++++ tests/python/ui/test_selection_support.py | 8 +++ 14 files changed, 212 insertions(+), 33 deletions(-) create mode 100644 tests/python/debug/test_debug_utils_file_lock.py create mode 100644 tests/python/startup/test_startup_lifecycle_lock.py diff --git a/python/envctl_engine/actions/action_command_execution_support.py b/python/envctl_engine/actions/action_command_execution_support.py index f18734d8f..51e394167 100644 --- a/python/envctl_engine/actions/action_command_execution_support.py +++ b/python/envctl_engine/actions/action_command_execution_support.py @@ -18,6 +18,8 @@ "migrate": "run_migrate_action", } +_BULK_MUTATING_COMMANDS = frozenset({"commit", "migrate", "pr", "ship"}) + class ActionSpinner(Protocol): def start(self) -> None: ... @@ -59,6 +61,12 @@ def execute(self) -> int: self._emit_finish(1, error=error) return 1 + bulk_confirmation_error = self._bulk_confirmation_error(targets) + if bulk_confirmation_error is not None: + print(bulk_confirmation_error) + self._emit_finish(1, error=bulk_confirmation_error) + return 1 + handler = self._handler() if handler is None: return self.runtime.unsupported_command(self.route.command) @@ -70,6 +78,15 @@ def execute(self) -> int: self._emit_finish(code) return code + def _bulk_confirmation_error(self, targets: list[object]) -> str | None: + if self.route.command not in _BULK_MUTATING_COMMANDS: + return None + if not bool(self.route.flags.get("all")) or len(targets) <= 1: + return None + if bool(self.route.flags.get("yes")) or bool(self.route.flags.get("force")): + return None + return f"{self.route.command} --all targets {len(targets)} projects and requires --yes or --force" + def _execute_special_worktree_command(self) -> int | None: if self.route.command == "self-destruct-worktree": code = self.orchestrator.run_self_destruct_worktree_action(self.route) diff --git a/python/envctl_engine/actions/project_action_domain.py b/python/envctl_engine/actions/project_action_domain.py index 813047190..aea45f876 100644 --- a/python/envctl_engine/actions/project_action_domain.py +++ b/python/envctl_engine/actions/project_action_domain.py @@ -70,7 +70,6 @@ "_unmerged_stage_entries", "_write_pr_body_file", "detect_default_branch", - "detect_pr_base_branch", "existing_pr_url", "probe_dirty_worktree", "resolve_git_root", @@ -297,10 +296,6 @@ def detect_default_branch(git_root: Path) -> str: return git_state_support.detect_default_branch(git_root, git_output=_git_output) -def detect_pr_base_branch(git_root: Path) -> str: - return git_state_support.detect_pr_base_branch(git_root, git_output=_git_output) - - def existing_pr_url(git_root: Path, branch: str) -> str: return git_state_support.existing_pr_url( git_root, @@ -340,7 +335,7 @@ def _resolve_pr_base_branch(context: ActionProjectContext, git_root: Path) -> st explicit = str(context.env.get("ENVCTL_PR_BASE", "")).strip() if explicit: return explicit - return detect_pr_base_branch(git_root) + return git_state_support.detect_pr_base_branch(git_root, git_output=_git_output) _resolve_analyze_mode = review_iteration_support.resolve_analyze_mode diff --git a/python/envctl_engine/debug/debug_utils.py b/python/envctl_engine/debug/debug_utils.py index 83ddcbbd2..7511cee73 100644 --- a/python/envctl_engine/debug/debug_utils.py +++ b/python/envctl_engine/debug/debug_utils.py @@ -84,11 +84,6 @@ def file_lock(path: Path, *, timeout: float) -> Iterator[None]: except OSError: pass handle.close() - try: - if path.exists() and path.stat().st_size == 0: - path.unlink() - except OSError: - pass def debug_env_value(env: dict[str, str], key: str) -> str | None: diff --git a/python/envctl_engine/runtime/help_topic_actions.py b/python/envctl_engine/runtime/help_topic_actions.py index 84b358a6c..2d4182ee1 100644 --- a/python/envctl_engine/runtime/help_topic_actions.py +++ b/python/envctl_engine/runtime/help_topic_actions.py @@ -65,7 +65,7 @@ "pr": CommandHelpTopic( command="pr", summary="create a pull request for the selected branch/worktree", - usage=("envctl pr [--project ] [--pr-base ]",), + usage=("envctl pr [--project |--all --yes] [--pr-base ]",), what_it_does=( "resolves the current branch, checks for an existing PR, and creates one with gh or repo helper scripts", "commits and pushes dirty worktree changes first using envctl commit behavior when needed", @@ -73,6 +73,7 @@ ), flags=( "--project create PR for selected worktree/project", + "--all --yes explicitly approve creating/updating PRs for every discovered target", "--pr-base set the target base branch", "--interactive opt into prompts/selection instead of default headless action mode", ), @@ -83,7 +84,7 @@ "commit": CommandHelpTopic( command="commit", summary="commit-only fallback that preserves envctl-local control artifacts", - usage=("envctl commit [--project |--main] [-m |--commit-message-file ]",), + usage=("envctl commit [--project |--main|--all --yes] [-m |--commit-message-file ]",), what_it_does=( "use envctl ship for normal AI handoff; reserve commit for commit-only maintenance or fallback flows", "stages normal changed paths and intentionally skips protected envctl-local artifacts", @@ -96,6 +97,7 @@ "--commit-message-file read commit message from a file", "--project commit inside selected worktree/project", "--main target the main checkout", + "--all --yes explicitly approve committing every discovered target", ), examples=( "envctl commit --main -m 'Ship focused fix'", @@ -107,7 +109,7 @@ "ship": CommandHelpTopic( command="ship", summary="commit, push, create/update PR, and report GitHub checks for the current or selected target", - usage=("envctl ship [--project ] [-m ] [--human]",), + usage=("envctl ship [--project |--all --yes] [-m ] [--human]",), what_it_does=( "when run inside a generated worktree, infers that worktree without requiring --project", "owns the normal AI handoff flow instead of requiring separate commit, push, or PR commands", @@ -123,6 +125,7 @@ ), flags=( "--project ship one worktree/project", + "--all --yes explicitly approve shipping every discovered target", "-m use explicit commit message text for the commit phase", "--json compatibility no-op; JSON is the default", "--human print compact terminal output instead of JSON", @@ -155,7 +158,7 @@ "migrate": CommandHelpTopic( command="migrate", summary="run backend database migrations for selected targets", - usage=("envctl migrate [--project |--main|--all]",), + usage=("envctl migrate [--project |--main|--all --yes]",), what_it_does=( "runs the configured/default backend migration command, usually Alembic upgrade head", "loads backend env files and reuses saved dependency URLs when a current run exists", @@ -164,7 +167,7 @@ flags=( "--project migrate one worktree/project", "--main migrate the main checkout", - "--all migrate all selected/discovered targets where supported", + "--all --yes explicitly approve migrating all discovered targets", ), examples=("envctl migrate --project feature-a-1", "envctl migrate --main"), aliases=("migration", "migrations", "m", "--migrate"), diff --git a/python/envctl_engine/startup/lifecycle.py b/python/envctl_engine/startup/lifecycle.py index a82529b67..2eba8eaf0 100644 --- a/python/envctl_engine/startup/lifecycle.py +++ b/python/envctl_engine/startup/lifecycle.py @@ -1,8 +1,10 @@ from __future__ import annotations from functools import partial +from pathlib import Path from typing import Any +from envctl_engine.debug.debug_utils import file_lock from envctl_engine.planning.plan_agent.launch import launch_plan_agent_terminals from envctl_engine.planning.plan_agent.omx_transport import validate_plan_agent_attach_target from envctl_engine.planning.plan_agent.tmux_transport import attach_plan_agent_terminal @@ -50,12 +52,30 @@ select_start_tree_projects, trees_start_selection_required, ) +from envctl_engine.shared.parsing import parse_float from envctl_engine.ui.debug_snapshot import emit_startup_plan_handoff_snapshot from envctl_engine.ui.spinner import spinner, use_spinner_policy from envctl_engine.ui.spinner_service import emit_spinner_policy, resolve_spinner_policy def execute_startup_lifecycle(orchestrator: Any, route: Any) -> int: + runtime = orchestrator.runtime + timeout_raw = runtime.env.get("ENVCTL_STARTUP_LOCK_TIMEOUT_SECONDS") or runtime.config.raw.get( + "ENVCTL_STARTUP_LOCK_TIMEOUT_SECONDS" + ) + timeout = max(parse_float(timeout_raw, 3600.0), 1.0) + lock_path = Path(runtime.runtime_root) / "locks" / "startup.lock" + try: + with file_lock(lock_path, timeout=timeout): + return _execute_startup_lifecycle_locked(orchestrator, route) + except TimeoutError as exc: + message = f"Startup could not acquire the repository lifecycle lock: {exc}" + runtime._emit("startup.lock.timeout", lock_path=str(lock_path), timeout_seconds=timeout) + print(message) + return 1 + + +def _execute_startup_lifecycle_locked(orchestrator: Any, route: Any) -> int: runtime = orchestrator.runtime session = create_startup_session(runtime, route) finalize_failure = partial( diff --git a/python/envctl_engine/ui/dashboard/pr_selection_support.py b/python/envctl_engine/ui/dashboard/pr_selection_support.py index bfbc06fdd..85cfad303 100644 --- a/python/envctl_engine/ui/dashboard/pr_selection_support.py +++ b/python/envctl_engine/ui/dashboard/pr_selection_support.py @@ -4,7 +4,7 @@ import subprocess from typing import Any, Callable, cast -from envctl_engine.actions.project_action_domain import detect_pr_base_branch +from envctl_engine.actions.action_git_state_support import detect_pr_base_branch from envctl_engine.runtime.command_router import Route from envctl_engine.state.models import RunState from envctl_engine.ui.dashboard.pr_flow import run_pr_flow @@ -12,6 +12,20 @@ from envctl_engine.ui.selector_model import SelectorItem +def _dashboard_git_output(git_root: Path, args: list[str]) -> str: + result = subprocess.run( + ["git", "-C", str(git_root), *args], + text=True, + capture_output=True, + check=False, + ) + return result.stdout if result.returncode == 0 else "" + + +def _detect_pr_base_branch(git_root: Path) -> str: + return detect_pr_base_branch(git_root, git_output=_dashboard_git_output) + + def apply_pr_selection(owner: Any, route: Route, state: RunState, rt: object) -> Route | None: runtime_any = cast(Any, rt) default_branch = owner._default_pr_base_branch(runtime_any) @@ -83,7 +97,7 @@ def default_pr_base_branch( owner: Any, runtime: Any, *, - detect_default_branch_fn: Callable[[Path], str] = detect_pr_base_branch, + detect_default_branch_fn: Callable[[Path], str] = _detect_pr_base_branch, pr_git_root_fn: Callable[[Any, Any], Path] = pr_git_root, ) -> str: git_root = pr_git_root_fn(owner, runtime) diff --git a/python/envctl_engine/ui/selection_support.py b/python/envctl_engine/ui/selection_support.py index 99ab5f157..b46efe457 100644 --- a/python/envctl_engine/ui/selection_support.py +++ b/python/envctl_engine/ui/selection_support.py @@ -110,9 +110,11 @@ def no_target_selected_message( return f"No {label} target selected." mode = str(route.mode).strip().lower() discovery = "envctl --list-trees --json" if mode == "trees" else "envctl --list-targets --main --json" + all_selector = "--all --yes" if command in {"commit", "migrate", "pr", "ship"} else "--all" return ( f"No {label} target selected. " - f"In headless mode, run '{discovery}' and retry with '--project ', '--service ', or '--all'." + f"In headless mode, run '{discovery}' and retry with '--project ', '--service ', " + f"or '{all_selector}'." ) diff --git a/tests/python/actions/test_action_command_execution_support.py b/tests/python/actions/test_action_command_execution_support.py index b982bfa98..bb8213ad8 100644 --- a/tests/python/actions/test_action_command_execution_support.py +++ b/tests/python/actions/test_action_command_execution_support.py @@ -44,6 +44,7 @@ def __init__(self) -> None: self.statuses: list[str] = [] self.restored = False self.spinner = _Spinner() + self.ship_calls = 0 def resolve_targets(self, _route, *, trees_only: bool) -> tuple[list[object], str | None]: return [SimpleNamespace(name="Main")], None @@ -66,6 +67,10 @@ def restore() -> None: def run_test_action(self, _route, _targets: list[object]) -> int: return 0 + def run_ship_action(self, _route, _targets: list[object]) -> int: + self.ship_calls += 1 + return 0 + class _DeferredOutputOrchestrator(_Orchestrator): def __init__(self) -> None: @@ -136,6 +141,55 @@ def test_execute_action_command_returns_target_resolution_error(self) -> None: orchestrator.runtime.events, ) + def test_execute_action_command_requires_confirmation_for_bulk_mutation(self) -> None: + orchestrator = _Orchestrator() + orchestrator.resolve_targets = lambda _route, trees_only: ( # type: ignore[assignment] + [SimpleNamespace(name="one"), SimpleNamespace(name="two")], + None, + ) + route = SimpleNamespace(command="ship", mode="trees", flags={"all": True}) + + code = execute_action_command( + orchestrator, + route, + spinner_factory=lambda *_args, **_kwargs: _policy_context(orchestrator.spinner), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + use_spinner_policy_fn=_policy_context, + ) + + self.assertEqual(code, 1) + self.assertEqual(orchestrator.ship_calls, 0) + self.assertIn( + { + "event": "action.command.finish", + "command": "ship", + "code": 1, + "error": "ship --all targets 2 projects and requires --yes or --force", + }, + orchestrator.runtime.events, + ) + + def test_execute_action_command_allows_confirmed_bulk_mutation(self) -> None: + orchestrator = _Orchestrator() + orchestrator.resolve_targets = lambda _route, trees_only: ( # type: ignore[assignment] + [SimpleNamespace(name="one"), SimpleNamespace(name="two")], + None, + ) + route = SimpleNamespace(command="ship", mode="trees", flags={"all": True, "yes": True}) + + code = execute_action_command( + orchestrator, + route, + spinner_factory=lambda *_args, **_kwargs: _policy_context(orchestrator.spinner), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + use_spinner_policy_fn=_policy_context, + ) + + self.assertEqual(code, 0) + self.assertEqual(orchestrator.ship_calls, 1) + def test_execute_action_command_flushes_deferred_output_before_finish(self) -> None: orchestrator = _DeferredOutputOrchestrator() route = SimpleNamespace(command="test", mode="main", flags={}) diff --git a/tests/python/actions/test_actions_cli_pr.py b/tests/python/actions/test_actions_cli_pr.py index 0a85ace1f..745843f6f 100644 --- a/tests/python/actions/test_actions_cli_pr.py +++ b/tests/python/actions/test_actions_cli_pr.py @@ -31,7 +31,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch( "envctl_engine.actions.project_action_domain.existing_pr_url", @@ -93,7 +93,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.probe_dirty_worktree") as dirty_probe, @@ -144,7 +144,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.probe_dirty_worktree") as dirty_probe, @@ -221,7 +221,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -300,7 +300,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -367,7 +367,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -429,7 +429,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch.dict(os.environ, {"ENVCTL_PR_BODY": "Typed PR body\n\nWith details."}, clear=False), - patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -493,7 +493,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 patch.dict(os.environ, {"ENVCTL_ACTION_INTERACTIVE": "1"}, clear=False), patch("sys.stdin.isatty", return_value=True), patch("builtins.input", side_effect=AssertionError("input() should not be called for pr action")), - patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -551,7 +551,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -600,7 +600,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -657,7 +657,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), diff --git a/tests/python/actions/test_actions_cli_ship.py b/tests/python/actions/test_actions_cli_ship.py index 5fc02c739..0a8640510 100644 --- a/tests/python/actions/test_actions_cli_ship.py +++ b/tests/python/actions/test_actions_cli_ship.py @@ -236,7 +236,7 @@ def fake_run(git_root: Path, args: list[str]): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.shutil.which", return_value="/usr/bin/git"), - patch("envctl_engine.actions.project_action_domain.detect_pr_base_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain._run_git", side_effect=fake_run), patch("envctl_engine.actions.project_action_domain.run_commit_action", return_value=0) as commit_action, diff --git a/tests/python/actions/test_actions_migrate_parity.py b/tests/python/actions/test_actions_migrate_parity.py index 84bc451e6..b2217f6b3 100644 --- a/tests/python/actions/test_actions_migrate_parity.py +++ b/tests/python/actions/test_actions_migrate_parity.py @@ -546,7 +546,7 @@ def test_migrate_action_uses_distinct_env_contracts_for_each_target_in_multi_tar fake_runner = _FakeRunner(returncode=0) engine.process_runner = fake_runner # type: ignore[assignment] - route = parse_route(["migrate", "--all"], env={"ENVCTL_DEFAULT_MODE": "trees"}) + route = parse_route(["migrate", "--all", "--yes"], env={"ENVCTL_DEFAULT_MODE": "trees"}) code = engine.dispatch(route) self.assertEqual(code, 0) @@ -567,4 +567,3 @@ def test_migrate_action_uses_distinct_env_contracts_for_each_target_in_multi_tar ) self.assertEqual(first_env.get("SQLALCHEMY_DATABASE_URL"), first_env.get("DATABASE_URL")) self.assertEqual(second_env.get("SQLALCHEMY_DATABASE_URL"), second_env.get("DATABASE_URL")) - diff --git a/tests/python/debug/test_debug_utils_file_lock.py b/tests/python/debug/test_debug_utils_file_lock.py new file mode 100644 index 000000000..46cb5a970 --- /dev/null +++ b/tests/python/debug/test_debug_utils_file_lock.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from pathlib import Path + +from envctl_engine.debug.debug_utils import file_lock + + +def test_file_lock_keeps_stable_inode_after_release(tmp_path: Path) -> None: + lock_path = tmp_path / "stable.lock" + + with file_lock(lock_path, timeout=1.0): + inode = lock_path.stat().st_ino + + assert lock_path.stat().st_ino == inode diff --git a/tests/python/startup/test_startup_lifecycle_lock.py b/tests/python/startup/test_startup_lifecycle_lock.py new file mode 100644 index 000000000..a7c39f0f5 --- /dev/null +++ b/tests/python/startup/test_startup_lifecycle_lock.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace + +from envctl_engine.startup import lifecycle + + +def test_execute_startup_lifecycle_holds_repo_scope_lock(monkeypatch, tmp_path: Path) -> None: + seen: list[object] = [] + + @contextmanager + def recording_lock(path: Path, *, timeout: float): + seen.extend((path, timeout, "entered")) + yield + seen.append("exited") + + runtime = SimpleNamespace( + runtime_root=tmp_path, + env={}, + config=SimpleNamespace(raw={}), + ) + orchestrator = SimpleNamespace(runtime=runtime) + monkeypatch.setattr(lifecycle, "file_lock", recording_lock) + monkeypatch.setattr( + lifecycle, + "_execute_startup_lifecycle_locked", + lambda received_orchestrator, route: seen.append((received_orchestrator, route)) or 7, + ) + + assert lifecycle.execute_startup_lifecycle(orchestrator, "route") == 7 + assert seen == [tmp_path / "locks" / "startup.lock", 3600.0, "entered", (orchestrator, "route"), "exited"] + + +def test_execute_startup_lifecycle_reports_lock_timeout(monkeypatch, tmp_path: Path, capsys) -> None: + @contextmanager + def timing_out_lock(path: Path, *, timeout: float): + raise TimeoutError(f"Timed out acquiring lock: {path}") + yield + + events: list[tuple[str, dict[str, object]]] = [] + runtime = SimpleNamespace( + runtime_root=tmp_path, + env={"ENVCTL_STARTUP_LOCK_TIMEOUT_SECONDS": "2.5"}, + config=SimpleNamespace(raw={}), + _emit=lambda event, **payload: events.append((event, payload)), + ) + monkeypatch.setattr(lifecycle, "file_lock", timing_out_lock) + + assert lifecycle.execute_startup_lifecycle(SimpleNamespace(runtime=runtime), "route") == 1 + assert "repository lifecycle lock" in capsys.readouterr().out + assert events == [ + ( + "startup.lock.timeout", + {"lock_path": str(tmp_path / "locks" / "startup.lock"), "timeout_seconds": 2.5}, + ) + ] diff --git a/tests/python/ui/test_selection_support.py b/tests/python/ui/test_selection_support.py index bca70906e..a988856b0 100644 --- a/tests/python/ui/test_selection_support.py +++ b/tests/python/ui/test_selection_support.py @@ -108,6 +108,14 @@ def test_no_target_selected_message_and_explicit_target_detection_match_route_co "--project ", no_target_selected_message("restart", route=_route(mode="trees"), interactive_allowed=False), ) + self.assertIn( + "--all --yes", + no_target_selected_message("ship", route=_route(mode="trees"), interactive_allowed=False), + ) + self.assertNotIn( + "--all --yes", + no_target_selected_message("test", route=_route(mode="trees"), interactive_allowed=False), + ) self.assertTrue(route_has_explicit_target(_route(flags={"all": True}), runtime)) self.assertTrue(route_has_explicit_target(_route(projects=["Main"]), runtime)) From 0c5e8c7b33d7b84c05790a587af896c348310802 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 01:12:58 +0300 Subject: [PATCH 04/26] Consolidate PR preview controller ownership --- docs/reference/commands.md | 16 ++++++++++++++ .../runtime/help_topic_inspection.py | 2 ++ .../runtime/test_pr_preview_controller.py | 21 +++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 1ee514fb2..6f77c8939 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -265,6 +265,22 @@ Behavior: - records GitHub deployments and marks them inactive when previews stop - is intended for trusted CI runners with `gh`, `git`, Docker, and envctl available +Integration ownership: + +- envctl is the canonical owner of preview event parsing, branch-to-PR + matching, worktree lifecycle, runtime cleanup, comments, and deployments; + consuming repositories should invoke the installed command instead of + vendoring a controller copy +- the caller workflow should forward `pull_request_target`, feature-branch + `push`, `schedule`, and `workflow_dispatch` events through + `GITHUB_EVENT_NAME` and `GITHUB_EVENT_PATH` +- feature-branch `push` events are restricted to open same-repository PRs with + the configured preview label; deleted branches, tags, forks, and unmatched + branches are ignored +- install or select the desired envctl revision before invoking the command; + this keeps controller rollout/version policy in CI while behavior and tests + remain centralized in envctl + ## Runtime Resolution Events diff --git a/python/envctl_engine/runtime/help_topic_inspection.py b/python/envctl_engine/runtime/help_topic_inspection.py index 696b062bb..a4aa18ad3 100644 --- a/python/envctl_engine/runtime/help_topic_inspection.py +++ b/python/envctl_engine/runtime/help_topic_inspection.py @@ -248,6 +248,8 @@ notes=( "intended for CI on a trusted self-hosted runner with gh, docker, git, and envctl available", "configuration is supplied through ENVCTL_PREVIEW_* environment variables", + "consume this command from CI instead of vendoring a repository-local controller copy", + "feature-branch push events only redeploy matching labeled same-repository PRs", ), aliases=("pr-preview", "github-pr-preview", "--pr-preview-controller", "--pr-preview"), related=("import", "start", "qa-user", "endpoints"), diff --git a/tests/python/runtime/test_pr_preview_controller.py b/tests/python/runtime/test_pr_preview_controller.py index 3af1acffb..7d00383b8 100644 --- a/tests/python/runtime/test_pr_preview_controller.py +++ b/tests/python/runtime/test_pr_preview_controller.py @@ -1238,6 +1238,27 @@ def test_push_event_refreshes_preview_ttl_for_labeled_branch(tmp_path): assert "- Reason: push to labeled PR branch feature/demo" in runner.comments[-1] +def test_controller_run_dispatches_github_push_payload(tmp_path, monkeypatch): + controller = load_controller() + event_path = tmp_path / "push.json" + event_path.write_text(json.dumps(push_payload(after="newsha")), encoding="utf-8") + config = make_config(controller, tmp_path) + instance = controller.PreviewController(config, FakeRunner(controller)) + received = [] + + monkeypatch.setattr( + instance, + "run_push_event", + lambda event: received.append(event) or 7, + ) + args = controller.build_parser().parse_args( + ["--event-name", "push", "--event-path", str(event_path)] + ) + + assert instance.run(args) == 7 + assert received == [push_payload(after="newsha")] + + def test_push_event_uses_pushed_sha_before_same_head_skip(tmp_path): controller = load_controller() root = tmp_path / "control" / "trees" / "imported" / "feature-demo" From ba5e8e88724f2e1c95bc4ce0bc61bce79851d793 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 01:24:49 +0300 Subject: [PATCH 05/26] Remove obsolete planning selection state and dead compatibility code --- .../planning/plan_agent/config.py | 31 +----- .../planning/plan_agent/constants.py | 1 - .../planning/plan_agent/launch_policy.py | 8 -- .../planning/plan_agent/workflow.py | 12 -- .../envctl_engine/planning/worktree_domain.py | 42 ------- .../planning/worktree_orchestrator.py | 23 ---- .../worktree_plan_project_selection.py | 2 +- .../planning/worktree_prompt_selection.py | 16 +-- .../planning/worktree_runtime_bridge.py | 3 - .../planning/worktree_selection_memory.py | 67 ----------- .../worktree_selection_runtime_bridge.py | 5 - .../runtime/engine_runtime_planning_facade.py | 19 ---- .../startup/resume_restore_support.py | 4 - .../plan_agent_launch_support_test_support.py | 7 +- .../planning/test_plan_agent_module_layout.py | 4 +- .../planning/test_planning_selection.py | 11 -- .../test_planning_worktree_setup_selection.py | 7 +- .../test_worktree_prompt_selection.py | 63 ++++------- .../test_worktree_selection_memory.py | 105 ------------------ ...t_engine_runtime_plan_selection_startup.py | 31 ------ ...test_structure_planning_worktree_layout.py | 10 -- 21 files changed, 36 insertions(+), 435 deletions(-) delete mode 100644 python/envctl_engine/planning/worktree_selection_memory.py delete mode 100644 tests/python/planning/test_worktree_selection_memory.py diff --git a/python/envctl_engine/planning/plan_agent/config.py b/python/envctl_engine/planning/plan_agent/config.py index ced72c8f5..9142b443c 100644 --- a/python/envctl_engine/planning/plan_agent/config.py +++ b/python/envctl_engine/planning/plan_agent/config.py @@ -2,7 +2,7 @@ # ruff: noqa: F401 import shutil -from typing import Any, Literal +from typing import Any from envctl_engine.config import EngineConfig, _apply_plan_agent_aliases @@ -18,7 +18,6 @@ normalize_plan_agent_surface_transport, parse_codex_cycles, uses_direct_submission, - workflow_mode_for_launch_config, ) from envctl_engine.planning.plan_agent.models import PlanAgentLaunchConfig @@ -39,14 +38,6 @@ def _launch_policy( ) -def _parse_codex_cycles(raw: object) -> tuple[int, str | None]: - return parse_codex_cycles(raw) - - -def _workflow_mode_for_launch_config(launch_config: PlanAgentLaunchConfig) -> str: - return workflow_mode_for_launch_config(launch_config) - - def _codex_tui_queue_workflow_supported(launch_config: PlanAgentLaunchConfig) -> bool: return codex_tui_queue_workflow_supported(launch_config) @@ -68,22 +59,6 @@ def resolve_plan_agent_launch_config( return _launch_policy(config, env, route=route).resolve_launch_config() -def _default_plan_agent_cli_command(cli: str, *, codex_yolo_enabled: bool = True) -> str: - return default_plan_agent_cli_command(cli, codex_yolo_enabled=codex_yolo_enabled) - - -def _default_plan_agent_surface_transport() -> Literal["cmux", "tmux"]: - return default_plan_agent_surface_transport(command_available=shutil.which) - - -def _resolve_available_plan_agent_surface_transport(raw: str) -> str: - return normalize_plan_agent_surface_transport(raw) - - -def _cmux_transport_explicitly_requested(config: EngineConfig, env: dict[str, str], route: object | None) -> bool: - return _launch_policy(config, env, route=route).cmux_transport_explicitly_requested() - - def _route_requests_ulw(route: object | None) -> bool: return bool(getattr(route, "flags", {}).get("ulw")) @@ -105,10 +80,6 @@ def plan_agent_launch_prereq_commands( return _launch_policy(config, env, route=route).prereq_commands() -def _command_executable(raw_command: str) -> str | None: - return command_executable(raw_command) - - def _missing_launch_commands(runtime: Any, launch_config: PlanAgentLaunchConfig) -> list[str]: return missing_launch_commands(runtime, launch_config) diff --git a/python/envctl_engine/planning/plan_agent/constants.py b/python/envctl_engine/planning/plan_agent/constants.py index aae7d6396..501084232 100644 --- a/python/envctl_engine/planning/plan_agent/constants.py +++ b/python/envctl_engine/planning/plan_agent/constants.py @@ -66,7 +66,6 @@ "queued follow-up messages", "follow-up inputs", ) -_OPENCODE_READY_PROMPT_RE = re.compile(r"^[ \t]*[>›❯»][ \t]*.*$") _OPENCODE_LOADING_MARKERS = ( "loading", "starting", diff --git a/python/envctl_engine/planning/plan_agent/launch_policy.py b/python/envctl_engine/planning/plan_agent/launch_policy.py index 093a91298..352a4ca40 100644 --- a/python/envctl_engine/planning/plan_agent/launch_policy.py +++ b/python/envctl_engine/planning/plan_agent/launch_policy.py @@ -16,8 +16,6 @@ _DEFAULT_PRESET, _DEFAULT_SHELL, _PLAN_AGENT_CODEX_CYCLE_CAP, - _PLAN_AGENT_WORKFLOW_CODEX_CYCLES, - _PLAN_AGENT_WORKFLOW_SINGLE_PROMPT, _SUPPORTED_PLAN_AGENT_CLIS, ) from envctl_engine.planning.plan_agent.models import PlanAgentLaunchConfig @@ -306,12 +304,6 @@ def parse_codex_cycles(raw: object) -> tuple[int, str | None]: return value, None -def workflow_mode_for_launch_config(launch_config: PlanAgentLaunchConfig) -> str: - if codex_tui_queue_workflow_supported(launch_config) and launch_config.codex_cycles > 0: - return _PLAN_AGENT_WORKFLOW_CODEX_CYCLES - return _PLAN_AGENT_WORKFLOW_SINGLE_PROMPT - - def codex_tui_queue_workflow_supported(launch_config: PlanAgentLaunchConfig) -> bool: return launch_config.cli == "codex" and launch_config.transport in {"cmux", "tmux", "omx"} diff --git a/python/envctl_engine/planning/plan_agent/workflow.py b/python/envctl_engine/planning/plan_agent/workflow.py index d186f2529..846f064d3 100644 --- a/python/envctl_engine/planning/plan_agent/workflow.py +++ b/python/envctl_engine/planning/plan_agent/workflow.py @@ -35,7 +35,6 @@ _OMX_WORKFLOW_KEYWORDS, _OPENCODE_LOADING_MARKERS, _OPENCODE_READY_MARKERS, - _OPENCODE_READY_PROMPT_RE, _PLAN_AGENT_CODEX_CYCLE_CAP, _PLAN_AGENT_TAB_TITLE_MAX_LEN, _PLAN_AGENT_WORKFLOW_CODEX_CYCLES, @@ -59,20 +58,13 @@ from envctl_engine.planning.plan_agent.config import ( PlanAgentLaunchPolicy, _cli_ready_delay_seconds, - _cmux_transport_explicitly_requested, _codex_tui_queue_workflow_supported, - _command_executable, - _default_plan_agent_cli_command, - _default_plan_agent_surface_transport, _guidance_attach_command, _launch_policy, _missing_launch_commands, - _parse_codex_cycles, - _resolve_available_plan_agent_surface_transport, _route_requests_ulw, _ulw_route_supported, _uses_direct_submission, - _workflow_mode_for_launch_config, cli_ready_delay_seconds, codex_tui_queue_workflow_supported, command_executable, @@ -85,7 +77,6 @@ plan_agent_launch_prereq_commands, resolve_plan_agent_launch_config, uses_direct_submission, - workflow_mode_for_launch_config, ) from envctl_engine.planning.plan_agent.models import ( AgentTerminalLaunchResult, @@ -128,9 +119,6 @@ PlanAgentWorkflowBuilder, _browser_e2e_instruction_text, _build_plan_agent_workflow, - _finalization_instruction_text, - _first_cycle_completion_instruction_text, - _intermediate_cycle_completion_instruction_text, _load_plan_agent_followup_prompt, _pr_review_comments_instruction_text, _slash_command, diff --git a/python/envctl_engine/planning/worktree_domain.py b/python/envctl_engine/planning/worktree_domain.py index 93a746d87..9c61281b1 100644 --- a/python/envctl_engine/planning/worktree_domain.py +++ b/python/envctl_engine/planning/worktree_domain.py @@ -44,12 +44,6 @@ feature_project_candidates as _feature_project_candidates_impl, project_sort_key_for_feature as _project_sort_key_for_feature_impl, ) -from envctl_engine.planning.worktree_selection_memory import ( - initial_plan_selected_counts as _initial_plan_selected_counts_impl, - load_plan_selection_memory as _load_plan_selection_memory_impl, - plan_selection_memory_path as _plan_selection_memory_path_impl, - save_plan_selection_memory as _save_plan_selection_memory_impl, -) from envctl_engine.planning.worktree_shared_artifacts import ( link_repo_local_shared_artifacts as _link_repo_local_shared_artifacts_impl, ) @@ -303,13 +297,10 @@ def _prompt_planning_selection( self: Any, planning_files: list[str], raw_projects: list[tuple[str, Path]], - *, - persist_memory: bool = True, ) -> dict[str, int] | None: return _planning_runtime_bridge(self).prompt_planning_selection( planning_files=planning_files, raw_projects=raw_projects, - persist_memory=persist_memory, ) @@ -330,20 +321,6 @@ def _adjust_plan_counts_for_fresh_ai_launch( ) -def _initial_plan_selected_counts( - self: Any, - *, - planning_files: list[str], - existing_counts: dict[str, int], -) -> dict[str, int]: - remembered = self._load_plan_selection_memory() - return _initial_plan_selected_counts_impl( - planning_files=planning_files, - existing_counts=existing_counts, - remembered_counts=remembered, - ) - - def _run_planning_selection_menu( self: Any, *, @@ -430,10 +407,6 @@ def _resolve_planning_selection_target( ) -def _plan_selection_memory_path(self: Any) -> Path: - return _plan_selection_memory_path_impl(runtime_root=self.runtime_root) - - def _planning_root(self: Any) -> Path: return _planning_root_impl(planning_dir=self.config.planning_dir) @@ -442,21 +415,6 @@ def _planning_done_root(self: Any) -> Path: return _planning_done_root_impl(planning_dir=self.config.planning_dir) -def _load_plan_selection_memory(self: Any) -> dict[str, int]: - return _load_plan_selection_memory_impl( - runtime_root=self.runtime_root, - runtime_legacy_root=self.runtime_legacy_root, - ) - - -def _save_plan_selection_memory(self: Any, selected_counts: dict[str, int]) -> None: - _save_plan_selection_memory_impl( - runtime_root=self.runtime_root, - runtime_legacy_root=self.runtime_legacy_root, - selected_counts=selected_counts, - ) - - def _planning_keep_plan_enabled(self: Any, route: Route) -> bool: return _planning_keep_plan_enabled_impl(route=route, env=self.env, config_raw=self.config.raw) diff --git a/python/envctl_engine/planning/worktree_orchestrator.py b/python/envctl_engine/planning/worktree_orchestrator.py index 4e3d69869..faabe8c22 100644 --- a/python/envctl_engine/planning/worktree_orchestrator.py +++ b/python/envctl_engine/planning/worktree_orchestrator.py @@ -7,12 +7,9 @@ from envctl_engine.planning.plan_agent.models import PlanSelectionResult, PlanWorktreeSyncResult from envctl_engine.planning.worktree_domain import ( _apply_setup_worktree_selection as domain_apply_setup_worktree_selection, - _initial_plan_selected_counts as domain_initial_plan_selected_counts, - _load_plan_selection_memory as domain_load_plan_selection_memory, _planning_keep_plan_enabled as domain_planning_keep_plan_enabled, _prompt_planning_selection as domain_prompt_planning_selection, _run_planning_selection_menu as domain_run_planning_selection_menu, - _save_plan_selection_memory as domain_save_plan_selection_memory, _select_plan_projects as domain_select_plan_projects, _sync_plan_worktrees_from_plan_counts as domain_sync_plan_worktrees_from_plan_counts, ) @@ -25,9 +22,6 @@ class PlanningWorktreeOrchestrator: _apply_setup_worktree_selection = domain_apply_setup_worktree_selection _select_plan_projects = domain_select_plan_projects _prompt_planning_selection = domain_prompt_planning_selection - _initial_plan_selected_counts = domain_initial_plan_selected_counts - _load_plan_selection_memory = domain_load_plan_selection_memory - _save_plan_selection_memory = domain_save_plan_selection_memory _planning_keep_plan_enabled = domain_planning_keep_plan_enabled _sync_plan_worktrees_from_plan_counts = domain_sync_plan_worktrees_from_plan_counts _import_remote_branch_worktree = domain_import_remote_branch_worktree @@ -66,17 +60,6 @@ def prompt_planning_selection( ) -> dict[str, int] | None: return self._prompt_planning_selection(planning_files, raw_projects) - def initial_plan_selected_counts( - self, - *, - planning_files: list[str], - existing_counts: dict[str, int], - ) -> dict[str, int]: - return self._initial_plan_selected_counts( - planning_files=planning_files, - existing_counts=existing_counts, - ) - def run_planning_selection_menu( self, *, @@ -111,12 +94,6 @@ def _run_planning_selection_menu( existing_counts=existing_counts, ) - def load_plan_selection_memory(self) -> dict[str, int]: - return self._load_plan_selection_memory() - - def save_plan_selection_memory(self, selected_counts: dict[str, int]) -> None: - self._save_plan_selection_memory(selected_counts) - def planning_keep_plan_enabled(self, route: Route) -> bool: return self._planning_keep_plan_enabled(route) diff --git a/python/envctl_engine/planning/worktree_plan_project_selection.py b/python/envctl_engine/planning/worktree_plan_project_selection.py index 8a6ce8ea1..f84f1fdb6 100644 --- a/python/envctl_engine/planning/worktree_plan_project_selection.py +++ b/python/envctl_engine/planning/worktree_plan_project_selection.py @@ -190,7 +190,7 @@ def select_plan_projects( return PlanSelectionResult(raw_projects=raw_projects, selected_contexts=[], error="No TTY available.") dry_run = bool(getattr(route, "flags", {}).get("dry_run")) - plan_counts = prompt_planning_selection(planning_files, raw_projects, persist_memory=not dry_run) + plan_counts = prompt_planning_selection(planning_files, raw_projects) if plan_counts is None: output("Planning selection cancelled.") return PlanSelectionResult( diff --git a/python/envctl_engine/planning/worktree_prompt_selection.py b/python/envctl_engine/planning/worktree_prompt_selection.py index 4de1215c0..83c83b00c 100644 --- a/python/envctl_engine/planning/worktree_prompt_selection.py +++ b/python/envctl_engine/planning/worktree_prompt_selection.py @@ -10,21 +10,15 @@ def prompt_planning_selection( *, planning_files: list[str], raw_projects: list[tuple[str, Path]], - initial_plan_selected_counts: Callable[..., dict[str, int]], run_planning_selection_menu: Callable[..., dict[str, int] | None], - save_plan_selection_memory: Callable[[dict[str, int]], None], - persist_memory: bool = True, ) -> dict[str, int] | None: existing_counts = planning_existing_counts(projects=raw_projects, planning_files=planning_files) - selected_counts = initial_plan_selected_counts( - planning_files=planning_files, - existing_counts=existing_counts, - ) - chosen = run_planning_selection_menu( + selected_counts = { + plan_file: max(int(existing_counts.get(plan_file, 0)), 0) + for plan_file in planning_files + } + return run_planning_selection_menu( planning_files=planning_files, selected_counts=selected_counts, existing_counts=existing_counts, ) - if chosen and persist_memory: - save_plan_selection_memory(chosen) - return chosen diff --git a/python/envctl_engine/planning/worktree_runtime_bridge.py b/python/envctl_engine/planning/worktree_runtime_bridge.py index 621ad5407..28036805b 100644 --- a/python/envctl_engine/planning/worktree_runtime_bridge.py +++ b/python/envctl_engine/planning/worktree_runtime_bridge.py @@ -201,13 +201,10 @@ def prompt_planning_selection( self, planning_files: list[str], raw_projects: list[tuple[str, Path]], - *, - persist_memory: bool = True, ) -> dict[str, int] | None: return self.selection_bridge().prompt_planning_selection( planning_files=planning_files, raw_projects=raw_projects, - persist_memory=persist_memory, ) def run_planning_selection_menu( diff --git a/python/envctl_engine/planning/worktree_selection_memory.py b/python/envctl_engine/planning/worktree_selection_memory.py deleted file mode 100644 index 67520bc5f..000000000 --- a/python/envctl_engine/planning/worktree_selection_memory.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import annotations - -import json -from datetime import UTC, datetime -from pathlib import Path - - -def plan_selection_memory_path(*, runtime_root: Path) -> Path: - return runtime_root / "planning_selection.json" - - -def load_plan_selection_memory(*, runtime_root: Path, runtime_legacy_root: Path) -> dict[str, int]: - path = plan_selection_memory_path(runtime_root=runtime_root) - legacy_path = runtime_legacy_root / "planning_selection.json" - for candidate in (path, legacy_path): - if not candidate.is_file(): - continue - try: - payload = json.loads(candidate.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - continue - if not isinstance(payload, dict): - continue - remembered = payload.get("selected_counts", {}) - if not isinstance(remembered, dict): - continue - result: dict[str, int] = {} - for key, value in remembered.items(): - try: - parsed = int(value) - except (TypeError, ValueError): - continue - if parsed >= 0: - result[str(key)] = parsed - if result: - return result - return {} - - -def save_plan_selection_memory( - *, - runtime_root: Path, - runtime_legacy_root: Path, - selected_counts: dict[str, int], -) -> None: - path = plan_selection_memory_path(runtime_root=runtime_root) - payload = { - "selected_counts": dict(sorted({k: int(v) for k, v in selected_counts.items() if int(v) > 0}.items())), - "saved_at": datetime.now(UTC).isoformat(), - } - text = json.dumps(payload, indent=2, sort_keys=True) - path.write_text(text, encoding="utf-8") - runtime_legacy_root.mkdir(parents=True, exist_ok=True) - (runtime_legacy_root / "planning_selection.json").write_text(text, encoding="utf-8") - - -def initial_plan_selected_counts( - *, - planning_files: list[str], - existing_counts: dict[str, int], - remembered_counts: dict[str, int], -) -> dict[str, int]: - selected_counts: dict[str, int] = {} - for plan_file in planning_files: - existing = int(existing_counts.get(plan_file, 0)) - selected_counts[plan_file] = max(existing, 0) - return selected_counts diff --git a/python/envctl_engine/planning/worktree_selection_runtime_bridge.py b/python/envctl_engine/planning/worktree_selection_runtime_bridge.py index a67435c7a..05b19e7d5 100644 --- a/python/envctl_engine/planning/worktree_selection_runtime_bridge.py +++ b/python/envctl_engine/planning/worktree_selection_runtime_bridge.py @@ -45,17 +45,12 @@ def prompt_planning_selection( self, planning_files: list[str], raw_projects: list[tuple[str, Path]], - *, - persist_memory: bool = True, ) -> dict[str, int] | None: runtime = self.runtime return prompt_planning_selection( planning_files=planning_files, raw_projects=raw_projects, - initial_plan_selected_counts=runtime._initial_plan_selected_counts, run_planning_selection_menu=runtime._run_planning_selection_menu, - save_plan_selection_memory=runtime._save_plan_selection_memory, - persist_memory=persist_memory, ) def run_planning_selection_menu( diff --git a/python/envctl_engine/runtime/engine_runtime_planning_facade.py b/python/envctl_engine/runtime/engine_runtime_planning_facade.py index 46b587fa2..bdc61e245 100644 --- a/python/envctl_engine/runtime/engine_runtime_planning_facade.py +++ b/python/envctl_engine/runtime/engine_runtime_planning_facade.py @@ -16,7 +16,6 @@ _planning_done_root as domain_planning_done_root, _planning_menu_apply_key as domain_planning_menu_apply_key, _planning_root as domain_planning_root, - _plan_selection_memory_path as domain_plan_selection_memory_path, _preferred_tree_root_for_feature as domain_preferred_tree_root_for_feature, _project_sort_key_for_feature as domain_project_sort_key_for_feature, _read_planning_menu_escape_sequence as domain_read_planning_menu_escape_sequence, @@ -54,7 +53,6 @@ class RuntimePlanningFacadeMixin: _resolve_planning_selection_target = domain_resolve_planning_selection_target _planning_root = domain_planning_root _planning_done_root = domain_planning_done_root - _plan_selection_memory_path = domain_plan_selection_memory_path _create_feature_worktrees = domain_create_feature_worktrees _worktree_add_failure = domain_worktree_add_failure _setup_worktree_placeholder_fallback_enabled = domain_setup_worktree_placeholder_fallback_enabled @@ -79,17 +77,6 @@ def _prompt_planning_selection( ) -> dict[str, int] | None: return _planning_orchestrator(self).prompt_planning_selection(planning_files, raw_projects) - def _initial_plan_selected_counts( - self, - *, - planning_files: list[str], - existing_counts: dict[str, int], - ) -> dict[str, int]: - return _planning_orchestrator(self).initial_plan_selected_counts( - planning_files=planning_files, - existing_counts=existing_counts, - ) - def _run_planning_selection_menu( self, *, @@ -103,12 +90,6 @@ def _run_planning_selection_menu( existing_counts=existing_counts, ) - def _load_plan_selection_memory(self) -> dict[str, int]: - return _planning_orchestrator(self).load_plan_selection_memory() - - def _save_plan_selection_memory(self, selected_counts: dict[str, int]) -> None: - _planning_orchestrator(self).save_plan_selection_memory(selected_counts) - def _planning_keep_plan_enabled(self, route: Route) -> bool: return _planning_orchestrator(self).planning_keep_plan_enabled(route) diff --git a/python/envctl_engine/startup/resume_restore_support.py b/python/envctl_engine/startup/resume_restore_support.py index 03b9f73cb..0f89d1376 100644 --- a/python/envctl_engine/startup/resume_restore_support.py +++ b/python/envctl_engine/startup/resume_restore_support.py @@ -9,10 +9,6 @@ ResumeRestoreDependencies, ResumeRestoreRunner, ) -from envctl_engine.startup.resume_restore_results import ( - format_project_timing_line as _format_project_timing_line, # noqa: F401 - mark_restore_failure_requirements as _mark_restore_failure_requirements, # noqa: F401 -) from envctl_engine.startup.resume_restore_policy import ( _configured_restore_service_types as _configured_restore_service_types, _port_allocator as _port_allocator, diff --git a/tests/python/planning/plan_agent_launch_support_test_support.py b/tests/python/planning/plan_agent_launch_support_test_support.py index 260aa7385..0ba55bf58 100644 --- a/tests/python/planning/plan_agent_launch_support_test_support.py +++ b/tests/python/planning/plan_agent_launch_support_test_support.py @@ -27,6 +27,7 @@ tmux_session, tmux_transport, workflow, + workflow_build, ) from envctl_engine.planning.plan_agent.models import _WorkspaceLaunchTarget from envctl_engine.planning.plan_agent_launch_support import CreatedPlanWorktree, launch_plan_agent_terminals @@ -37,9 +38,9 @@ _prompt_submit_screen_looks_ready = terminal_screen._prompt_submit_screen_looks_ready _tab_title_for_worktree = workflow._tab_title_for_worktree _build_plan_agent_workflow = cast(Any, workflow._build_plan_agent_workflow) -_finalization_instruction_text = cast(Any, workflow._finalization_instruction_text) -_first_cycle_completion_instruction_text = cast(Any, workflow._first_cycle_completion_instruction_text) -_intermediate_cycle_completion_instruction_text = cast(Any, workflow._intermediate_cycle_completion_instruction_text) +_finalization_instruction_text = cast(Any, workflow_build._finalization_instruction_text) +_first_cycle_completion_instruction_text = cast(Any, workflow_build._first_cycle_completion_instruction_text) +_intermediate_cycle_completion_instruction_text = cast(Any, workflow_build._intermediate_cycle_completion_instruction_text) _browser_e2e_instruction_text = cast(Any, workflow._browser_e2e_instruction_text) _pr_review_comments_instruction_text = cast(Any, workflow._pr_review_comments_instruction_text) _wait_for_codex_queue_ready = cast(Any, cmux_transport._wait_for_codex_queue_ready) diff --git a/tests/python/planning/test_plan_agent_module_layout.py b/tests/python/planning/test_plan_agent_module_layout.py index d5de7b910..a43b5ddd0 100644 --- a/tests/python/planning/test_plan_agent_module_layout.py +++ b/tests/python/planning/test_plan_agent_module_layout.py @@ -198,10 +198,12 @@ def test_launch_module_owns_public_dispatch_functions(self) -> None: def test_core_helpers_live_in_owner_modules(self) -> None: expected = { "config.py": { - "_parse_codex_cycles", "resolve_plan_agent_launch_config", "plan_agent_launch_prereq_commands", }, + "launch_policy.py": { + "parse_codex_cycles", + }, "launch_evaluation.py": { "PlanAgentLaunchEvaluation", "build_plan_agent_launch_evaluation", diff --git a/tests/python/planning/test_planning_selection.py b/tests/python/planning/test_planning_selection.py index ed77bcb2a..cb3a7d8f3 100644 --- a/tests/python/planning/test_planning_selection.py +++ b/tests/python/planning/test_planning_selection.py @@ -18,7 +18,6 @@ resolve_planning_files, select_projects_for_plan_files, ) -from envctl_engine.planning.worktree_selection_memory import initial_plan_selected_counts def _init_repo_with_commit(repo: Path) -> None: @@ -257,15 +256,5 @@ def test_planning_existing_counts_ignore_stale_branch_names_without_discovered_w self.assertEqual(counts["backend/task.md"], 0) - def test_initial_plan_selected_counts_do_not_resurrect_deleted_worktrees_from_memory(self) -> None: - selected = initial_plan_selected_counts( - planning_files=["backend/task.md", "frontend/task.md"], - existing_counts={"backend/task.md": 0, "frontend/task.md": 2}, - remembered_counts={"backend/task.md": 3, "frontend/task.md": 1}, - ) - - self.assertEqual(selected, {"backend/task.md": 0, "frontend/task.md": 2}) - - if __name__ == "__main__": unittest.main() diff --git a/tests/python/planning/test_planning_worktree_setup_selection.py b/tests/python/planning/test_planning_worktree_setup_selection.py index b1d4785ba..2751b87db 100644 --- a/tests/python/planning/test_planning_worktree_setup_selection.py +++ b/tests/python/planning/test_planning_worktree_setup_selection.py @@ -73,7 +73,6 @@ def test_interactive_plan_dry_run_previews_without_syncing_worktrees_or_memory(s orchestrator = engine.planning_worktree_orchestrator route = parse_route(["--plan", "--dry-run"], env={}) sync_calls: list[dict[str, int]] = [] - save_calls: list[dict[str, int]] = [] def fake_menu( planning_files: list[str], @@ -88,13 +87,9 @@ def fake_sync(**kwargs: object) -> object: sync_calls.append(dict(kwargs["plan_counts"])) # type: ignore[index,arg-type] return engine._sync_plan_worktrees_from_plan_counts(**kwargs) # type: ignore[arg-type] - def fake_save(chosen: dict[str, int]) -> None: - save_calls.append(dict(chosen)) - with ( patch.object(orchestrator, "_run_planning_selection_menu", side_effect=fake_menu), patch.object(orchestrator, "_sync_plan_worktrees_from_plan_counts", side_effect=fake_sync), - patch.object(orchestrator, "_save_plan_selection_memory", side_effect=fake_save), patch.object(sys.stdin, "isatty", return_value=True), patch.object(sys.stdout, "isatty", return_value=True), ): @@ -102,8 +97,8 @@ def fake_save(chosen: dict[str, int]) -> None: self.assertEqual([ctx.name for ctx in selected], ["feature_task-1"]) self.assertEqual(sync_calls, []) - self.assertEqual(save_calls, []) self.assertFalse((repo / "trees" / "feature_task" / "1").exists()) + self.assertEqual(list(runtime.rglob("planning_selection.json")), []) result = orchestrator.last_plan_selection_result() self.assertEqual([worktree.name for worktree in result.created_worktrees], ["feature_task-1"]) diff --git a/tests/python/planning/test_worktree_prompt_selection.py b/tests/python/planning/test_worktree_prompt_selection.py index 7932205c5..4e2992d04 100644 --- a/tests/python/planning/test_worktree_prompt_selection.py +++ b/tests/python/planning/test_worktree_prompt_selection.py @@ -2,13 +2,13 @@ import unittest from pathlib import Path -from typing import Any +from unittest.mock import patch from envctl_engine.planning.worktree_prompt_selection import prompt_planning_selection class WorktreePromptSelectionTests(unittest.TestCase): - def test_prompt_planning_selection_seeds_menu_from_existing_counts_and_memory(self) -> None: + def test_prompt_planning_selection_seeds_menu_from_current_existing_counts(self) -> None: calls: dict[str, object] = {} def choose(**kwargs: object) -> dict[str, int]: @@ -18,51 +18,30 @@ def choose(**kwargs: object) -> dict[str, int]: chosen = prompt_planning_selection( planning_files=["backend/task.md", "frontend/task.md"], raw_projects=[("backend_task-1", Path("/repo/trees/backend_task/1"))], - initial_plan_selected_counts=lambda **kwargs: { - "backend/task.md": kwargs["existing_counts"]["backend/task.md"], - "frontend/task.md": 2, - }, run_planning_selection_menu=choose, - save_plan_selection_memory=self._unexpected, - persist_memory=False, ) self.assertEqual(chosen, {"backend/task.md": 1, "frontend/task.md": 2}) menu_call = calls["menu"] self.assertEqual(menu_call["existing_counts"], {"backend/task.md": 1, "frontend/task.md": 0}) - self.assertEqual(menu_call["selected_counts"], {"backend/task.md": 1, "frontend/task.md": 2}) + self.assertEqual(menu_call["selected_counts"], {"backend/task.md": 1, "frontend/task.md": 0}) - def test_prompt_planning_selection_persists_chosen_counts_when_enabled(self) -> None: - saved: list[dict[str, int]] = [] - - chosen = prompt_planning_selection( - planning_files=["backend/task.md"], - raw_projects=[], - persist_memory=True, - initial_plan_selected_counts=lambda **_kwargs: {"backend/task.md": 1}, - run_planning_selection_menu=lambda **_kwargs: {"backend/task.md": 3}, - save_plan_selection_memory=saved.append, - ) - - self.assertEqual(chosen, {"backend/task.md": 3}) - self.assertEqual(saved, [{"backend/task.md": 3}]) - - def test_prompt_planning_selection_skips_memory_for_cancel_empty_or_dry_run(self) -> None: - for persist_memory, menu_result in [(True, None), (True, {}), (False, {"backend/task.md": 1})]: - with self.subTest(persist_memory=persist_memory, menu_result=menu_result): - saved: list[dict[str, int]] = [] - - chosen = prompt_planning_selection( - planning_files=["backend/task.md"], - raw_projects=[], - persist_memory=persist_memory, - initial_plan_selected_counts=lambda **_kwargs: {"backend/task.md": 1}, - run_planning_selection_menu=lambda **_kwargs: menu_result, - save_plan_selection_memory=saved.append, - ) - - self.assertEqual(chosen, menu_result) - self.assertEqual(saved, []) + def test_prompt_planning_selection_clamps_invalid_existing_counts(self) -> None: + calls: dict[str, object] = {} - def _unexpected(self, *_args: Any, **_kwargs: Any) -> Any: - raise AssertionError("callback should not be called") + def choose(**kwargs: object) -> None: + calls.update(kwargs) + return None + + with patch( + "envctl_engine.planning.worktree_prompt_selection.planning_existing_counts", + return_value={"backend/task.md": -2, "frontend/task.md": 3}, + ): + chosen = prompt_planning_selection( + planning_files=["backend/task.md", "frontend/task.md"], + raw_projects=[], + run_planning_selection_menu=choose, + ) + + self.assertIsNone(chosen) + self.assertEqual(calls["selected_counts"], {"backend/task.md": 0, "frontend/task.md": 3}) diff --git a/tests/python/planning/test_worktree_selection_memory.py b/tests/python/planning/test_worktree_selection_memory.py deleted file mode 100644 index 05ed4affb..000000000 --- a/tests/python/planning/test_worktree_selection_memory.py +++ /dev/null @@ -1,105 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -import tempfile -import unittest - -from envctl_engine.planning.worktree_selection_memory import ( - initial_plan_selected_counts, - load_plan_selection_memory, - plan_selection_memory_path, - save_plan_selection_memory, -) - - -class WorktreeSelectionMemoryTests(unittest.TestCase): - def test_plan_selection_memory_path_uses_runtime_root(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - runtime_root = Path(tmpdir) / "runtime" - - self.assertEqual( - plan_selection_memory_path(runtime_root=runtime_root), - runtime_root / "planning_selection.json", - ) - - def test_load_plan_selection_memory_prefers_current_file_and_filters_values(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - runtime_root = root / "runtime" - legacy_root = root / "legacy" - runtime_root.mkdir() - legacy_root.mkdir() - (runtime_root / "planning_selection.json").write_text( - json.dumps( - { - "selected_counts": { - "feature/a.md": "2", - "feature/b.md": 0, - "feature/c.md": -1, - "feature/d.md": "invalid", - } - } - ), - encoding="utf-8", - ) - (legacy_root / "planning_selection.json").write_text( - json.dumps({"selected_counts": {"feature/a.md": 9}}), - encoding="utf-8", - ) - - self.assertEqual( - load_plan_selection_memory(runtime_root=runtime_root, runtime_legacy_root=legacy_root), - {"feature/a.md": 2, "feature/b.md": 0}, - ) - - def test_load_plan_selection_memory_falls_back_to_legacy_file(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - runtime_root = root / "runtime" - legacy_root = root / "legacy" - runtime_root.mkdir() - legacy_root.mkdir() - (runtime_root / "planning_selection.json").write_text("not json", encoding="utf-8") - (legacy_root / "planning_selection.json").write_text( - json.dumps({"selected_counts": {"feature/a.md": "3"}}), - encoding="utf-8", - ) - - self.assertEqual( - load_plan_selection_memory(runtime_root=runtime_root, runtime_legacy_root=legacy_root), - {"feature/a.md": 3}, - ) - - def test_save_plan_selection_memory_writes_sorted_positive_counts_to_current_and_legacy(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - runtime_root = root / "runtime" - legacy_root = root / "legacy" - runtime_root.mkdir() - - save_plan_selection_memory( - runtime_root=runtime_root, - runtime_legacy_root=legacy_root, - selected_counts={"z.md": 2, "a.md": 1, "zero.md": 0}, - ) - - current_payload = json.loads((runtime_root / "planning_selection.json").read_text(encoding="utf-8")) - legacy_payload = json.loads((legacy_root / "planning_selection.json").read_text(encoding="utf-8")) - self.assertEqual(current_payload, legacy_payload) - self.assertEqual(list(current_payload["selected_counts"].items()), [("a.md", 1), ("z.md", 2)]) - self.assertIsInstance(current_payload["saved_at"], str) - - def test_initial_plan_selected_counts_uses_current_existing_counts_only(self) -> None: - self.assertEqual( - initial_plan_selected_counts( - planning_files=["feature/a.md", "feature/b.md", "feature/c.md"], - existing_counts={"feature/a.md": 2}, - remembered_counts={"feature/a.md": 9, "feature/b.md": 4, "feature/c.md": -1}, - ), - {"feature/a.md": 2, "feature/b.md": 0, "feature/c.md": 0}, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/python/runtime/test_engine_runtime_plan_selection_startup.py b/tests/python/runtime/test_engine_runtime_plan_selection_startup.py index e518f4842..6d1da16b2 100644 --- a/tests/python/runtime/test_engine_runtime_plan_selection_startup.py +++ b/tests/python/runtime/test_engine_runtime_plan_selection_startup.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import os import re import tempfile @@ -75,36 +74,6 @@ def test_plan_without_selection_and_without_tty_fails_when_planning_files_exist( self.assertEqual(code, 1) - def test_initial_plan_selected_counts_prefers_existing_counts(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - repo = Path(tmpdir) / "repo" - runtime = Path(tmpdir) / "runtime" - (repo / ".git").mkdir(parents=True, exist_ok=True) - engine = PythonEngineRuntime(self._config(repo, runtime), env={}) - selected = engine._initial_plan_selected_counts( - planning_files=["backend/task-a.md", "backend/task-b.md"], - existing_counts={"backend/task-a.md": 2, "backend/task-b.md": 1}, - ) - self.assertEqual(selected, {"backend/task-a.md": 2, "backend/task-b.md": 1}) - - def test_initial_plan_selected_counts_ignores_memory_when_existing_missing(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - repo = Path(tmpdir) / "repo" - runtime = Path(tmpdir) / "runtime" - (repo / ".git").mkdir(parents=True, exist_ok=True) - engine = PythonEngineRuntime(self._config(repo, runtime), env={}) - memory_path = runtime / "python-engine" / "planning_selection.json" - memory_path.parent.mkdir(parents=True, exist_ok=True) - memory_path.write_text( - json.dumps({"selected_counts": {"backend/task-a.md": 3}}, indent=2, sort_keys=True), - encoding="utf-8", - ) - selected = engine._initial_plan_selected_counts( - planning_files=["backend/task-a.md", "backend/task-b.md"], - existing_counts={}, - ) - self.assertEqual(selected, {"backend/task-a.md": 0, "backend/task-b.md": 0}) - def test_planning_menu_apply_key_supports_arrow_navigation_and_count_adjustment(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo = Path(tmpdir) / "repo" diff --git a/tests/python/shared/test_structure_planning_worktree_layout.py b/tests/python/shared/test_structure_planning_worktree_layout.py index 36bbb178a..9fe34af50 100644 --- a/tests/python/shared/test_structure_planning_worktree_layout.py +++ b/tests/python/shared/test_structure_planning_worktree_layout.py @@ -278,16 +278,6 @@ def test_worktree_main_task_has_owned_module(self) -> None: facade.read_text(encoding="utf-8"), ) - def test_worktree_selection_memory_has_owned_module(self) -> None: - owner = REPO_ROOT / "python" / "envctl_engine" / "planning" / "worktree_selection_memory.py" - facade = REPO_ROOT / "python" / "envctl_engine" / "planning" / "worktree_domain.py" - - self.assertTrue(owner.is_file()) - self.assertIn( - "from envctl_engine.planning.worktree_selection_memory import", - facade.read_text(encoding="utf-8"), - ) - def test_worktree_project_catalog_has_owned_module(self) -> None: owner = REPO_ROOT / "python" / "envctl_engine" / "planning" / "worktree_project_catalog.py" facade = REPO_ROOT / "python" / "envctl_engine" / "planning" / "worktree_domain.py" From 6b5d749494597fa6ef92f98eb2b7d55de3cb08a0 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 01:38:21 +0300 Subject: [PATCH 06/26] Harden multi-target action selection and execution --- .../actions/action_target_support.py | 531 ++++++++++++------ .../actions/test_action_target_support.py | 226 ++++++++ 2 files changed, 598 insertions(+), 159 deletions(-) diff --git a/python/envctl_engine/actions/action_target_support.py b/python/envctl_engine/actions/action_target_support.py index 7eba9d430..010649384 100644 --- a/python/envctl_engine/actions/action_target_support.py +++ b/python/envctl_engine/actions/action_target_support.py @@ -68,7 +68,8 @@ def action_target_names_with_roots(targets: Sequence[object]) -> list[str]: def build_action_target_contexts(targets: Sequence[object]) -> list[ActionTargetContext]: - total = max(len(targets), 1) + identities = action_target_identities(targets) + total = len(identities) return [ ActionTargetContext( name=identity.name, @@ -77,7 +78,7 @@ def build_action_target_contexts(targets: Sequence[object]) -> list[ActionTarget total=total, target_obj=identity.target_obj, ) - for index, identity in enumerate(action_target_identities(targets), start=1) + for index, identity in enumerate(identities, start=1) ] @@ -100,42 +101,216 @@ def emit_action_output( return True +def _normalized_service_targets(service_targets: list[object]) -> list[str]: + normalized: list[str] = [] + for target in service_targets: + value = str(target).strip().lower().removeprefix("service:") + if value: + normalized.append(value) + return normalized + + +def _projects_by_service_selector(runtime: Any, state: object | None) -> dict[str, list[str]]: + projects_by_selector: dict[str, list[str]] = {} + services = getattr(state, "services", {}) if state is not None else {} + for service_name, service in services.items(): + project = runtime.project_name_from_service(service_name) + if not project: + continue + service_type = str(getattr(service, "type", "") or "").strip().lower() + selectors = {service_name.lower(), service_type} - {""} + for selector in selectors: + projects_by_selector.setdefault(selector, []).append(project) + return projects_by_selector + + +def _deduplicate_project_names(projects: list[str]) -> list[str]: + deduped: list[str] = [] + seen: set[str] = set() + for project in projects: + lowered = project.lower() + if lowered not in seen: + seen.add(lowered) + deduped.append(project) + return deduped + + def projects_for_services(runtime: Any, service_targets: list[object]) -> list[str]: - normalized_targets = [ - str(target).strip().lower().removeprefix("service:") - for target in service_targets - if str(target).strip() - ] + projects, _unresolved = _resolve_projects_for_services(runtime, service_targets) + return projects + + +def _resolve_projects_for_services( + runtime: Any, + service_targets: list[object], +) -> tuple[list[str], tuple[str, ...]]: + normalized_targets = _normalized_service_targets(service_targets) if not normalized_targets: - return [] + return [], () state = runtime.load_existing_state(mode="trees") or runtime.load_existing_state(mode="main") + projects_by_selector = _projects_by_service_selector(runtime, state) resolved: list[str] = [] + unresolved: list[str] = [] for target in normalized_targets: - matched = False - if state is not None: - for service_name, service in state.services.items(): - service_type = str(getattr(service, "type", "") or "").strip().lower() - if service_name.lower() == target or service_type == target: - project = runtime.project_name_from_service(service_name) - if project: - resolved.append(project) - matched = True - if matched: - continue - project = runtime.project_name_from_service(target) - if project: + matched_projects = projects_by_selector.get(target, []) + if matched_projects: + resolved.extend(matched_projects) + elif project := runtime.project_name_from_service(target): resolved.append(project) + else: + unresolved.append(target) + return _deduplicate_project_names(resolved), tuple(unresolved) - deduped: list[str] = [] - seen: set[str] = set() - for project in resolved: - lowered = project.lower() - if lowered in seen: - continue - seen.add(lowered) - deduped.append(project) - return deduped + +def _discover_action_candidates(runtime: Any, route: Route, *, trees_only: bool) -> list[object]: + mode = "trees" if trees_only else route.mode + candidates = list(runtime.discover_projects(mode=mode)) + if not candidates and not trees_only and route.mode == "main": + return list(runtime.discover_projects(mode="trees")) + return candidates + + +def _all_action_candidates(candidates: list[object]) -> tuple[list[object], str | None]: + if not candidates: + return [], "No projects discovered for selected mode." + return candidates, None + + +@dataclass(frozen=True) +class _ActionTargetRequest: + run_all: bool + untested_selected: bool + project_selectors: frozenset[str] + explicit_targeting: bool + unresolved_service_selectors: tuple[str, ...] + + +@dataclass +class _ActionTargetResolver: + runtime: Any + route: Route + trees_only: bool + resolve_current_worktree_target: Callable[..., object | None] + interactive_selection_allowed: Callable[[Route], bool] + no_target_selected_message: Callable[[Route], str] + + def resolve(self) -> tuple[list[object], str | None]: + request = self._request_from_route() + if current_result := self._current_worktree_result(request): + return current_result + candidates = _discover_action_candidates(self.runtime, self.route, trees_only=self.trees_only) + if request.run_all: + return _all_action_candidates(candidates) + if request.project_selectors: + return self._match_request(candidates, request) + return self._resolve_without_project_selectors(candidates, request) + + def _resolve_without_project_selectors( + self, + candidates: list[object], + request: _ActionTargetRequest, + ) -> tuple[list[object], str | None]: + if len(candidates) == 1 and not request.explicit_targeting: + return candidates, None + if not candidates and not request.explicit_targeting: + return _all_action_candidates(candidates) + if request.explicit_targeting: + return self._match_request(candidates, request) + interactive_request, error = self._request_from_interactive_selection(candidates) + if error is not None: + return [], error + return self._match_request(candidates, interactive_request) + + def _current_worktree_result( + self, + request: _ActionTargetRequest, + ) -> tuple[list[object], str | None] | None: + if self.trees_only and self.route.command == "self-destruct-worktree": + current = self.resolve_current_worktree_target() + if current is not None: + return [current], None + return [], "self-destruct-worktree must be run from inside a discovered worktree." + current_worktree_commands = {"commit", "pr", "ship", "test", "test-focused"} + if self.trees_only or self.route.command not in current_worktree_commands or request.explicit_targeting: + return None + current = self.resolve_current_worktree_target( + require_configured_main_root=self.route.mode == "main", + require_configured_root_match=True, + ) + return ([current], None) if current is not None else None + + def _request_from_route(self) -> _ActionTargetRequest: + project_selectors = {name.lower() for name in self.route.projects} + project_selectors.update(self.runtime.selectors_from_passthrough(self.route.passthrough_args)) + services = self.route.flags.get("services") + has_service_selectors = isinstance(services, list) and bool(services) + unresolved_service_selectors: tuple[str, ...] = () + if isinstance(services, list): + service_projects, unresolved_service_selectors = _resolve_projects_for_services( + self.runtime, + services, + ) + project_selectors.update(project.lower() for project in service_projects) + run_all = bool(self.route.flags.get("all")) + untested_selected = bool(self.route.flags.get("untested")) + return _ActionTargetRequest( + run_all=run_all, + untested_selected=untested_selected, + project_selectors=frozenset(project_selectors), + explicit_targeting=bool(project_selectors or has_service_selectors or run_all or untested_selected), + unresolved_service_selectors=unresolved_service_selectors, + ) + + def _request_from_interactive_selection( + self, + candidates: list[object], + ) -> tuple[_ActionTargetRequest, str | None]: + if not self.interactive_selection_allowed(self.route): + return self._request_from_route(), self.no_target_selected_message(self.route) + selection = cast( + TargetSelection, + self.runtime.select_project_targets( + prompt=f"Select {self.route.command} target", + projects=candidates, + allow_all=True, + allow_untested=self.route.command == "test", + multi=True, + ), + ) + if selection.cancelled: + return self._request_from_route(), self.no_target_selected_message(self.route) + selection.apply_to_route(self.route) + return self._request_from_route(), None + + def _match_request( + self, + candidates: list[object], + request: _ActionTargetRequest, + ) -> tuple[list[object], str | None]: + if request.run_all: + return _all_action_candidates(candidates) + if request.unresolved_service_selectors: + requested = ", ".join(request.unresolved_service_selectors) + return [], f"No matching targets found for: {requested}" + if not request.project_selectors: + return self._empty_request_result(request) + selected = [ + candidate + for candidate in candidates + if candidate.name.lower() in request.project_selectors + ] + selected_names = {candidate.name.lower() for candidate in selected} + missing_selectors = request.project_selectors - selected_names + if missing_selectors: + requested = ", ".join(sorted(missing_selectors)) + return [], f"No matching targets found for: {requested}" + return selected, None + + def _empty_request_result(self, request: _ActionTargetRequest) -> tuple[list[object], str | None]: + if self.route.command == "test" and request.untested_selected: + return [], None + return [], self.no_target_selected_message(self.route) def resolve_action_targets( @@ -147,77 +322,129 @@ def resolve_action_targets( interactive_selection_allowed: Callable[[Route], bool], no_target_selected_message: Callable[[Route], str], ) -> tuple[list[object], str | None]: - if trees_only and route.command == "self-destruct-worktree": - current = resolve_current_worktree_target() - if current is not None: - return [current], None - return [], "self-destruct-worktree must be run from inside a discovered worktree." - current_worktree_commands = {"commit", "pr", "ship", "test", "test-focused"} - if not trees_only and route.command in current_worktree_commands: - current = resolve_current_worktree_target( - require_configured_main_root=route.mode == "main", - require_configured_root_match=True, + return _ActionTargetResolver( + runtime=runtime, + route=route, + trees_only=trees_only, + resolve_current_worktree_target=resolve_current_worktree_target, + interactive_selection_allowed=interactive_selection_allowed, + no_target_selected_message=no_target_selected_message, + ).resolve() + + +def _exception_message(exc: Exception) -> str: + detail = str(exc).strip() + return f"{type(exc).__name__}: {detail}" if detail else type(exc).__name__ + + +def _completed_process_error(completed: Any) -> str | None: + if completed.returncode == 0: + return None + stdout = str(getattr(completed, "stdout", "") or "").strip() + stderr = str(getattr(completed, "stderr", "") or "").strip() + if stderr and stdout and stderr != stdout: + return f"{stderr}\n\nstdout:\n{stdout}" + return stderr or stdout or f"exit:{completed.returncode}" + + +def _run_action_target( + context: ActionTargetContext, + *, + resolve_command: Callable[[ActionTargetContext], ActionCommandResolution], + build_env: Callable[[ActionTargetContext], Mapping[str, str]], + process_run: Callable[[list[str], Path, Mapping[str, str]], Any], +) -> tuple[Any | None, str | None]: + try: + resolution = resolve_command(context) + if resolution.command is None or resolution.cwd is None: + return None, resolution.error or "no command resolved" + completed = process_run(resolution.command, resolution.cwd, build_env(context)) + return completed, _completed_process_error(completed) + except Exception as exc: # Keep independent targets running after one operational failure. + return None, _exception_message(exc) + + +def _report_action_failure( + context: ActionTargetContext, + error: str, + *, + command_name: str, + interactive_command: bool, + should_print: bool, + emit_status: Callable[[str], None], + printer: Callable[[str], None], + on_failure: Callable[[ActionTargetContext, str], None] | None, + failure_status_formatter: Callable[[ActionTargetContext, str], str] | None, +) -> None: + if on_failure is not None: + on_failure(context, error) + if should_print: + printer(f"{command_name} action failed for {context.name}: {error}") + if interactive_command: + status = ( + failure_status_formatter(context, error) + if failure_status_formatter is not None + else f"{command_name} failed for {context.name}: {error}" ) - if current is not None: - return [current], None - if trees_only: - candidates = runtime.discover_projects(mode="trees") - else: - candidates = runtime.discover_projects(mode=route.mode) - if not candidates and route.mode == "main": - candidates = runtime.discover_projects(mode="trees") - - run_all = bool(route.flags.get("all")) - untested_selected = bool(route.flags.get("untested")) - project_selectors = {name.lower() for name in route.projects} - project_selectors.update(runtime.selectors_from_passthrough(route.passthrough_args)) - - services = route.flags.get("services") - if isinstance(services, list): - for project in projects_for_services(runtime, services): - project_selectors.add(project.lower()) - - if run_all: - if not candidates: - return [], "No projects discovered for selected mode." - return candidates, None - - if not project_selectors and not run_all: - if len(candidates) == 1: - return candidates, None - if interactive_selection_allowed(route): - selection = cast( - TargetSelection, - runtime.select_project_targets( - prompt=f"Select {route.command} target", - projects=candidates, - allow_all=True, - allow_untested=route.command == "test", - multi=True, - ), - ) - if selection.cancelled: - return [], no_target_selected_message(route) - selection.apply_to_route(route) - run_all = bool(route.flags.get("all")) - project_selectors = {name.lower() for name in route.projects} - untested_selected = bool(route.flags.get("untested")) - else: - return [], no_target_selected_message(route) - - if not project_selectors and not run_all and not untested_selected: - return [], no_target_selected_message(route) - if run_all: - if not candidates: - return [], "No projects discovered for selected mode." - return candidates, None - selected = [candidate for candidate in candidates if candidate.name.lower() in project_selectors] - if not selected: - if route.command == "test" and untested_selected: - return [], None - requested = ", ".join(sorted(project_selectors)) - return [], f"No matching targets found for: {requested}" - return selected, None + emit_status(status) + + +def _report_action_success( + context: ActionTargetContext, + completed: Any, + *, + command_name: str, + interactive_command: bool, + emit_success_output: bool, + print_noninteractive_successes: bool, + emit_status: Callable[[str], None], + printer: Callable[[str], None], + on_success: Callable[[ActionTargetContext, Any], None] | None, + success_print_formatter: Callable[[ActionTargetContext, Any], str] | None, + success_status_formatter: Callable[[ActionTargetContext, Any], str] | None, +) -> None: + if emit_success_output: + emit_action_output(completed.stdout, emit_status=emit_status, printer=printer) + if on_success is not None: + on_success(context, completed) + if not interactive_command and print_noninteractive_successes: + message = ( + success_print_formatter(context, completed) + if success_print_formatter is not None + else f"{command_name} action succeeded for {context.name}." + ) + if message: + printer(message) + status = ( + success_status_formatter(context, completed) + if success_status_formatter is not None + else f"{command_name} succeeded for {context.name}" + ) + if status: + emit_status(status) + + +def _prepare_action_contexts( + targets: Sequence[object], + *, + command_name: str, + emit_status: Callable[[str], None], + printer: Callable[[str], None], +) -> tuple[list[ActionTargetContext], int]: + contexts = build_action_target_contexts(targets) + invalid_target_count = len(targets) - len(contexts) + if invalid_target_count: + message = ( + f"{command_name} action skipped {invalid_target_count} invalid target(s) " + "without both a name and root." + ) + printer(message) + emit_status(message) + elif not contexts: + message = f"{command_name} action failed: no targets were provided." + printer(message) + emit_status(message) + return contexts, invalid_target_count def execute_targeted_action( @@ -240,67 +467,53 @@ def execute_targeted_action( print_noninteractive_failures: bool = True, print_noninteractive_successes: bool = True, ) -> int: - def should_print_failure() -> bool: - return (not interactive_command and print_noninteractive_failures) or ( - interactive_command and interactive_print_failures - ) - - failures = 0 - for context in build_action_target_contexts(targets): + contexts, failures = _prepare_action_contexts( + targets, + command_name=command_name, + emit_status=emit_status, + printer=printer, + ) + if not contexts: + return 1 + + should_print_failure = (not interactive_command and print_noninteractive_failures) or ( + interactive_command and interactive_print_failures + ) + for context in contexts: emit_status(f"Running {command_name} for {context.name} ({context.index}/{context.total})...") - resolution = resolve_command(context) - if resolution.command is None or resolution.cwd is None: - raw_error = resolution.error or "no command resolved" - message = f"{command_name} action failed for {context.name}: {raw_error}" - if on_failure is not None: - on_failure(context, raw_error) - if should_print_failure(): - printer(message) - if interactive_command: - if failure_status_formatter is not None: - emit_status(failure_status_formatter(context, raw_error)) - else: - emit_status(f"{command_name} failed for {context.name}: {raw_error}") - failures += 1 - continue - - completed = process_run(resolution.command, resolution.cwd, build_env(context)) - if completed.returncode != 0: - stdout = str(getattr(completed, "stdout", "") or "").strip() - stderr = str(getattr(completed, "stderr", "") or "").strip() - if stderr and stdout and stderr != stdout: - error = f"{stderr}\n\nstdout:\n{stdout}" - else: - error = stderr or stdout or f"exit:{completed.returncode}" - message = f"{command_name} action failed for {context.name}: {error}" - if on_failure is not None: - on_failure(context, error) - if should_print_failure(): - printer(message) - if interactive_command: - if failure_status_formatter is not None: - emit_status(failure_status_formatter(context, error)) - else: - emit_status(f"{command_name} failed for {context.name}: {error}") + completed, error = _run_action_target( + context, + resolve_command=resolve_command, + build_env=build_env, + process_run=process_run, + ) + if error is not None: + _report_action_failure( + context, + error, + command_name=command_name, + interactive_command=interactive_command, + should_print=should_print_failure, + emit_status=emit_status, + printer=printer, + on_failure=on_failure, + failure_status_formatter=failure_status_formatter, + ) failures += 1 continue - - if emit_success_output: - emit_action_output(completed.stdout, emit_status=emit_status, printer=printer) - if on_success is not None: - on_success(context, completed) - if not interactive_command and print_noninteractive_successes: - if success_print_formatter is not None: - message = success_print_formatter(context, completed) - else: - message = f"{command_name} action succeeded for {context.name}." - if message: - printer(message) - if success_status_formatter is not None: - status_message = success_status_formatter(context, completed) - else: - status_message = f"{command_name} succeeded for {context.name}" - if status_message: - emit_status(status_message) + assert completed is not None + _report_action_success( + context, + completed, + command_name=command_name, + interactive_command=interactive_command, + emit_success_output=emit_success_output, + print_noninteractive_successes=print_noninteractive_successes, + emit_status=emit_status, + printer=printer, + on_success=on_success, + success_print_formatter=success_print_formatter, + success_status_formatter=success_status_formatter, + ) return 0 if failures == 0 else 1 diff --git a/tests/python/actions/test_action_target_support.py b/tests/python/actions/test_action_target_support.py index 40a8b4463..e7f6c63f7 100644 --- a/tests/python/actions/test_action_target_support.py +++ b/tests/python/actions/test_action_target_support.py @@ -14,6 +14,7 @@ action_target_identities, action_target_names, action_target_names_with_roots, + build_action_target_contexts, execute_targeted_action, emit_action_output, projects_for_services, @@ -50,6 +51,17 @@ def test_action_target_identities_normalize_name_root_and_skip_invalid_targets(s self.assertEqual(action_target_names(targets), ["feature-a-1", "missing-root"]) self.assertEqual(action_target_names_with_roots(targets), ["feature-a-1"]) + def test_build_action_target_contexts_reports_progress_over_valid_targets_only(self) -> None: + contexts = build_action_target_contexts( + [ + _Target(name="feature-a-1", root="/repo/trees/feature-a/1"), + _Target(name="", root="/repo/trees/invalid/1"), + _Target(name="feature-b-1", root="/repo/trees/feature-b/1"), + ] + ) + + self.assertEqual([(context.index, context.total) for context in contexts], [(1, 2), (2, 2)]) + def test_projects_for_services_resolves_from_state_and_deduplicates(self) -> None: state = SimpleNamespace( services={ @@ -67,6 +79,29 @@ def test_projects_for_services_resolves_from_state_and_deduplicates(self) -> Non self.assertEqual(resolved, ["feature-a-1", "feature-b-1"]) + def test_projects_for_services_resolves_all_projects_for_shared_service_type_once(self) -> None: + state = SimpleNamespace( + services={ + "feature-a-1 Worker": SimpleNamespace(type="worker"), + "feature-b-1 Worker": SimpleNamespace(type="worker"), + } + ) + mapped: list[str] = [] + + def project_name_from_service(name: str) -> str: + mapped.append(name) + return name.split()[0] if name.startswith("feature-") else "" + + runtime = SimpleNamespace( + load_existing_state=lambda mode: state if mode == "trees" else None, + project_name_from_service=project_name_from_service, + ) + + resolved = projects_for_services(runtime, ["worker", "service:worker"]) + + self.assertEqual(resolved, ["feature-a-1", "feature-b-1"]) + self.assertEqual(mapped, ["feature-a-1 Worker", "feature-b-1 Worker"]) + def test_resolve_action_targets_uses_single_candidate_without_prompt(self) -> None: target = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") runtime = SimpleNamespace( @@ -110,6 +145,103 @@ def test_resolve_action_targets_prefers_current_worktree_for_tree_mode_action_co self.assertIsNone(error) self.assertEqual(calls, [{"require_configured_main_root": False, "require_configured_root_match": True}]) + def test_resolve_action_targets_explicit_project_overrides_current_worktree(self) -> None: + current = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") + requested = _Target(name="feature-b-1", root="/tmp/repo/trees/feature-b/1") + runtime = SimpleNamespace( + discover_projects=lambda mode: [current, requested] if mode == "trees" else [], + selectors_from_passthrough=lambda _args: [], + ) + route = parse_route( + ["test", "--project", "feature-b-1"], + env={"ENVCTL_DEFAULT_MODE": "trees"}, + ) + + selected, error = resolve_action_targets( + runtime=runtime, + route=route, + trees_only=False, + resolve_current_worktree_target=lambda **_kwargs: current, + interactive_selection_allowed=lambda _route: False, + no_target_selected_message=lambda _route: "no target", + ) + + self.assertEqual(selected, [requested]) + self.assertIsNone(error) + + def test_resolve_action_targets_unknown_service_does_not_fall_through_to_single_project(self) -> None: + target = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") + runtime = SimpleNamespace( + discover_projects=lambda mode: [target] if mode == "trees" else [], + selectors_from_passthrough=lambda _args: [], + load_existing_state=lambda mode: None, + project_name_from_service=lambda _name: "", + ) + route = parse_route( + ["test", "--service", "missing-worker"], + env={"ENVCTL_DEFAULT_MODE": "trees"}, + ) + + selected, error = resolve_action_targets( + runtime=runtime, + route=route, + trees_only=False, + resolve_current_worktree_target=lambda **_kwargs: target, + interactive_selection_allowed=lambda _route: False, + no_target_selected_message=lambda _route: "no target", + ) + + self.assertEqual(selected, []) + self.assertEqual(error, "No matching targets found for: missing-worker") + + def test_resolve_action_targets_rejects_partial_service_match(self) -> None: + target = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") + state = SimpleNamespace( + services={"feature-a-1 Worker": SimpleNamespace(type="worker")} + ) + runtime = SimpleNamespace( + discover_projects=lambda mode: [target] if mode == "trees" else [], + selectors_from_passthrough=lambda _args: [], + load_existing_state=lambda mode: state if mode == "trees" else None, + project_name_from_service=lambda name: name.split()[0] if name.startswith("feature-") else "", + ) + route = parse_route( + ["test", "--service", "worker", "--service", "misspelled"], + env={"ENVCTL_DEFAULT_MODE": "trees"}, + ) + + selected, error = resolve_action_targets( + runtime=runtime, + route=route, + trees_only=False, + resolve_current_worktree_target=lambda **_kwargs: target, + interactive_selection_allowed=lambda _route: False, + no_target_selected_message=lambda _route: "no target", + ) + + self.assertEqual(selected, []) + self.assertEqual(error, "No matching targets found for: misspelled") + + def test_resolve_action_targets_untested_does_not_auto_select_single_project(self) -> None: + target = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") + runtime = SimpleNamespace( + discover_projects=lambda mode: [target] if mode == "trees" else [], + selectors_from_passthrough=lambda _args: [], + ) + route = parse_route(["test", "--untested"], env={"ENVCTL_DEFAULT_MODE": "trees"}) + + selected, error = resolve_action_targets( + runtime=runtime, + route=route, + trees_only=False, + resolve_current_worktree_target=lambda **_kwargs: target, + interactive_selection_allowed=lambda _route: False, + no_target_selected_message=lambda _route: "no target", + ) + + self.assertEqual(selected, []) + self.assertIsNone(error) + def test_resolve_action_targets_applies_interactive_selection(self) -> None: target_a = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") target_b = _Target(name="feature-b-1", root="/tmp/repo/trees/feature-b/1") @@ -142,6 +274,29 @@ def apply_to_route(self, route): # noqa: ANN001 self.assertEqual(calls[0]["prompt"], "Select test target") self.assertTrue(calls[0]["allow_untested"]) + def test_resolve_action_targets_rejects_partial_explicit_match(self) -> None: + target = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") + runtime = SimpleNamespace( + discover_projects=lambda mode: [target] if mode == "trees" else [], + selectors_from_passthrough=lambda _args: [], + ) + route = parse_route( + ["test", "--project", "feature-a-1", "--project", "misspelled"], + env={"ENVCTL_DEFAULT_MODE": "trees"}, + ) + + selected, error = resolve_action_targets( + runtime=runtime, + route=route, + trees_only=False, + resolve_current_worktree_target=lambda **_kwargs: None, + interactive_selection_allowed=lambda _route: False, + no_target_selected_message=lambda _route: "no target", + ) + + self.assertEqual(selected, []) + self.assertEqual(error, "No matching targets found for: misspelled") + def test_emit_action_output_trims_and_emits_status(self) -> None: printed: list[str] = [] emitted: list[str] = [] @@ -363,6 +518,77 @@ def test_execute_targeted_action_can_replace_generic_success_messages(self) -> N self.assertNotIn("ship action succeeded for Main.", printed) self.assertNotIn("ship succeeded for Main", emitted) + def test_execute_targeted_action_counts_invalid_targets_and_keeps_valid_progress_accurate(self) -> None: + printed: list[str] = [] + emitted: list[str] = [] + + code = execute_targeted_action( + targets=[ + _Target(name="", root="/tmp/invalid"), + _Target(name="feature-a-1", root="/tmp/feature-a-1"), + ], + command_name="test", + interactive_command=False, + resolve_command=lambda context: ActionCommandResolution(command=["true"], cwd=context.root), + build_env=lambda _context: {}, + process_run=lambda _command, _cwd, _env: _Completed(returncode=0), + emit_status=emitted.append, + printer=printed.append, + ) + + self.assertEqual(code, 1) + self.assertIn("test action skipped 1 invalid target(s) without both a name and root.", printed) + self.assertIn("Running test for feature-a-1 (1/1)...", emitted) + self.assertIn("test action succeeded for feature-a-1.", printed) + + def test_execute_targeted_action_contains_callback_exception_and_continues_other_targets(self) -> None: + targets = [ + _Target(name="feature-a-1", root="/tmp/feature-a-1"), + _Target(name="feature-b-1", root="/tmp/feature-b-1"), + ] + processed: list[str] = [] + failures: list[tuple[str, str]] = [] + + def build_env(context): # noqa: ANN001,ANN202 + if context.name == "feature-a-1": + raise RuntimeError("broken environment") + return {} + + code = execute_targeted_action( + targets=targets, + command_name="test", + interactive_command=False, + resolve_command=lambda context: ActionCommandResolution(command=["true"], cwd=context.root), + build_env=build_env, + process_run=lambda _command, cwd, _env: processed.append(cwd.name) or _Completed(returncode=0), + emit_status=lambda _message: None, + printer=lambda _message: None, + on_failure=lambda context, error: failures.append((context.name, error)), + ) + + self.assertEqual(code, 1) + self.assertEqual(failures, [("feature-a-1", "RuntimeError: broken environment")]) + self.assertEqual(processed, ["feature-b-1"]) + + def test_execute_targeted_action_rejects_empty_target_list(self) -> None: + printed: list[str] = [] + emitted: list[str] = [] + + code = execute_targeted_action( + targets=[], + command_name="review", + interactive_command=False, + resolve_command=lambda _context: ActionCommandResolution(command=["true"], cwd=Path("/tmp")), + build_env=lambda _context: {}, + process_run=lambda _command, _cwd, _env: _Completed(returncode=0), + emit_status=emitted.append, + printer=printed.append, + ) + + self.assertEqual(code, 1) + self.assertEqual(printed, ["review action failed: no targets were provided."]) + self.assertEqual(emitted, printed) + if __name__ == "__main__": unittest.main() From 9e797788701d12917b4a7a4f97faf9a511691782 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 01:48:27 +0300 Subject: [PATCH 07/26] Remove plan-agent workflow compatibility hub --- python/envctl_engine/planning/menu.py | 2 +- .../planning/plan_agent/cmux_transport.py | 6 +- .../plan_agent/superset_desktop_support.py | 2 +- .../planning/plan_agent/superset_transport.py | 4 +- .../plan_agent/tmux_identity_support.py | 2 +- .../planning/plan_agent/tmux_transport.py | 2 +- .../planning/plan_agent/workflow.py | 222 ------------------ .../plan_agent/workflow_runtime_support.py | 68 ++++++ .../planning/plan_agent_launch_support.py | 2 +- .../ui/dashboard/ai_session_rendering.py | 4 +- .../plan_agent_launch_support_test_support.py | 16 +- .../test_plan_agent_launch_cmux_goal.py | 4 +- .../test_plan_agent_launch_cmux_review.py | 10 +- .../test_plan_agent_launch_omx_workflow.py | 5 +- .../test_plan_agent_launch_prompt_options.py | 2 +- .../test_plan_agent_launch_workflow_prompt.py | 28 +-- .../planning/test_plan_agent_module_layout.py | 7 +- .../planning/test_planning_menu_rendering.py | 1 + .../runtime/test_runtime_context_protocols.py | 1 - tests/python/shared/test_import_audit.py | 9 +- 20 files changed, 125 insertions(+), 272 deletions(-) delete mode 100644 python/envctl_engine/planning/plan_agent/workflow.py create mode 100644 python/envctl_engine/planning/plan_agent/workflow_runtime_support.py diff --git a/python/envctl_engine/planning/menu.py b/python/envctl_engine/planning/menu.py index eb61569d7..e173a41d8 100644 --- a/python/envctl_engine/planning/menu.py +++ b/python/envctl_engine/planning/menu.py @@ -109,7 +109,7 @@ def _run_legacy( finally: self._flush_input_buffer(fd=fd) self._restore_terminal_state(fd=fd, original_state=original_state) - self._flush_input_buffer(fd=fd) + self.flush_pending_input(fd=fd) sys.stdout.write("\033[0m\033[?25h\n") sys.stdout.flush() diff --git a/python/envctl_engine/planning/plan_agent/cmux_transport.py b/python/envctl_engine/planning/plan_agent/cmux_transport.py index d874b819f..dfd4a7f36 100644 --- a/python/envctl_engine/planning/plan_agent/cmux_transport.py +++ b/python/envctl_engine/planning/plan_agent/cmux_transport.py @@ -25,12 +25,14 @@ _print_launch_summary, _queue_failure_event_context, ) -from envctl_engine.planning.plan_agent.workflow import ( +from envctl_engine.planning.plan_agent.workflow_runtime_support import ( _codex_goal_text_for_worktree, _emit_codex_goal_event, + _surface_respawn_command, +) +from envctl_engine.planning.plan_agent.workflow_review_support import ( _review_original_plan_path, _review_prompt_arguments, - _surface_respawn_command, ) from envctl_engine.planning.plan_agent.workflow_build import ( _build_plan_agent_workflow, diff --git a/python/envctl_engine/planning/plan_agent/superset_desktop_support.py b/python/envctl_engine/planning/plan_agent/superset_desktop_support.py index b1303edb7..f73d94c16 100644 --- a/python/envctl_engine/planning/plan_agent/superset_desktop_support.py +++ b/python/envctl_engine/planning/plan_agent/superset_desktop_support.py @@ -9,7 +9,7 @@ from envctl_engine.planning.plan_agent.models import CreatedPlanWorktree, PlanAgentLaunchConfig, PlanAgentLaunchOutcome from envctl_engine.planning.plan_agent.recovery import _print_launch_summary -from envctl_engine.planning.plan_agent.workflow import _tab_title_for_worktree +from envctl_engine.planning.plan_agent.workflow_build import _tab_title_for_worktree from envctl_engine.runtime.runtime_context import resolve_process_runtime diff --git a/python/envctl_engine/planning/plan_agent/superset_transport.py b/python/envctl_engine/planning/plan_agent/superset_transport.py index a82172a47..094b24ac7 100644 --- a/python/envctl_engine/planning/plan_agent/superset_transport.py +++ b/python/envctl_engine/planning/plan_agent/superset_transport.py @@ -31,11 +31,11 @@ write_superset_codex_goal_launcher, ) import envctl_engine.planning.plan_agent.superset_worktree_launch_support as superset_worktree_launch_support -from envctl_engine.planning.plan_agent.workflow import ( +from envctl_engine.planning.plan_agent.workflow_runtime_support import ( _codex_goal_text_for_worktree, _emit_codex_goal_event, - _workflow_step_prompt_text, ) +from envctl_engine.planning.plan_agent.workflow_prompt_support import _workflow_step_prompt_text _ensure_superset_codex_goal_agent = ensure_superset_codex_goal_agent diff --git a/python/envctl_engine/planning/plan_agent/tmux_identity_support.py b/python/envctl_engine/planning/plan_agent/tmux_identity_support.py index ae2a7da99..34a413d6d 100644 --- a/python/envctl_engine/planning/plan_agent/tmux_identity_support.py +++ b/python/envctl_engine/planning/plan_agent/tmux_identity_support.py @@ -9,7 +9,7 @@ ) from envctl_engine.planning.plan_agent.models import CreatedPlanWorktree -from envctl_engine.planning.plan_agent.workflow import _tab_title_for_worktree +from envctl_engine.planning.plan_agent.workflow_build import _tab_title_for_worktree def tmux_session_name_for_worktree(repo_root: Path, worktree: CreatedPlanWorktree, *, cli: str) -> str: diff --git a/python/envctl_engine/planning/plan_agent/tmux_transport.py b/python/envctl_engine/planning/plan_agent/tmux_transport.py index fde1eea4a..ecdfc7cbc 100644 --- a/python/envctl_engine/planning/plan_agent/tmux_transport.py +++ b/python/envctl_engine/planning/plan_agent/tmux_transport.py @@ -44,7 +44,7 @@ _prompt_existing_tmux_session_action, _should_prompt_existing_tmux_session, ) -from envctl_engine.planning.plan_agent.workflow import ( +from envctl_engine.planning.plan_agent.workflow_runtime_support import ( _codex_goal_text_for_worktree, _emit_codex_goal_event, _wrap_omx_initial_prompt_for_workflow, diff --git a/python/envctl_engine/planning/plan_agent/workflow.py b/python/envctl_engine/planning/plan_agent/workflow.py deleted file mode 100644 index 846f064d3..000000000 --- a/python/envctl_engine/planning/plan_agent/workflow.py +++ /dev/null @@ -1,222 +0,0 @@ -from __future__ import annotations - -# ruff: noqa: F401 -from typing import Any - -from envctl_engine.planning.plan_agent.constants import ( - _AI_CLI_SHELL_FAILURE_MARKERS, - _ANSI_ESCAPE_RE, - _BROWSER_E2E_FOLLOWUP_TEMPLATE, - _CLI_READY_DELAY_SECONDS_BY_CLI, - _CLI_READY_POLL_INTERVAL_SECONDS, - _CODEX_BYPASS_FLAGS, - _CODEX_LOADING_MARKERS, - _CODEX_QUEUE_CONFIRMED_MARKERS, - _CODEX_QUEUE_MAX_TAB_ATTEMPTS, - _CODEX_QUEUE_READY_HINT, - _CODEX_QUEUE_READY_POLL_INTERVAL_SECONDS, - _CODEX_QUEUE_READY_TIMEOUT_SECONDS, - _CODEX_READY_MARKERS, - _CODEX_READY_PROMPT_RE, - _DEFAULT_CLI_READY_DELAY_SECONDS, - _DEFAULT_PRESET, - _DEFAULT_SHELL, - _DONE_PLANNING_ROOT, - _FINALIZATION_INSTRUCTION_TEMPLATE, - _FIRST_CYCLE_COMPLETION_TEMPLATE, - _INTERMEDIATE_CYCLE_COMPLETION_TEMPLATE, - _LOW_SIGNAL_TAB_TITLE_WORDS, - _OMX_SESSION_READY_POLL_INTERVAL_SECONDS, - _OMX_SESSION_READY_TIMEOUT_SECONDS, - _OMX_SESSION_STATE_RELATIVE_PATH, - _OMX_SPAWN_OUTPUT_EXCERPT_CHARS, - _OMX_TMUX_EXTENDED_KEYS_RELATIVE_PATH, - _OMX_TMUX_LOCK_STALE_SECONDS, - _OMX_WORKFLOW_KEYWORDS, - _OPENCODE_LOADING_MARKERS, - _OPENCODE_READY_MARKERS, - _PLAN_AGENT_CODEX_CYCLE_CAP, - _PLAN_AGENT_TAB_TITLE_MAX_LEN, - _PLAN_AGENT_WORKFLOW_CODEX_CYCLES, - _PLAN_AGENT_WORKFLOW_SINGLE_PROMPT, - _PLANNING_ROOT, - _PR_REVIEW_COMMENTS_FOLLOWUP_TEMPLATE, - _PROMPT_PRE_SUBMIT_DELAY_SECONDS, - _PROMPT_SHAPING_COMMAND_TOKEN_RE, - _PROMPT_SUBMIT_READY_DELAY_SECONDS, - _PROMPT_SUBMIT_READY_POLL_INTERVAL_SECONDS, - _PROMPT_SUBMIT_READY_TIMEOUT_SECONDS, - _PROMPT_TEMPLATE_PACKAGE, - _READ_SCREEN_LINE_COUNT, - _REVIEW_WORKTREE_PRESET, - _SUPPORTED_PLAN_AGENT_CLIS, - _SURFACE_READY_DELAY_SECONDS, - _TMUX_WINDOW_READY_POLL_INTERVAL_SECONDS, - _TMUX_WINDOW_READY_TIMEOUT_SECONDS, - _WORKTREE_PROVENANCE_PATH, -) -from envctl_engine.planning.plan_agent.config import ( - PlanAgentLaunchPolicy, - _cli_ready_delay_seconds, - _codex_tui_queue_workflow_supported, - _guidance_attach_command, - _launch_policy, - _missing_launch_commands, - _route_requests_ulw, - _ulw_route_supported, - _uses_direct_submission, - cli_ready_delay_seconds, - codex_tui_queue_workflow_supported, - command_executable, - default_plan_agent_cli_command, - default_plan_agent_surface_transport, - guidance_attach_command, - missing_launch_commands, - normalize_plan_agent_surface_transport, - parse_codex_cycles, - plan_agent_launch_prereq_commands, - resolve_plan_agent_launch_config, - uses_direct_submission, -) -from envctl_engine.planning.plan_agent.models import ( - AgentTerminalLaunchResult, - AiCliReadyResult, - CreatedPlanWorktree, - PlanAgentAttachTarget, - PlanAgentAttachValidation, - PlanAgentLaunchConfig, - PlanAgentLaunchOutcome, - PlanAgentLaunchResult, - PlanSelectionResult, - PlanWorktreeSyncResult, - ReviewAgentLaunchReadiness, - _OmxSessionRecord, - _OmxSpawnProcessRecord, - _PlanAgentWorkflow, - _PlanAgentWorkflowStep, - _QueueFailure, - _WorkspaceLaunchTarget, -) -from envctl_engine.planning.plan_agent.terminal_screen import ( - _CODEX_GOAL_ACTIVE_MARKERS, - _codex_goal_screen_looks_active, - _codex_queue_message_needs_tab, - _codex_queue_screen_confirms_queued, - _codex_queue_screen_looks_ready, - _codex_queue_text_is_visible, - _format_ai_cli_ready_failure, - _normalized_screen_text, - _post_submit_screen_looks_accepted, - _prompt_picker_screen_looks_ready, - _prompt_submit_screen_looks_ready, - _screen_excerpt, - _screen_looks_active, - _screen_looks_ready, - _screen_tail_text, - _strip_ansi_sequences, -) -from envctl_engine.planning.plan_agent.workflow_build import ( - PlanAgentWorkflowBuilder, - _browser_e2e_instruction_text, - _build_plan_agent_workflow, - _load_plan_agent_followup_prompt, - _pr_review_comments_instruction_text, - _slash_command, - _tab_title_for_worktree, -) -from envctl_engine.planning.plan_agent.workflow_prompt_support import ( - _append_runtime_addresses_for_preset, - _component_port, - _dependency_address, - _dependency_address_lines, - _dependency_label, - _int_or_none, - _latest_runtime_state, - _original_plan_file_path, - _original_task_source_prompt_section, - _resolve_preset_submission_text, - _runtime_addresses_prompt_section, - _service_address_lines, - _shape_prompt_text, - _shape_queue_message_text, - _state_project_matches_worktree, - _state_service_matches_worktree, - _workflow_step_prompt_text, - optional_state_repository, - resolve_codex_direct_prompt_body, - resolve_opencode_direct_prompt_body, -) -from envctl_engine.planning.plan_agent.workflow_review_support import ( - _active_plan_selector_for_path, - _cli_display_path, - _feature_name_from_project_name, - _infer_plan_file_from_feature, - _plan_matches_for_feature, - _recorded_plan_file_from_worktree, - _resolve_recorded_plan_file, - _review_original_plan_path, - _review_prompt_arguments, - resolve_plan_agent_launch_command, -) -import envctl_engine.runtime.runtime_context as runtime_context - - -def _surface_respawn_command(launch_config: PlanAgentLaunchConfig, worktree: CreatedPlanWorktree) -> str: - _ = worktree - return launch_config.shell - - -def _wrap_omx_initial_prompt_for_workflow(text: str, *, workflow: str) -> str: - normalized_workflow = str(workflow or "").strip().lower() - if normalized_workflow not in _OMX_WORKFLOW_KEYWORDS: - return text - stripped = str(text).lstrip() - prefix = f"${normalized_workflow}" - if stripped == prefix or stripped.startswith(f"{prefix} ") or stripped.startswith(f"{prefix}\n"): - return text - return f"{prefix}\n\n{text}" - - -def _codex_goal_text_for_worktree( - *, - worktree: CreatedPlanWorktree, - preset: str, - workflow_mode: str, - omx_workflow: str, -) -> str: - _ = (worktree, preset, workflow_mode) - lines = [ - "Implement MAIN_TASK.md end-to-end in this worktree.", - "Read it first, update code/tests, run focused validation, then ship with `envctl ship -m \"\"`.", - ] - normalized_omx = str(omx_workflow or "").strip().lower() - if normalized_omx: - lines.append(f"Keep ${normalized_omx} completion contract active.") - return " ".join(lines) - - -def _emit_codex_goal_event( - runtime: Any, - event: str, - *, - cli: str, - workflow: _PlanAgentWorkflow, - transport: str, - worktree: CreatedPlanWorktree, - reason: str | None = None, - **target: object, -) -> None: - payload: dict[str, object] = { - **target, - "worktree": worktree.name, - "cli": cli, - "workflow_mode": workflow.mode, - "codex_cycles": workflow.codex_cycles, - "transport": transport, - } - if reason is not None: - payload["reason"] = reason - runtime._emit(event, **payload) - - -__all__ = tuple(name for name in globals() if not name.startswith("__")) diff --git a/python/envctl_engine/planning/plan_agent/workflow_runtime_support.py b/python/envctl_engine/planning/plan_agent/workflow_runtime_support.py new file mode 100644 index 000000000..3962ee7f2 --- /dev/null +++ b/python/envctl_engine/planning/plan_agent/workflow_runtime_support.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from typing import Any + +from envctl_engine.planning.plan_agent.constants import _OMX_WORKFLOW_KEYWORDS +from envctl_engine.planning.plan_agent.models import ( + CreatedPlanWorktree, + PlanAgentLaunchConfig, + _PlanAgentWorkflow, +) + + +def _surface_respawn_command(launch_config: PlanAgentLaunchConfig, worktree: CreatedPlanWorktree) -> str: + _ = worktree + return launch_config.shell + + +def _wrap_omx_initial_prompt_for_workflow(text: str, *, workflow: str) -> str: + normalized_workflow = str(workflow or "").strip().lower() + if normalized_workflow not in _OMX_WORKFLOW_KEYWORDS: + return text + stripped = str(text).lstrip() + prefix = f"${normalized_workflow}" + if stripped == prefix or stripped.startswith(f"{prefix} ") or stripped.startswith(f"{prefix}\n"): + return text + return f"{prefix}\n\n{text}" + + +def _codex_goal_text_for_worktree( + *, + worktree: CreatedPlanWorktree, + preset: str, + workflow_mode: str, + omx_workflow: str, +) -> str: + _ = (worktree, preset, workflow_mode) + lines = [ + "Implement MAIN_TASK.md end-to-end in this worktree.", + "Read it first, update code/tests, run focused validation, then ship with `envctl ship -m \"\"`.", + ] + normalized_omx = str(omx_workflow or "").strip().lower() + if normalized_omx: + lines.append(f"Keep ${normalized_omx} completion contract active.") + return " ".join(lines) + + +def _emit_codex_goal_event( + runtime: Any, + event: str, + *, + cli: str, + workflow: _PlanAgentWorkflow, + transport: str, + worktree: CreatedPlanWorktree, + reason: str | None = None, + **target: object, +) -> None: + payload: dict[str, object] = { + **target, + "worktree": worktree.name, + "cli": cli, + "workflow_mode": workflow.mode, + "codex_cycles": workflow.codex_cycles, + "transport": transport, + } + if reason is not None: + payload["reason"] = reason + runtime._emit(event, **payload) diff --git a/python/envctl_engine/planning/plan_agent_launch_support.py b/python/envctl_engine/planning/plan_agent_launch_support.py index d6e36cedf..1bf85de21 100644 --- a/python/envctl_engine/planning/plan_agent_launch_support.py +++ b/python/envctl_engine/planning/plan_agent_launch_support.py @@ -28,7 +28,7 @@ from envctl_engine.planning.plan_agent.omx_transport import validate_plan_agent_attach_target from envctl_engine.planning.plan_agent.recovery import plan_agent_native_recovery_command from envctl_engine.planning.plan_agent.tmux_transport import attach_plan_agent_terminal -from envctl_engine.planning.plan_agent.workflow import resolve_plan_agent_launch_command +from envctl_engine.planning.plan_agent.workflow_review_support import resolve_plan_agent_launch_command __all__ = ( "AgentTerminalLaunchResult", diff --git a/python/envctl_engine/ui/dashboard/ai_session_rendering.py b/python/envctl_engine/ui/dashboard/ai_session_rendering.py index f937e4df0..7f058366e 100644 --- a/python/envctl_engine/ui/dashboard/ai_session_rendering.py +++ b/python/envctl_engine/ui/dashboard/ai_session_rendering.py @@ -22,7 +22,9 @@ def print_dashboard_ai_session_row( current_tmux_target_fn: Callable[..., tuple[str, str]], ) -> None: import subprocess # noqa: PLC0415 - from envctl_engine.planning.plan_agent.workflow import resolve_plan_agent_launch_command # noqa: PLC0415 + from envctl_engine.planning.plan_agent.workflow_review_support import ( # noqa: PLC0415 + resolve_plan_agent_launch_command, + ) from envctl_engine.runtime.session_management import list_tmux_sessions # noqa: PLC0415 project_root = project_root_fn(self, state=state, project=project) diff --git a/tests/python/planning/plan_agent_launch_support_test_support.py b/tests/python/planning/plan_agent_launch_support_test_support.py index 0ba55bf58..48ffaaca6 100644 --- a/tests/python/planning/plan_agent_launch_support_test_support.py +++ b/tests/python/planning/plan_agent_launch_support_test_support.py @@ -26,26 +26,28 @@ terminal_screen, tmux_session, tmux_transport, - workflow, workflow_build, + workflow_prompt_support, + workflow_review_support, + workflow_runtime_support, ) -from envctl_engine.planning.plan_agent.models import _WorkspaceLaunchTarget +from envctl_engine.planning.plan_agent.models import _PlanAgentWorkflowStep, _WorkspaceLaunchTarget from envctl_engine.planning.plan_agent_launch_support import CreatedPlanWorktree, launch_plan_agent_terminals from envctl_engine.runtime.command_router import parse_route from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord _screen_looks_ready = terminal_screen._screen_looks_ready _prompt_submit_screen_looks_ready = terminal_screen._prompt_submit_screen_looks_ready -_tab_title_for_worktree = workflow._tab_title_for_worktree -_build_plan_agent_workflow = cast(Any, workflow._build_plan_agent_workflow) +_tab_title_for_worktree = workflow_build._tab_title_for_worktree +_build_plan_agent_workflow = cast(Any, workflow_build._build_plan_agent_workflow) _finalization_instruction_text = cast(Any, workflow_build._finalization_instruction_text) _first_cycle_completion_instruction_text = cast(Any, workflow_build._first_cycle_completion_instruction_text) _intermediate_cycle_completion_instruction_text = cast(Any, workflow_build._intermediate_cycle_completion_instruction_text) -_browser_e2e_instruction_text = cast(Any, workflow._browser_e2e_instruction_text) -_pr_review_comments_instruction_text = cast(Any, workflow._pr_review_comments_instruction_text) +_browser_e2e_instruction_text = cast(Any, workflow_build._browser_e2e_instruction_text) +_pr_review_comments_instruction_text = cast(Any, workflow_build._pr_review_comments_instruction_text) _wait_for_codex_queue_ready = cast(Any, cmux_transport._wait_for_codex_queue_ready) _codex_goal_screen_looks_active = cast(Any, terminal_screen._codex_goal_screen_looks_active) -_workflow_step_prompt_text = cast(Any, workflow._workflow_step_prompt_text) +_workflow_step_prompt_text = cast(Any, workflow_prompt_support._workflow_step_prompt_text) _ensure_tmux_window = cast(Any, tmux_transport._ensure_tmux_window) _tmux_session_name_for_worktree = cast(Any, tmux_transport._tmux_session_name_for_worktree) _run_tmux_worktree_bootstrap = cast(Any, tmux_transport._run_tmux_worktree_bootstrap) diff --git a/tests/python/planning/test_plan_agent_launch_cmux_goal.py b/tests/python/planning/test_plan_agent_launch_cmux_goal.py index 15c24f567..e12741c8d 100644 --- a/tests/python/planning/test_plan_agent_launch_cmux_goal.py +++ b/tests/python/planning/test_plan_agent_launch_cmux_goal.py @@ -7,7 +7,7 @@ class PlanAgentLaunchCmuxGoalTests(PlanAgentLaunchSupportTestCase): def test_codex_goal_text_is_compact_and_main_task_first(self) -> None: - goal = workflow._codex_goal_text_for_worktree( + goal = workflow_runtime_support._codex_goal_text_for_worktree( worktree=CreatedPlanWorktree(name="feature-a-1", root=Path("/repo"), plan_file="features/a.md"), preset="implement_task", workflow_mode="codex_cycles", @@ -24,7 +24,7 @@ def test_codex_goal_text_is_compact_and_main_task_first(self) -> None: self.assertLessEqual(len(goal), 180) def test_codex_goal_text_preserves_omx_completion_contract(self) -> None: - goal = workflow._codex_goal_text_for_worktree( + goal = workflow_runtime_support._codex_goal_text_for_worktree( worktree=CreatedPlanWorktree(name="feature-a-1", root=Path("/repo"), plan_file="features/a.md"), preset="implement_task", workflow_mode="single_prompt", diff --git a/tests/python/planning/test_plan_agent_launch_cmux_review.py b/tests/python/planning/test_plan_agent_launch_cmux_review.py index 9c192b976..e3817f49e 100644 --- a/tests/python/planning/test_plan_agent_launch_cmux_review.py +++ b/tests/python/planning/test_plan_agent_launch_cmux_review.py @@ -168,7 +168,7 @@ def test_review_launch_resolves_original_plan_file_from_worktree_provenance(self json.dumps({"schema_version": 1, "plan_file": "implementations/feature-a.md"}) + "\n", encoding="utf-8", ) - original_plan_path = workflow._review_original_plan_path( + original_plan_path = workflow_review_support._review_original_plan_path( "feature-a-1", project_root, repo_root=repo, @@ -182,7 +182,7 @@ def test_review_launch_returns_none_when_original_plan_file_cannot_be_resolved(s project_root = repo / "trees" / "feature-a" / "1" project_root.mkdir(parents=True, exist_ok=True) - original_plan_path = workflow._review_original_plan_path( + original_plan_path = workflow_review_support._review_original_plan_path( "feature-a-1", project_root, repo_root=repo, @@ -204,7 +204,7 @@ def test_review_launch_does_not_infer_when_recorded_plan_file_is_missing(self) - encoding="utf-8", ) - original_plan_path = workflow._review_original_plan_path( + original_plan_path = workflow_review_support._review_original_plan_path( "implementations_task-1", project_root, repo_root=repo, @@ -221,7 +221,7 @@ def test_review_launch_can_infer_original_plan_file_without_provenance(self) -> original_plan.parent.mkdir(parents=True, exist_ok=True) original_plan.write_text("# done plan\n", encoding="utf-8") - original_plan_path = workflow._review_original_plan_path( + original_plan_path = workflow_review_support._review_original_plan_path( "implementations_task-1", project_root, repo_root=repo, @@ -241,7 +241,7 @@ def test_review_launch_prefers_active_plan_over_archived_copy(self) -> None: active_plan.write_text("# active plan\n", encoding="utf-8") archived_plan.write_text("# archived plan\n", encoding="utf-8") - original_plan_path = workflow._review_original_plan_path( + original_plan_path = workflow_review_support._review_original_plan_path( "features_task", project_root, repo_root=repo, diff --git a/tests/python/planning/test_plan_agent_launch_omx_workflow.py b/tests/python/planning/test_plan_agent_launch_omx_workflow.py index 846d34f17..8d8a5476d 100644 --- a/tests/python/planning/test_plan_agent_launch_omx_workflow.py +++ b/tests/python/planning/test_plan_agent_launch_omx_workflow.py @@ -175,7 +175,10 @@ def test_omx_workflow_launch_does_not_double_wrap_existing_keyword(self) -> None with self.subTest(workflow_name=workflow_name): prompt = f"${workflow_name}\n\nIMPLEMENT TASK BODY" self.assertEqual( - workflow._wrap_omx_initial_prompt_for_workflow(prompt, workflow=workflow_name), + workflow_runtime_support._wrap_omx_initial_prompt_for_workflow( + prompt, + workflow=workflow_name, + ), prompt, ) diff --git a/tests/python/planning/test_plan_agent_launch_prompt_options.py b/tests/python/planning/test_plan_agent_launch_prompt_options.py index c0ca018fb..57b3a2037 100644 --- a/tests/python/planning/test_plan_agent_launch_prompt_options.py +++ b/tests/python/planning/test_plan_agent_launch_prompt_options.py @@ -48,7 +48,7 @@ def test_resolve_plan_agent_launch_config_no_ulw_loop_route_disables_default_pre ) rt = self._runtime(repo, runtime, env={"HOME": tmpdir}) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( rt, launch_config=launch_config, cli="opencode", diff --git a/tests/python/planning/test_plan_agent_launch_workflow_prompt.py b/tests/python/planning/test_plan_agent_launch_workflow_prompt.py index 8637a2148..33e26176e 100644 --- a/tests/python/planning/test_plan_agent_launch_workflow_prompt.py +++ b/tests/python/planning/test_plan_agent_launch_workflow_prompt.py @@ -26,7 +26,7 @@ def test_resolve_preset_submission_text_defaults_ulw_loop_for_explicit_opencode( ) rt = self._runtime(repo, runtime, env={"HOME": tmpdir}) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( rt, launch_config=launch_config, cli="opencode", @@ -48,7 +48,7 @@ def test_workflow_step_prompt_text_uses_codex_skill_command_for_direct_steps(sel env={}, process_runner=_RecordingRunner(), ) - step = workflow._PlanAgentWorkflowStep(kind="submit_direct_prompt", text="implement_task") + step = _PlanAgentWorkflowStep(kind="submit_direct_prompt", text="implement_task") prompt_text, error = _workflow_step_prompt_text( runtime, @@ -89,7 +89,7 @@ def test_workflow_step_prompt_text_keeps_codex_skill_command_single_line_with_wo env={}, process_runner=_RecordingRunner(), ) - step = workflow._PlanAgentWorkflowStep(kind="submit_direct_prompt", text="implement_task") + step = _PlanAgentWorkflowStep(kind="submit_direct_prompt", text="implement_task") worktree = CreatedPlanWorktree(name="feature-a-1", root=worktree_root, plan_file="feature-a.md") prompt_text, error = _workflow_step_prompt_text( @@ -116,7 +116,7 @@ def test_codex_preset_submission_text_collapses_arguments_into_skill_command(sel env={}, process_runner=_RecordingRunner(), ) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( runtime, launch_config=_launch_config_for_tests(cli="codex"), cli="codex", @@ -162,7 +162,7 @@ def test_resolve_preset_submission_text_returns_opencode_direct_body_when_enable ulw_suffix=False, ) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( runtime, launch_config=launch_config, cli="opencode", @@ -199,7 +199,7 @@ def test_resolve_preset_submission_text_keeps_slash_mode_for_opencode_by_default ulw_suffix=False, ) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( runtime, launch_config=launch_config, cli="opencode", @@ -240,7 +240,7 @@ def test_resolve_preset_submission_text_prepends_ulw_loop_for_direct_prompts(sel ulw_suffix=False, ) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( runtime, launch_config=launch_config, cli="opencode", @@ -278,7 +278,7 @@ def test_resolve_preset_submission_text_appends_ulw_suffix_in_slash_mode(self) - ulw_suffix=True, ) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( runtime, launch_config=launch_config, cli="opencode", @@ -315,7 +315,7 @@ def test_resolve_preset_submission_text_rejects_ulw_loop_prefix_in_slash_mode(se ulw_suffix=False, ) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( runtime, launch_config=launch_config, cli="opencode", @@ -326,7 +326,7 @@ def test_resolve_preset_submission_text_rejects_ulw_loop_prefix_in_slash_mode(se self.assertEqual(error, "prompt_resolution_failed: ulw_loop_prefix_requires_direct_prompt") def test_shape_prompt_text_rejects_multiple_slash_commands_for_direct_prompts(self) -> None: - shaped, error = workflow._shape_prompt_text( + shaped, error = workflow_prompt_support._shape_prompt_text( "/implement_task", direct_prompt=True, ulw_loop_prefix=True, @@ -337,7 +337,7 @@ def test_shape_prompt_text_rejects_multiple_slash_commands_for_direct_prompts(se self.assertEqual(error, "prompt_resolution_failed: multiple_slash_commands_not_allowed") def test_shape_prompt_text_rejects_embedded_second_slash_command_for_direct_prompts(self) -> None: - shaped, error = workflow._shape_prompt_text( + shaped, error = workflow_prompt_support._shape_prompt_text( "Implement this directly.\n/implement_task", direct_prompt=True, ulw_loop_prefix=True, @@ -348,7 +348,7 @@ def test_shape_prompt_text_rejects_embedded_second_slash_command_for_direct_prom self.assertEqual(error, "prompt_resolution_failed: multiple_slash_commands_not_allowed") def test_shape_prompt_text_preserves_existing_ulw_loop_prefix_and_allows_suffix(self) -> None: - shaped, error = workflow._shape_prompt_text( + shaped, error = workflow_prompt_support._shape_prompt_text( "/ulw-loop Implement this directly.", direct_prompt=True, ulw_loop_prefix=True, @@ -359,7 +359,7 @@ def test_shape_prompt_text_preserves_existing_ulw_loop_prefix_and_allows_suffix( self.assertEqual(shaped, "/ulw-loop Implement this directly. ulw") def test_shape_prompt_text_allows_absolute_path_literals_in_direct_prompts(self) -> None: - shaped, error = workflow._shape_prompt_text( + shaped, error = workflow_prompt_support._shape_prompt_text( 'Review bundle: "/tmp/review.md"\nWorktree directory: "/tmp/tree"', direct_prompt=True, ulw_loop_prefix=True, @@ -371,7 +371,7 @@ def test_shape_prompt_text_allows_absolute_path_literals_in_direct_prompts(self) self.assertIn('Review bundle: "/tmp/review.md"', shaped) def test_review_prompt_arguments_preserve_newlines_for_direct_codex_submission(self) -> None: - rendered = workflow._review_prompt_arguments( + rendered = workflow_review_support._review_prompt_arguments( project_name="feature-a-1", project_root=Path("/tmp/worktree path"), review_bundle_path=Path("/tmp/review bundle.md"), diff --git a/tests/python/planning/test_plan_agent_module_layout.py b/tests/python/planning/test_plan_agent_module_layout.py index a43b5ddd0..9c4942c18 100644 --- a/tests/python/planning/test_plan_agent_module_layout.py +++ b/tests/python/planning/test_plan_agent_module_layout.py @@ -67,13 +67,13 @@ def test_plan_agent_package_modules_exist(self) -> None: "tmux_worktree_launch_support.py", "tmux_launch_support.py", "tmux_transport.py", - "workflow.py", "workflow_build.py", "workflow_e2e_prompt_context.py", "workflow_prompt_support.py", "workflow_runtime_addresses.py", "workflow_queue_support.py", "workflow_review_support.py", + "workflow_runtime_support.py", } actual = {path.name for path in PLAN_AGENT_ROOT.glob("*.py")} self.assertTrue(expected.issubset(actual)) @@ -235,9 +235,10 @@ def test_core_helpers_live_in_owner_modules(self) -> None: "_state_project_matches_worktree", "_state_service_matches_worktree", }, - "workflow.py": { + "workflow_runtime_support.py": { "_codex_goal_text_for_worktree", "_emit_codex_goal_event", + "_surface_respawn_command", "_wrap_omx_initial_prompt_for_workflow", }, "workflow_queue_support.py": { @@ -466,7 +467,7 @@ def test_no_placeholder_extraction_modules_remain(self) -> None: self.assertNotIn("Constants stay in launch", text) def test_plan_agent_facades_use_explicit_compatibility_imports(self) -> None: - for filename in ("config.py", "launch.py", "workflow.py", "recovery.py"): + for filename in ("config.py", "launch.py", "recovery.py"): path = PLAN_AGENT_ROOT / filename tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) wildcard_imports = [ diff --git a/tests/python/planning/test_planning_menu_rendering.py b/tests/python/planning/test_planning_menu_rendering.py index 92e98bef9..b3d173168 100644 --- a/tests/python/planning/test_planning_menu_rendering.py +++ b/tests/python/planning/test_planning_menu_rendering.py @@ -118,6 +118,7 @@ def test_run_flushes_pending_input_on_exit(self) -> None: patch("sys.stdin.fileno", return_value=7), patch("termios.tcgetattr", return_value=[0, 0, 0, 0, 0, 0]), patch("termios.tcsetattr"), + patch("termios.tcflush"), patch("tty.setraw"), patch("os.read", return_value=b"\n"), patch.object(PlanningSelectionMenu, "flush_pending_input") as flush_mock, diff --git a/tests/python/runtime/test_runtime_context_protocols.py b/tests/python/runtime/test_runtime_context_protocols.py index 60ee8e62a..66a1b1961 100644 --- a/tests/python/runtime/test_runtime_context_protocols.py +++ b/tests/python/runtime/test_runtime_context_protocols.py @@ -158,7 +158,6 @@ def test_artifact_path_helpers_use_repository_when_available_and_runtime_root_fa def test_command_support_uses_runtime_context_dependency_helpers(self) -> None: support_paths = [ REPO_ROOT / "python/envctl_engine/runtime/engine_runtime_event_support.py", - REPO_ROOT / "python/envctl_engine/planning/plan_agent/workflow.py", REPO_ROOT / "python/envctl_engine/actions/action_test_interrupt_support.py", REPO_ROOT / "python/envctl_engine/runtime/playwright_command_support.py", REPO_ROOT / "python/envctl_engine/runtime/qa_user_command_support.py", diff --git a/tests/python/shared/test_import_audit.py b/tests/python/shared/test_import_audit.py index d2d6387d8..e34d32047 100644 --- a/tests/python/shared/test_import_audit.py +++ b/tests/python/shared/test_import_audit.py @@ -160,7 +160,6 @@ ENGINE_ROOT / "planning" / "plan_agent" / "terminal_screen.py", }, "_build_plan_agent_workflow": { - ENGINE_ROOT / "planning" / "plan_agent" / "workflow.py", ENGINE_ROOT / "planning" / "plan_agent" / "workflow_build.py", }, "_spawn_omx_session_for_worktree": { @@ -179,11 +178,9 @@ ENGINE_ROOT / "planning" / "plan_agent" / "config.py", }, "_workflow_step_prompt_text": { - ENGINE_ROOT / "planning" / "plan_agent" / "workflow.py", ENGINE_ROOT / "planning" / "plan_agent" / "workflow_prompt_support.py", }, "_resolve_preset_submission_text": { - ENGINE_ROOT / "planning" / "plan_agent" / "workflow.py", ENGINE_ROOT / "planning" / "plan_agent" / "workflow_prompt_support.py", }, "_codex_queue_screen_looks_ready": { @@ -214,13 +211,13 @@ ENGINE_ROOT / "planning" / "plan_agent" / "recovery.py", }, "_codex_goal_text_for_worktree": { - ENGINE_ROOT / "planning" / "plan_agent" / "workflow.py", + ENGINE_ROOT / "planning" / "plan_agent" / "workflow_runtime_support.py", }, "_emit_codex_goal_event": { - ENGINE_ROOT / "planning" / "plan_agent" / "workflow.py", + ENGINE_ROOT / "planning" / "plan_agent" / "workflow_runtime_support.py", }, "_wrap_omx_initial_prompt_for_workflow": { - ENGINE_ROOT / "planning" / "plan_agent" / "workflow.py", + ENGINE_ROOT / "planning" / "plan_agent" / "workflow_runtime_support.py", }, "_queue_failure_event_context": { ENGINE_ROOT / "planning" / "plan_agent" / "recovery.py", From e546a81020107d412a922b2cfc823ce94972d9f3 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:19:02 +0300 Subject: [PATCH 08/26] Deeply harden runtime state and lifecycle ownership --- docs/reference/commands.md | 13 + docs/reference/important-flags.md | 4 +- docs/user/common-workflows.md | 6 + docs/user/getting-started.md | 2 +- .../actions/action_git_state_support.py | 122 +- .../actions/action_pr_git_support.py | 47 + .../actions/action_pr_workflow_support.py | 119 +- .../actions/action_ship_checks_phase.py | 46 + .../actions/action_ship_conflicts.py | 3 +- .../actions/action_ship_pr_context.py | 49 + .../actions/action_ship_pr_phase.py | 14 + .../actions/action_ship_support.py | 58 +- .../action_worktree_target_resolution.py | 64 +- .../actions/project_action_domain.py | 20 +- .../project_action_workflow_factory.py | 6 +- .../actions/project_action_workflows.py | 7 +- python/envctl_engine/config/__init__.py | 14 +- .../plan_agent/workflow_runtime_addresses.py | 15 +- .../planning/worktree_provenance.py | 21 +- .../runtime/engine_runtime_artifacts.py | 52 +- .../runtime/engine_runtime_construction.py | 30 +- .../runtime/engine_runtime_dispatch.py | 33 +- .../runtime/engine_runtime_event_support.py | 38 +- .../engine_runtime_lifecycle_facade.py | 12 +- .../runtime/engine_runtime_startup_facade.py | 16 +- .../runtime/engine_runtime_startup_support.py | 97 + .../runtime/engine_runtime_state_lookup.py | 11 +- .../runtime/help_topic_actions.py | 3 +- .../runtime/help_topic_lifecycle.py | 26 +- .../runtime/inspection_support.py | 177 +- .../runtime/lifecycle_cleanup_orchestrator.py | 325 ++- .../runtime/lifecycle_dependency_stop.py | 23 +- .../runtime/lifecycle_operation_lease.py | 118 + .../runtime/lifecycle_requirement_ports.py | 123 +- .../lifecycle_service_stop_selection.py | 18 +- .../runtime/lifecycle_service_termination.py | 292 ++- .../runtime/lifecycle_targeted_state.py | 151 ++ .../runtime/lifecycle_worktree_cleanup.py | 60 +- .../envctl_engine/runtime/service_manager.py | 151 +- .../runtime/state_fingerprint_support.py | 8 +- .../runtime/state_inspection_support.py | 159 ++ .../shared/git_branch_support.py | 86 + python/envctl_engine/shared/ports.py | 296 ++- python/envctl_engine/shared/process_cwd.py | 55 + .../shared/process_lifecycle_probe.py | 116 +- .../shared/process_termination.py | 192 ++ python/envctl_engine/shared/protocols.py | 27 +- .../startup/disabled_startup_resolution.py | 14 +- python/envctl_engine/startup/finalization.py | 27 +- .../startup/finalization_failure.py | 129 +- .../startup/finalization_run_state.py | 6 +- python/envctl_engine/startup/protocols.py | 14 +- .../startup/requirements_component_startup.py | 20 + .../startup/requirements_execution.py | 8 +- .../startup/requirements_project_startup.py | 79 +- .../startup/restart_prestop_support.py | 118 +- .../startup/resume_orchestrator.py | 31 +- .../startup/run_reuse_dashboard_restore.py | 10 +- .../startup/run_reuse_decision.py | 22 +- .../startup/run_reuse_fresh_start.py | 163 +- .../startup/run_reuse_identity.py | 26 +- .../startup/run_reuse_resolution.py | 23 +- .../envctl_engine/startup/run_reuse_resume.py | 23 +- .../startup/selected_context_startup.py | 78 +- .../startup/service_attach_execution.py | 231 +- python/envctl_engine/startup/session.py | 58 + .../startup/startup_execution_support.py | 140 +- .../startup/startup_selection_support.py | 20 +- python/envctl_engine/state/__init__.py | 6 + python/envctl_engine/state/fingerprints.py | 20 + python/envctl_engine/state/lookup.py | 47 + python/envctl_engine/state/models.py | 1 + python/envctl_engine/state/persistence.py | 193 ++ python/envctl_engine/state/project_runtime.py | 127 +- python/envctl_engine/state/repository.py | 1976 ++++++++++++++--- python/envctl_engine/state/run_index.py | 582 +++++ .../envctl_engine/state/state_aggregation.py | 322 +++ .../actions/ship_workflow_test_support.py | 3 + .../actions/test_action_git_state_support.py | 165 +- .../test_action_pr_workflow_support.py | 220 +- .../actions/test_action_ship_support.py | 3 +- .../test_action_ship_workflow_lifecycle.py | 81 +- .../actions/test_action_worktree_runner.py | 84 +- tests/python/actions/test_actions_cli_pr.py | 28 +- tests/python/actions/test_actions_cli_ship.py | 24 +- .../plan_agent_launch_support_test_support.py | 10 +- ...t_plan_agent_workflow_runtime_addresses.py | 17 +- ...test_planning_worktree_setup_provenance.py | 60 +- ...ngine_runtime_real_startup_test_support.py | 5 + .../runtime/lifecycle_parity_test_support.py | 8 +- .../runtime/test_engine_runtime_artifacts.py | 85 +- ...ine_runtime_command_parity_state_config.py | 73 +- .../runtime/test_engine_runtime_dispatch.py | 224 +- .../test_engine_runtime_event_support.py | 26 + .../test_engine_runtime_lifecycle_support.py | 30 +- .../test_engine_runtime_resume_startup.py | 21 +- .../test_engine_runtime_startup_support.py | 206 +- .../test_engine_runtime_state_lookup.py | 14 +- tests/python/runtime/test_help_topics.py | 9 + ...t_lifecycle_cleanup_spinner_integration.py | 583 ++++- .../runtime/test_lifecycle_dependency_stop.py | 65 +- .../test_lifecycle_requirement_ports.py | 121 +- .../test_lifecycle_service_stop_selection.py | 38 + .../test_lifecycle_service_termination.py | 716 +++++- .../test_lifecycle_worktree_cleanup.py | 54 +- .../runtime/test_runtime_scope_isolation.py | 104 +- .../shared/test_ports_lock_reclamation.py | 303 +++ tests/python/shared/test_process_cwd.py | 56 + .../test_process_runner_listener_detection.py | 136 +- .../python/shared/test_process_termination.py | 130 ++ .../test_requirements_project_startup.py | 151 ++ .../startup/test_restart_prestop_support.py | 232 +- .../python/startup/test_run_reuse_identity.py | 52 +- .../python/startup/test_run_reuse_support.py | 304 ++- .../startup/test_selected_context_startup.py | 128 +- .../startup/test_service_attach_execution.py | 378 +++- tests/python/startup/test_service_manager.py | 207 +- .../test_startup_dashboard_stopped_restore.py | 4 +- .../startup/test_startup_finalization.py | 290 ++- .../test_startup_fingerprint_replacement.py | 6 +- .../test_startup_orchestrator_flow_reuse.py | 210 +- .../startup/test_startup_selection_support.py | 21 +- .../startup/test_support_module_decoupling.py | 207 +- tests/python/state/test_fingerprints.py | 27 + tests/python/state/test_lookup.py | 85 + tests/python/state/test_persistence.py | 183 ++ .../state/test_project_runtime_resolution.py | 83 +- tests/python/state/test_run_index.py | 437 ++++ tests/python/state/test_state_aggregation.py | 139 ++ .../state/test_state_repository_contract.py | 1816 ++++++++++++++- .../state/test_state_repository_hardening.py | 638 ++++++ tests/python/state/test_state_roundtrip.py | 21 +- 132 files changed, 15682 insertions(+), 1478 deletions(-) create mode 100644 python/envctl_engine/actions/action_pr_git_support.py create mode 100644 python/envctl_engine/actions/action_ship_checks_phase.py create mode 100644 python/envctl_engine/actions/action_ship_pr_context.py create mode 100644 python/envctl_engine/runtime/lifecycle_operation_lease.py create mode 100644 python/envctl_engine/runtime/lifecycle_targeted_state.py create mode 100644 python/envctl_engine/runtime/state_inspection_support.py create mode 100644 python/envctl_engine/shared/git_branch_support.py create mode 100644 python/envctl_engine/shared/process_cwd.py create mode 100644 python/envctl_engine/shared/process_termination.py create mode 100644 python/envctl_engine/state/fingerprints.py create mode 100644 python/envctl_engine/state/lookup.py create mode 100644 python/envctl_engine/state/persistence.py create mode 100644 python/envctl_engine/state/run_index.py create mode 100644 python/envctl_engine/state/state_aggregation.py create mode 100644 tests/python/shared/test_process_cwd.py create mode 100644 tests/python/shared/test_process_termination.py create mode 100644 tests/python/startup/test_requirements_project_startup.py create mode 100644 tests/python/state/test_fingerprints.py create mode 100644 tests/python/state/test_lookup.py create mode 100644 tests/python/state/test_persistence.py create mode 100644 tests/python/state/test_run_index.py create mode 100644 tests/python/state/test_state_aggregation.py create mode 100644 tests/python/state/test_state_repository_hardening.py diff --git a/docs/reference/commands.md b/docs/reference/commands.md index b568e2d28..bd311a630 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -388,6 +388,14 @@ envctl kill --backend --headless # alias for stop --backend envctl kill-all --headless # alias for stop-all ``` +Plain dependency cleanup is intentionally storage-preserving: `stop --dependencies` +and plain `stop-all` release saved dependency records and scoped port locks, but +leave envctl-managed Docker dependency stacks and their storage available for a +later start to reuse. Use `stop-all --remove-volumes` to remove supported managed +containers and volumes, or `blast-all` for aggressive ecosystem cleanup. +Explicit `--main` and `--trees` cleanup is mode-scoped: it never stops or retires +the other mode's saved runtime. + For AI plan-agent launches, `--entire-system` starts configured or autodetected local app services. In a repo/worktree with no explicit backend, frontend, or additional-service configuration and no supported autodetectable app layout, envctl reports that no local app system is configured and continues without app services; the AI session still launches. @@ -522,6 +530,11 @@ Commit defaults: - write one complete next commit message in `.envctl-commit-message.md` rather than multiple fragmented summaries only when using the fallback ledger - envctl-local control artifacts (`.envctl*`, `.codegraph/`, `.serena/project.local.yml`, `MAIN_TASK.md`, `OLD_TASK_*.md`, `trees/`) stay local; if a broad `git add .` stages them, `envctl commit` unstages those protected paths before committing normal changes +PR defaults: + +- `envctl pr` and `envctl ship` target `dev` when that remote branch exists, otherwise `main`, otherwise `master`; use `--pr-base ` to override the selection explicitly +- an existing open PR for the current head branch is rediscovered before creation, so rerunning the command reports or updates the same PR instead of opening a duplicate + Optional plan-agent launch config for `--plan` and `--import`: `envctl ship` is the preferred AI handoff command. From inside the current diff --git a/docs/reference/important-flags.md b/docs/reference/important-flags.md index 4441f34f9..b15bdc3ed 100644 --- a/docs/reference/important-flags.md +++ b/docs/reference/important-flags.md @@ -41,9 +41,9 @@ pass `--interactive` only when you want prompts. | `--frontend` | Runtime scope: run or stop frontend app services only. | | `--fullstack` | Runtime scope: run or stop backend and frontend app services. | | `--both` | Alias for `--fullstack`. | -| `--dependencies` | Runtime scope: run or stop dependencies only (DB/Redis/etc.), without app services. | +| `--dependencies` | Runtime scope: run dependencies only, or detach their saved records/port locks on stop without removing reusable managed Docker stacks. | | `--deps` | Alias for `--dependencies`. | -| `--entire-system` | Runtime scope: run or stop dependencies plus backend and frontend services. | +| `--entire-system` | Runtime scope: run dependencies plus app services; on stop, terminate app services and detach dependency records/locks while keeping managed Docker stacks reusable. | | `--only-frontend` | Startup/plan modifier: launch only the frontend app service; skip backend, managed dependencies, and dependency prep. | | `--only-backend` | Startup/plan modifier: launch only the backend app service; skip frontend, managed dependencies, and dependency prep. | | `--no-deps` | Startup/plan modifier: skip managed dependencies and plan-agent dependency prep. | diff --git a/docs/user/common-workflows.md b/docs/user/common-workflows.md index f9de37747..a4872cd3e 100644 --- a/docs/user/common-workflows.md +++ b/docs/user/common-workflows.md @@ -68,6 +68,12 @@ envctl stop --entire-system --headless envctl kill-all --headless ``` +`stop --dependencies` and plain `stop-all` preserve managed Docker dependency +stacks and storage for reuse while releasing their saved records and port locks. +Use `stop-all --remove-volumes` or `blast-all` when those managed containers must +also be removed. Pass `--main` or `--trees` to clean only that saved runtime mode; +the other mode remains active. + When you invoke a specific action command directly, envctl stays non-interactive by default. For example, `envctl kill-all`, `envctl pr`, `envctl test`, `envctl logs`, and `envctl migrate` behave like their `--headless` forms. Add `--interactive` only when you want envctl to prompt diff --git a/docs/user/getting-started.md b/docs/user/getting-started.md index ce08c790c..c96f8c627 100644 --- a/docs/user/getting-started.md +++ b/docs/user/getting-started.md @@ -188,7 +188,7 @@ Why this works well: - `explain-startup` shows what the runtime is about to do - `--resume` is the fastest normal start path after previous runs - `dashboard`, `logs`, and `test` cover the common daily loop -- `stop-all` gives you a clean shutdown +- `stop-all` cleanly shuts down tracked app services and releases dependency records/locks; managed Docker dependency stacks and storage remain reusable (use `stop-all --remove-volumes` or `blast-all` to remove them) If you want a fresh start instead of resume: diff --git a/python/envctl_engine/actions/action_git_state_support.py b/python/envctl_engine/actions/action_git_state_support.py index d99a73ab5..53928f91f 100644 --- a/python/envctl_engine/actions/action_git_state_support.py +++ b/python/envctl_engine/actions/action_git_state_support.py @@ -1,10 +1,14 @@ from __future__ import annotations from dataclasses import dataclass +import json +import os from pathlib import Path import subprocess from typing import Callable +from envctl_engine.shared.git_branch_support import detect_preferred_default_branch + @dataclass(frozen=True, slots=True) class DirtyWorktreeReport: @@ -20,6 +24,12 @@ def dirty(self) -> bool: return self.staged or self.unstaged or self.untracked +@dataclass(frozen=True, slots=True) +class ExistingPullRequest: + url: str + base_branch: str + + def resolve_git_root(project_root: Path, repo_root: Path) -> Path: for candidate in (project_root, repo_root): if (candidate / ".git").exists(): @@ -71,53 +81,131 @@ def classify_dirty_porcelain(status_output: str) -> tuple[bool, bool, bool]: def detect_default_branch(git_root: Path, *, git_output: Callable[[Path, list[str]], str]) -> str: - ref = git_output(git_root, ["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]).strip() - if ref.startswith("origin/"): - return ref.split("origin/", 1)[1] - for candidate in ("main", "master"): - if git_output(git_root, ["rev-parse", "--verify", candidate]).strip(): - return candidate - return "main" + return detect_preferred_default_branch( + lambda args: git_output(git_root, list(args)), + probe_remote_heads=True, + ) -def existing_pr_url( +def existing_pull_request( git_root: Path, branch: str, *, gh_path: str | None, run_process: Callable[..., subprocess.CompletedProcess[str]], -) -> str: +) -> ExistingPullRequest | None: branch_name = branch.strip() if not branch_name or branch_name in {"HEAD", "unknown"}: - return "" + return None if gh_path is None: - return "" + return None listed = run_process( - [gh_path, "pr", "list", "--head", branch_name, "--state", "open", "--json", "url", "--jq", ".[0].url"], + [ + gh_path, + "pr", + "list", + "--head", + branch_name, + "--state", + "open", + "--limit", + "1", + "--json", + "url,baseRefName", + ], cwd=str(git_root), text=True, capture_output=True, check=False, ) if listed.returncode != 0: - return "" - return listed.stdout.strip() + return None + try: + payload = json.loads(listed.stdout or "[]") + except (TypeError, json.JSONDecodeError): + return None + if not isinstance(payload, list) or not payload or not isinstance(payload[0], dict): + return None + url = str(payload[0].get("url") or "").strip() + if not url: + return None + return ExistingPullRequest( + url=url, + base_branch=str(payload[0].get("baseRefName") or "").strip(), + ) -def run_git( +def existing_pr_url( git_root: Path, - args: list[str], + branch: str, *, + gh_path: str | None, + run_process: Callable[..., subprocess.CompletedProcess[str]], +) -> str: + existing = existing_pull_request( + git_root, + branch, + gh_path=gh_path, + run_process=run_process, + ) + return existing.url if existing is not None else "" + + +def update_pull_request_base( + git_root: Path, + pull_request: ExistingPullRequest, + base_branch: str, + *, + gh_path: str | None, run_process: Callable[..., subprocess.CompletedProcess[str]], ) -> subprocess.CompletedProcess[str]: + branch = base_branch.strip() + command = [*([gh_path] if gh_path else ["gh"]), "pr", "edit", pull_request.url, "--base", branch] + if gh_path is None: + return subprocess.CompletedProcess( + command, + 127, + stdout="", + stderr=( + "gh is required to update the existing pull request base. " + f"Install or authenticate gh, then run: gh pr edit {pull_request.url} --base {branch}" + ), + ) return run_process( - ["git", "-C", str(git_root), *args], + command, + cwd=str(git_root), text=True, capture_output=True, check=False, ) +def run_git( + git_root: Path, + args: list[str], + *, + run_process: Callable[..., subprocess.CompletedProcess[str]], +) -> subprocess.CompletedProcess[str]: + command = ["git", "-C", str(git_root), *args] + options: dict[str, object] = { + "text": True, + "capture_output": True, + "check": False, + } + if args[:1] == ["ls-remote"]: + options["timeout"] = 10 + options["env"] = {**os.environ, "GIT_TERMINAL_PROMPT": "0"} + try: + return run_process(command, **options) + except subprocess.TimeoutExpired as exc: + return subprocess.CompletedProcess( + command, + 124, + stdout=str(exc.stdout or ""), + stderr="Timed out while querying remote branches.", + ) + + def git_output( git_root: Path, args: list[str], diff --git a/python/envctl_engine/actions/action_pr_git_support.py b/python/envctl_engine/actions/action_pr_git_support.py new file mode 100644 index 000000000..bf20826a8 --- /dev/null +++ b/python/envctl_engine/actions/action_pr_git_support.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from pathlib import Path +import shutil +import subprocess + +from envctl_engine.actions.action_git_state_support import ( + ExistingPullRequest, + existing_pr_url as lookup_existing_pr_url, + existing_pull_request as lookup_existing_pull_request, + update_pull_request_base as update_existing_pull_request_base, +) + + +def existing_pr_url(git_root: Path, branch: str) -> str: + return lookup_existing_pr_url( + git_root, + branch, + gh_path=shutil.which("gh"), + run_process=subprocess.run, + ) + + +def existing_pull_request(git_root: Path, branch: str) -> ExistingPullRequest | None: + return lookup_existing_pull_request( + git_root, + branch, + gh_path=shutil.which("gh"), + run_process=subprocess.run, + ) + + +def update_pull_request_base( + git_root: Path, + pull_request: ExistingPullRequest, + base_branch: str, +) -> subprocess.CompletedProcess[str]: + return update_existing_pull_request_base( + git_root, + pull_request, + base_branch, + gh_path=shutil.which("gh"), + run_process=subprocess.run, + ) + + +__all__ = ["existing_pr_url", "existing_pull_request", "update_pull_request_base"] diff --git a/python/envctl_engine/actions/action_pr_workflow_support.py b/python/envctl_engine/actions/action_pr_workflow_support.py index 523b7897a..da6ecf87a 100644 --- a/python/envctl_engine/actions/action_pr_workflow_support.py +++ b/python/envctl_engine/actions/action_pr_workflow_support.py @@ -5,6 +5,12 @@ import subprocess from typing import Any, Callable +from envctl_engine.actions.action_git_state_support import ExistingPullRequest + + +ExistingPullRequestLookup = Callable[[Path, str], ExistingPullRequest | None] +PullRequestBaseUpdater = Callable[[Path, ExistingPullRequest, str], subprocess.CompletedProcess[str]] + def run_pr_workflow( context: Any, @@ -13,7 +19,8 @@ def run_pr_workflow( git_available: bool, git_output_fn: Callable[[Path, list[str]], str], resolve_base_branch_fn: Callable[[Any, Path], str], - existing_pr_url_fn: Callable[[Path, str], str], + existing_pull_request_fn: ExistingPullRequestLookup, + update_pull_request_base_fn: PullRequestBaseUpdater, probe_dirty_worktree_fn: Callable[[Path, Path, str], Any], run_commit_action_fn: Callable[[Any], int], pr_title_fn: Callable[[Any, Path, str], str], @@ -32,13 +39,31 @@ def run_pr_workflow( if head_branch in {"HEAD", "unknown"}: print(f"Skipping {context.project_name} (detached HEAD).") return 0 - base_branch = resolve_base_branch_fn(context, git_root) - existing_url = existing_pr_url_fn(git_root, head_branch) - if existing_url: - print(f"PR already exists: {existing_url}") + explicit_base = _explicit_pr_base(context) + existing = existing_pull_request_fn(git_root, head_branch) + if existing is not None: + if explicit_base and existing.base_branch != explicit_base: + if not _update_existing_pull_request_base( + git_root=git_root, + head_branch=head_branch, + existing=existing, + desired_base=explicit_base, + existing_pull_request_fn=existing_pull_request_fn, + update_pull_request_base_fn=update_pull_request_base_fn, + print_process_output_fn=print_process_output_fn, + ): + return 1 + print(f"PR already exists: {existing.url}") return 0 + base_branch = resolve_base_branch_fn(context, git_root) + if base_branch and head_branch == base_branch: + print( + f"Cannot open a pull request from {head_branch!r} into itself. Create or switch to a feature branch first." + ) + return 1 + dirty_report = probe_dirty_worktree_fn(context.project_root, context.repo_root, context.project_name) if dirty_report.dirty: print(f"Dirty worktree detected for {context.project_name}; committing and pushing before PR creation.") @@ -54,6 +79,7 @@ def run_pr_workflow( git_root=git_root, head_branch=head_branch, base_branch=base_branch, + existing_pull_request_fn=existing_pull_request_fn, print_process_output_fn=print_process_output_fn, run_process_fn=run_process_fn, ) @@ -70,11 +96,58 @@ def run_pr_workflow( pr_title_fn=pr_title_fn, pr_body_fn=pr_body_fn, write_pr_body_file_fn=write_pr_body_file_fn, + existing_pull_request_fn=existing_pull_request_fn, print_process_output_fn=print_process_output_fn, run_process_fn=run_process_fn, ) +def _explicit_pr_base(context: Any) -> str: + env = getattr(context, "env", {}) + try: + return str(env.get("ENVCTL_PR_BASE", "")).strip() + except AttributeError: + return "" + + +def _update_existing_pull_request_base( + *, + git_root: Path, + head_branch: str, + existing: ExistingPullRequest, + desired_base: str, + existing_pull_request_fn: ExistingPullRequestLookup, + update_pull_request_base_fn: PullRequestBaseUpdater, + print_process_output_fn: Callable[[subprocess.CompletedProcess[str]], None], +) -> bool: + updated = update_pull_request_base_fn(git_root, existing, desired_base) + if updated.returncode != 0: + refreshed = existing_pull_request_fn(git_root, head_branch) + if _pull_request_targets(refreshed, desired_base): + return True + print_process_output_fn(updated) + print( + f"Unable to update existing PR {existing.url} from base " + f"{existing.base_branch or ''!r} to {desired_base!r}." + ) + return False + + refreshed = existing_pull_request_fn(git_root, head_branch) + if not _pull_request_targets(refreshed, desired_base): + actual_base = refreshed.base_branch if refreshed is not None else "" + print( + f"Updated existing PR {existing.url}, but could not verify base {desired_base!r} " + f"(reported {actual_base!r})." + ) + return False + print(f"Updated PR base to {desired_base}: {refreshed.url}") + return True + + +def _pull_request_targets(pull_request: ExistingPullRequest | None, base_branch: str) -> bool: + return pull_request is not None and pull_request.base_branch == base_branch + + def _run_create_pr_helper( context: Any, *, @@ -82,6 +155,7 @@ def _run_create_pr_helper( git_root: Path, head_branch: str, base_branch: str, + existing_pull_request_fn: ExistingPullRequestLookup, print_process_output_fn: Callable[[subprocess.CompletedProcess[str]], None], run_process_fn: Callable[..., subprocess.CompletedProcess[str]], ) -> int: @@ -96,9 +170,17 @@ def _run_create_pr_helper( capture_output=True, check=False, ) - print_process_output_fn(created) if created.returncode != 0: + if _recover_concurrent_pull_request( + git_root=git_root, + head_branch=head_branch, + desired_base=base_branch, + existing_pull_request_fn=existing_pull_request_fn, + ): + return 0 + print_process_output_fn(created) return 1 + print_process_output_fn(created) return 0 @@ -112,6 +194,7 @@ def _run_gh_pr_create( pr_title_fn: Callable[[Any, Path, str], str], pr_body_fn: Callable[[Any, Path, str, str], str], write_pr_body_file_fn: Callable[[str], Path], + existing_pull_request_fn: ExistingPullRequestLookup, print_process_output_fn: Callable[[subprocess.CompletedProcess[str]], None], run_process_fn: Callable[..., subprocess.CompletedProcess[str]], ) -> int: @@ -123,12 +206,34 @@ def _run_gh_pr_create( args.extend(["--base", base_branch]) try: created = run_process_fn(args, cwd=str(git_root), text=True, capture_output=True, check=False) - print_process_output_fn(created) if created.returncode != 0: + if _recover_concurrent_pull_request( + git_root=git_root, + head_branch=head_branch, + desired_base=base_branch, + existing_pull_request_fn=existing_pull_request_fn, + ): + return 0 + print_process_output_fn(created) return 1 + print_process_output_fn(created) return 0 finally: try: body_file.unlink() except OSError: pass + + +def _recover_concurrent_pull_request( + *, + git_root: Path, + head_branch: str, + desired_base: str, + existing_pull_request_fn: ExistingPullRequestLookup, +) -> bool: + existing = existing_pull_request_fn(git_root, head_branch) + if not _pull_request_targets(existing, desired_base): + return False + print(f"PR already exists: {existing.url}") + return True diff --git a/python/envctl_engine/actions/action_ship_checks_phase.py b/python/envctl_engine/actions/action_ship_checks_phase.py new file mode 100644 index 000000000..f9103492c --- /dev/null +++ b/python/envctl_engine/actions/action_ship_checks_phase.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from envctl_engine.actions.action_ship_contract import emit_ship_progress +from envctl_engine.actions.action_ship_phase_status import ( + callable_accepts_keyword, + check_phase_status, + ship_status_is_success, +) + +GithubPrChecks = Callable[..., dict[str, object]] +FinishShip = Callable[..., int] + + +def run_ship_checks_phase( + *, + state: Any, + checks_fn: GithubPrChecks, + finish: FinishShip, +) -> int: + check_kwargs: dict[str, object] = { + "branch": state.branch, + "pr_url": state.pr_url, + "expected_head_sha": state.after_sha, + } + if callable_accepts_keyword(checks_fn, "progress_callback"): + check_kwargs["progress_callback"] = emit_ship_progress + checks = checks_fn(state.git_root, **check_kwargs) + status = check_phase_status(checks) + state.step_statuses.append(status) + return finish( + state, + status=status, + ok=ship_status_is_success(status), + commit_sha=state.after_sha, + pushed=state.pushed, + pr_url=state.pr_url, + pr_created=state.pr_created, + checks=checks, + merge_conflicts=state.merge_conflicts, + ) + + +__all__ = ["run_ship_checks_phase"] diff --git a/python/envctl_engine/actions/action_ship_conflicts.py b/python/envctl_engine/actions/action_ship_conflicts.py index d0047d032..919210dd4 100644 --- a/python/envctl_engine/actions/action_ship_conflicts.py +++ b/python/envctl_engine/actions/action_ship_conflicts.py @@ -62,9 +62,10 @@ def predicted_merge_conflict_report( resolve_base_ref: ResolveBaseRef, run_git: RunGit, git_output: GitOutput, + base_branch_override: str = "", ) -> dict[str, object]: del branch - base_branch = resolve_base_branch(context, git_root) + base_branch = base_branch_override.strip() or resolve_base_branch(context, git_root) base_ref = resolve_base_ref(git_root, base_branch) if not base_ref: return { diff --git a/python/envctl_engine/actions/action_ship_pr_context.py b/python/envctl_engine/actions/action_ship_pr_context.py new file mode 100644 index 000000000..fdc36b6f0 --- /dev/null +++ b/python/envctl_engine/actions/action_ship_pr_context.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import Any, Protocol + +from envctl_engine.actions.action_git_state_support import ExistingPullRequest + +ExistingPullRequestLookup = Callable[[Path, str], ExistingPullRequest | None] + + +class ShipPullRequestState(Protocol): + git_root: Path + branch: str + pr_url: str + pr_base_branch: str + + +def explicit_pr_base(context: Any) -> str: + env = getattr(context, "env", {}) + try: + return str(env.get("ENVCTL_PR_BASE", "")).strip() + except AttributeError: + return "" + + +def refresh_existing_pull_request( + state: ShipPullRequestState, + lookup: ExistingPullRequestLookup | None, +) -> None: + if lookup is None: + return + existing = lookup(state.git_root, state.branch) + if existing is None: + return + state.pr_url = existing.url + state.pr_base_branch = existing.base_branch + + +def conflict_base_branch_override(context: Any, state: ShipPullRequestState) -> str: + return explicit_pr_base(context) or state.pr_base_branch + + +__all__ = [ + "ExistingPullRequestLookup", + "conflict_base_branch_override", + "explicit_pr_base", + "refresh_existing_pull_request", +] diff --git a/python/envctl_engine/actions/action_ship_pr_phase.py b/python/envctl_engine/actions/action_ship_pr_phase.py index f328c3604..68b693a48 100644 --- a/python/envctl_engine/actions/action_ship_pr_phase.py +++ b/python/envctl_engine/actions/action_ship_pr_phase.py @@ -4,6 +4,7 @@ from typing import Any, Callable from envctl_engine.actions.action_ship_contract import emit_ship_progress +from envctl_engine.actions.action_ship_pr_context import explicit_pr_base as _explicit_pr_base FinishShip = Callable[..., int] @@ -18,6 +19,19 @@ def run_ship_pr_phase( finish: FinishShip, ) -> int | None: if state.pr_url: + explicit_base = _explicit_pr_base(context) + if explicit_base: + pr_code = run_pr_action(context) + if pr_code != 0: + return finish( + state, + status="pr_failed", + ok=False, + commit_sha=state.after_sha, + pr_url=state.pr_url, + pr_created=False, + ) + state.pr_url = existing_pr_url(state.git_root, state.branch) or state.pr_url state.step_statuses.append("pr_exists") emit_ship_progress(f"ship: PR already exists for {context.project_name}: {state.pr_url}") return None diff --git a/python/envctl_engine/actions/action_ship_support.py b/python/envctl_engine/actions/action_ship_support.py index 87c96088e..407eb1dc9 100644 --- a/python/envctl_engine/actions/action_ship_support.py +++ b/python/envctl_engine/actions/action_ship_support.py @@ -9,6 +9,7 @@ from envctl_engine.actions.action_ship_check_results import normalize_github_pr_checks # noqa: F401 from envctl_engine.actions.action_ship_checks import github_pr_checks as default_github_pr_checks +from envctl_engine.actions.action_ship_checks_phase import run_ship_checks_phase from envctl_engine.actions.action_ship_conflicts import ( existing_merge_conflict_report, parse_merge_tree_conflicts, # noqa: F401 @@ -17,17 +18,16 @@ ) from envctl_engine.actions.action_ship_contract import ( emit_ship_commit_progress, - emit_ship_progress, parse_ship_json_output, print_ship_result, # noqa: F401 ship_payload, # noqa: F401 ship_protected_paths, ) from envctl_engine.actions.action_ship_finish import finish_ship_workflow -from envctl_engine.actions.action_ship_phase_status import ( - callable_accepts_keyword, - check_phase_status, - ship_status_is_success, +from envctl_engine.actions.action_ship_pr_context import ( + ExistingPullRequestLookup, + conflict_base_branch_override, + refresh_existing_pull_request, ) from envctl_engine.actions.action_ship_pr_phase import run_ship_pr_label_phase, run_ship_pr_phase from envctl_engine.actions.action_ship_push import run_ship_push_phase @@ -39,6 +39,7 @@ GithubPrChecks = Callable[..., dict[str, object]] github_pr_checks = default_github_pr_checks + @dataclass(slots=True) class ShipWorkflowState: git_root: Path @@ -51,6 +52,7 @@ class ShipWorkflowState: committed: bool = False pushed: bool = False pr_url: str = "" + pr_base_branch: str = "" pr_created: bool = False step_statuses: list[str] = field(default_factory=list) merge_conflicts: dict[str, object] = field(default_factory=dict) @@ -72,6 +74,7 @@ class ShipWorkflowDependencies: partition_envctl_protected_paths: Callable[[str], Any] ordered_unique_paths: Callable[..., list[str]] github_pr_checks: GithubPrChecks | None = None + existing_pull_request: ExistingPullRequestLookup | None = None @dataclass(slots=True) @@ -144,7 +147,10 @@ def _run_commit_phase(self, state: ShipWorkflowState) -> int | None: self.context.repo_root, project_name=self.context.project_name, ) - state.pr_url = self.dependencies.existing_pr_url(state.git_root, state.branch) + if self.dependencies.existing_pull_request is None: + state.pr_url = self.dependencies.existing_pr_url(state.git_root, state.branch) + else: + refresh_existing_pull_request(state, self.dependencies.existing_pull_request) commit_code = self.dependencies.run_commit_action(self.context) state.after_sha = self.dependencies.git_output(state.git_root, ["rev-parse", "HEAD"]).strip() @@ -159,13 +165,16 @@ def _run_commit_phase(self, state: ShipWorkflowState) -> int | None: return self._finish(state, status="commit_failed", ok=False, commit_sha=state.after_sha) def _run_pr_phase(self, state: ShipWorkflowState) -> int | None: - return run_ship_pr_phase( + result = run_ship_pr_phase( state=state, context=self.context, run_pr_action=self.dependencies.run_pr_action, existing_pr_url=self.dependencies.existing_pr_url, finish=self._finish, ) + if result is None: + refresh_existing_pull_request(state, self.dependencies.existing_pull_request) + return result def _run_pr_label_phase(self, state: ShipWorkflowState) -> int | None: return run_ship_pr_label_phase( @@ -184,6 +193,7 @@ def _reject_predicted_merge_conflicts(self, state: ShipWorkflowState) -> int | N resolve_base_ref=self.dependencies.resolve_base_ref, run_git=self.dependencies.run_git, git_output=self.dependencies.git_output, + base_branch_override=conflict_base_branch_override(self.context, state), ) if state.merge_conflicts.get("state") != "conflicts": return None @@ -228,27 +238,10 @@ def _run_push_phase(self, state: ShipWorkflowState) -> int | None: return None def _run_checks_phase(self, state: ShipWorkflowState) -> int: - checks_fn = self.dependencies.github_pr_checks or default_github_pr_checks - check_kwargs: dict[str, object] = { - "branch": state.branch, - "pr_url": state.pr_url, - "expected_head_sha": state.after_sha, - } - if callable_accepts_keyword(checks_fn, "progress_callback"): - check_kwargs["progress_callback"] = emit_ship_progress - checks = checks_fn(state.git_root, **check_kwargs) - status = check_phase_status(checks) - state.step_statuses.append(status) - return self._finish( - state, - status=status, - ok=ship_status_is_success(status), - commit_sha=state.after_sha, - pushed=state.pushed, - pr_url=state.pr_url, - pr_created=state.pr_created, - checks=checks, - merge_conflicts=state.merge_conflicts, + return run_ship_checks_phase( + state=state, + checks_fn=self.dependencies.github_pr_checks or default_github_pr_checks, + finish=self._finish, ) def _finish( @@ -277,8 +270,10 @@ def _finish( merge_conflicts=merge_conflicts, ) + def run_ship_workflow( - context: Any, *, + context: Any, + *, resolve_git_root: Callable[[Path, Path], Path], git_available: bool, git_output: GitOutput, @@ -292,7 +287,9 @@ def run_ship_workflow( existing_pr_url: Callable[[Path, str], str], partition_envctl_protected_paths: Callable[[str], Any], ordered_unique_paths: Callable[..., list[str]], - github_pr_checks: GithubPrChecks | None = None) -> int: + github_pr_checks: GithubPrChecks | None = None, + existing_pull_request: ExistingPullRequestLookup | None = None, +) -> int: return ShipWorkflowRunner( context=context, dependencies=ShipWorkflowDependencies( @@ -310,5 +307,6 @@ def run_ship_workflow( partition_envctl_protected_paths=partition_envctl_protected_paths, ordered_unique_paths=ordered_unique_paths, github_pr_checks=github_pr_checks, + existing_pull_request=existing_pull_request, ), ).run() diff --git a/python/envctl_engine/actions/action_worktree_target_resolution.py b/python/envctl_engine/actions/action_worktree_target_resolution.py index 543309386..e530b8d81 100644 --- a/python/envctl_engine/actions/action_worktree_target_resolution.py +++ b/python/envctl_engine/actions/action_worktree_target_resolution.py @@ -33,9 +33,16 @@ def resolve(self) -> object | None: return None matches = self._matching_candidates(cwd, repo_root=repo_root, trees_dir_name=trees_dir_name) - if len(matches) != 1: + if len(matches) == 1: + return matches[0] + if matches or self.require_configured_main_root: return None - return matches[0] + return self._linked_worktree_candidate( + cwd, + repo_root=repo_root, + configured_root_path=configured_root_path, + raw_runtime=raw_runtime, + ) def _current_working_directory(self) -> Path: invocation_cwd = str(getattr(self.runtime, "env", {}).get("ENVCTL_INVOCATION_CWD") or "").strip() @@ -84,6 +91,40 @@ def _matching_candidates(self, cwd: Path, *, repo_root: Path, trees_dir_name: st if _path_is_at_or_under(cwd, Path(str(getattr(candidate, "root"))).resolve()) ] + @staticmethod + def _linked_worktree_candidate( + cwd: Path, + *, + repo_root: Path, + configured_root_path: Path | None, + raw_runtime: Any, + ) -> object | None: + """Represent a linked checkout even when it is outside the configured trees directory. + + Source-checkout workflows commonly create sibling worktrees. Git proves + that such a checkout belongs to the configured repository, while the + top-level and configured-root comparisons prevent a tree-mode action + from silently targeting the main checkout. + """ + + top_level_raw = _git_output(raw_runtime, cwd, ["rev-parse", "--show-toplevel"]) + if not top_level_raw: + return None + try: + top_level = Path(top_level_raw).expanduser().resolve() + except OSError: + return None + if not _path_is_at_or_under(cwd, top_level): + return None + normalized_repo_root = repo_root.resolve() + if top_level == normalized_repo_root: + return None + if configured_root_path is not None and top_level == configured_root_path: + return None + + branch = _git_output(raw_runtime, top_level, ["branch", "--show-current"]) + return SimpleNamespace(name=branch or top_level.name, root=top_level) + def resolve_current_worktree_target( *, @@ -140,9 +181,7 @@ def main_repo_root_for_worktree(*, worktree_root: Path, runtime: Any, trees_dir_ return top_level common_dir_path = Path(common_dir_raw) common_dir = ( - (worktree_root / common_dir_path).resolve() - if not common_dir_path.is_absolute() - else common_dir_path.resolve() + (worktree_root / common_dir_path).resolve() if not common_dir_path.is_absolute() else common_dir_path.resolve() ) if common_dir.name == ".git": return common_dir.parent @@ -179,3 +218,18 @@ def repo_root_from_worktree_layout(worktree_root: Path, trees_dir_name: str) -> def _path_is_at_or_under(path: Path, root: Path) -> bool: return path == root or root in path.parents + + +def _git_output(runtime: object, cwd: Path, args: list[str]) -> str: + try: + process_runtime = resolve_process_runtime(runtime) + completed = process_runtime.run( + ["git", *args], + cwd=cwd, + timeout=10.0, + ) + except Exception: # noqa: BLE001 - target inference must fail closed + return "" + if getattr(completed, "returncode", 1) != 0: + return "" + return str(getattr(completed, "stdout", "") or "").strip() diff --git a/python/envctl_engine/actions/project_action_domain.py b/python/envctl_engine/actions/project_action_domain.py index 316f75a1f..5ff6e9206 100644 --- a/python/envctl_engine/actions/project_action_domain.py +++ b/python/envctl_engine/actions/project_action_domain.py @@ -10,6 +10,7 @@ import envctl_engine.actions.action_commit_support as commit_support import envctl_engine.actions.action_pr_label_support as pr_label_support import envctl_engine.actions.action_git_state_support as git_state_support +import envctl_engine.actions.action_pr_git_support as pr_git_support import envctl_engine.actions.action_pr_message_support as pr_message_support import envctl_engine.actions.action_review_artifact_support as review_artifact_support import envctl_engine.actions.action_review_base_support as review_base_support @@ -46,6 +47,7 @@ __all__ = [ "ActionProjectContext", "DirtyWorktreeReport", + "ExistingPullRequest", "EnvctlProtectedPathPartition", "OriginalPlanResolution", "ReviewBaseResolution", @@ -70,6 +72,7 @@ "_unmerged_stage_entries", "_write_pr_body_file", "detect_default_branch", + "existing_pull_request", "existing_pr_url", "probe_dirty_worktree", "resolve_git_root", @@ -78,6 +81,7 @@ "run_pr_action", "run_review_action", "run_ship_action", + "update_pull_request_base", ] @@ -97,6 +101,7 @@ def interactive(self) -> bool: ReviewBaseResolutionError = review_base_support.ReviewBaseResolutionError OriginalPlanResolution = review_original_plan_support.OriginalPlanResolution + def _workflow_runner() -> ProjectActionWorkflowRunner: return workflow_factory.ProjectActionWorkflowFactory( git=workflow_factory.ProjectActionWorkflowGitSources( @@ -115,6 +120,8 @@ def _workflow_runner() -> ProjectActionWorkflowRunner: pull_request=workflow_factory.ProjectActionWorkflowPullRequestSources( resolve_base_branch_fn=_resolve_pr_base_branch, existing_pr_url_fn=existing_pr_url, + existing_pull_request_fn=existing_pull_request, + update_pull_request_base_fn=update_pull_request_base, probe_dirty_worktree_source_fn=probe_dirty_worktree, run_commit_action_fn=run_commit_action, pr_title_fn=_pr_title, @@ -279,8 +286,12 @@ def run_review_action(context: ActionProjectContext) -> int: DirtyWorktreeReport = git_state_support.DirtyWorktreeReport +ExistingPullRequest = git_state_support.ExistingPullRequest resolve_git_root = git_state_support.resolve_git_root _classify_dirty_porcelain = git_state_support.classify_dirty_porcelain +existing_pr_url = pr_git_support.existing_pr_url +existing_pull_request = pr_git_support.existing_pull_request +update_pull_request_base = pr_git_support.update_pull_request_base def probe_dirty_worktree(project_root: Path, repo_root: Path, *, project_name: str = "") -> DirtyWorktreeReport: @@ -296,15 +307,6 @@ def detect_default_branch(git_root: Path) -> str: return git_state_support.detect_default_branch(git_root, git_output=_git_output) -def existing_pr_url(git_root: Path, branch: str) -> str: - return git_state_support.existing_pr_url( - git_root, - branch, - gh_path=shutil.which("gh"), - run_process=subprocess.run, - ) - - _resolve_original_plan = review_original_plan_support.resolve_original_plan _read_worktree_provenance = review_original_plan_support.read_worktree_provenance _resolve_plan_file_from_record = review_original_plan_support.resolve_plan_file_from_record diff --git a/python/envctl_engine/actions/project_action_workflow_factory.py b/python/envctl_engine/actions/project_action_workflow_factory.py index 0b5d2fe4e..7258c7226 100644 --- a/python/envctl_engine/actions/project_action_workflow_factory.py +++ b/python/envctl_engine/actions/project_action_workflow_factory.py @@ -7,7 +7,7 @@ from typing import Any import envctl_engine.actions.action_ship_support as ship_support -from envctl_engine.actions.action_git_state_support import DirtyWorktreeReport +from envctl_engine.actions.action_git_state_support import DirtyWorktreeReport, ExistingPullRequest from envctl_engine.actions.action_review_context import ReviewActionContext from envctl_engine.actions.action_review_base_support import ReviewBaseResolution from envctl_engine.actions.action_review_original_plan_support import OriginalPlanResolution @@ -41,6 +41,8 @@ class ProjectActionWorkflowCommitSources: class ProjectActionWorkflowPullRequestSources: resolve_base_branch_fn: Callable[[Any, Path], str] existing_pr_url_fn: Callable[[Path, str], str] + existing_pull_request_fn: Callable[[Path, str], ExistingPullRequest | None] + update_pull_request_base_fn: Callable[[Path, ExistingPullRequest, str], subprocess.CompletedProcess[str]] probe_dirty_worktree_source_fn: Callable[..., DirtyWorktreeReport] run_commit_action_fn: Callable[[Any], int] pr_title_fn: Callable[[Any, Path, str], str] @@ -89,6 +91,8 @@ def build(self) -> ProjectActionWorkflowRunner: pull_request=ProjectActionPullRequestWorkflowDependencies( resolve_base_branch_fn=self.pull_request.resolve_base_branch_fn, existing_pr_url_fn=self.pull_request.existing_pr_url_fn, + existing_pull_request_fn=self.pull_request.existing_pull_request_fn, + update_pull_request_base_fn=self.pull_request.update_pull_request_base_fn, probe_dirty_worktree_fn=self.probe_dirty_worktree, run_commit_action_fn=self.pull_request.run_commit_action_fn, pr_title_fn=self.pull_request.pr_title_fn, diff --git a/python/envctl_engine/actions/project_action_workflows.py b/python/envctl_engine/actions/project_action_workflows.py index e516c7d4c..9cf3d43e6 100644 --- a/python/envctl_engine/actions/project_action_workflows.py +++ b/python/envctl_engine/actions/project_action_workflows.py @@ -7,6 +7,7 @@ from typing import Any import envctl_engine.actions.action_commit_support as commit_support +from envctl_engine.actions.action_git_state_support import ExistingPullRequest import envctl_engine.actions.action_pr_workflow_support as pr_workflow_support import envctl_engine.actions.action_review_workflow_support as review_workflow_support import envctl_engine.actions.action_ship_support as ship_support @@ -45,6 +46,8 @@ class ProjectActionCommitWorkflowDependencies: class ProjectActionPullRequestWorkflowDependencies: resolve_base_branch_fn: Callable[[Any, Path], str] existing_pr_url_fn: Callable[[Path, str], str] + existing_pull_request_fn: Callable[[Path, str], ExistingPullRequest | None] + update_pull_request_base_fn: Callable[[Path, ExistingPullRequest, str], subprocess.CompletedProcess[str]] probe_dirty_worktree_fn: Callable[[Path, Path, str], DirtyWorktreeReport] run_commit_action_fn: Callable[[Any], int] pr_title_fn: Callable[[Any, Path, str], str] @@ -126,7 +129,8 @@ def run_pr_action(self, context: Any) -> int: git_available=self.git_available, git_output_fn=self.git.git_output_fn, resolve_base_branch_fn=self.pull_request.resolve_base_branch_fn, - existing_pr_url_fn=self.pull_request.existing_pr_url_fn, + existing_pull_request_fn=self.pull_request.existing_pull_request_fn, + update_pull_request_base_fn=self.pull_request.update_pull_request_base_fn, probe_dirty_worktree_fn=self.pull_request.probe_dirty_worktree_fn, run_commit_action_fn=self.pull_request.run_commit_action_fn, pr_title_fn=self.pull_request.pr_title_fn, @@ -151,6 +155,7 @@ def run_ship_action(self, context: Any) -> int: add_ship_pr_label=self.pull_request.add_ship_pr_label_fn, probe_dirty_worktree=self._probe_dirty_worktree_for_ship, existing_pr_url=self.pull_request.existing_pr_url_fn, + existing_pull_request=self.pull_request.existing_pull_request_fn, partition_envctl_protected_paths=self.commit.partition_envctl_protected_paths_fn, ordered_unique_paths=self.commit.ordered_unique_paths_fn, github_pr_checks=self.pull_request.github_pr_checks_fn, diff --git a/python/envctl_engine/config/__init__.py b/python/envctl_engine/config/__init__.py index 00399fbe7..5f8fb637d 100644 --- a/python/envctl_engine/config/__init__.py +++ b/python/envctl_engine/config/__init__.py @@ -220,9 +220,6 @@ def load_config(env: Mapping[str, str] | None = None) -> EngineConfig: runtime_scope_id = _runtime_scope_id(base_dir=base_dir, env=env, resolved=resolved) runtime_scope_dir = runtime_dir / "python-engine" / runtime_scope_id - runtime_dir.mkdir(parents=True, exist_ok=True) - runtime_scope_dir.mkdir(parents=True, exist_ok=True) - dependency_ports: dict[str, dict[str, int]] = {} for definition in _dependency_definitions(): resources: dict[str, int] = {} @@ -283,9 +280,11 @@ def load_config(env: Mapping[str, str] | None = None) -> EngineConfig: plan_agent_cmux_workspace = str(resolved.get("ENVCTL_PLAN_AGENT_CMUX_WORKSPACE", "") or "").strip() plan_agent_superset_project = str(resolved.get("ENVCTL_PLAN_AGENT_SUPERSET_PROJECT", "") or "").strip() plan_agent_superset_workspace = str(resolved.get("ENVCTL_PLAN_AGENT_SUPERSET_WORKSPACE", "") or "").strip() - plan_agent_terminals_enable = parse_bool(resolved.get("ENVCTL_PLAN_AGENT_TERMINALS_ENABLE"), False) or bool( - plan_agent_cmux_workspace - ) or bool(plan_agent_superset_project or plan_agent_superset_workspace) + plan_agent_terminals_enable = ( + parse_bool(resolved.get("ENVCTL_PLAN_AGENT_TERMINALS_ENABLE"), False) + or bool(plan_agent_cmux_workspace) + or bool(plan_agent_superset_project or plan_agent_superset_workspace) + ) additional_services, additional_service_errors = _parse_additional_services(resolved) supabase_auth_users, supabase_auth_user_errors = _parse_supabase_auth_users(resolved) @@ -358,8 +357,7 @@ def load_config(env: Mapping[str, str] | None = None) -> EngineConfig: plan_agent_cli_cmd=str(resolved.get("ENVCTL_PLAN_AGENT_CLI_CMD", "") or "").strip(), plan_agent_cmux_workspace=plan_agent_cmux_workspace, plan_agent_surface_transport=( - str(resolved.get("ENVCTL_PLAN_AGENT_SURFACE_TRANSPORT", "cmux") or "cmux").strip().lower() - or "cmux" + str(resolved.get("ENVCTL_PLAN_AGENT_SURFACE_TRANSPORT", "cmux") or "cmux").strip().lower() or "cmux" ), plan_agent_superset_project=plan_agent_superset_project, plan_agent_superset_workspace=plan_agent_superset_workspace, diff --git a/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py b/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py index 70343d00e..99e6b5e41 100644 --- a/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py +++ b/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py @@ -7,6 +7,7 @@ from envctl_engine.planning.plan_agent.models import CreatedPlanWorktree from envctl_engine.runtime.runtime_context import optional_state_repository +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RunState @@ -16,7 +17,7 @@ class RuntimeAddressPromptBuilder: worktree: CreatedPlanWorktree | None = None def prompt_section(self) -> str: - state = _latest_runtime_state(self.runtime) + state = _latest_runtime_state(self.runtime, worktree=self.worktree) if state is None: return "" lines = [ @@ -56,11 +57,12 @@ def _runtime_addresses_prompt_section(runtime: Any, *, worktree: CreatedPlanWork return RuntimeAddressPromptBuilder(runtime, worktree=worktree).prompt_section() -def _latest_runtime_state(runtime: Any) -> RunState | None: +def _latest_runtime_state(runtime: Any, *, worktree: CreatedPlanWorktree | None = None) -> RunState | None: + project_names = [worktree.name] if worktree is not None and str(worktree.name).strip() else None state_repository = optional_state_repository(runtime) if state_repository is not None and hasattr(state_repository, "load_latest"): try: - state = state_repository.load_latest() + state = call_state_loader(state_repository.load_latest, project_names=project_names) except Exception: state = None if isinstance(state, RunState): @@ -69,7 +71,12 @@ def _latest_runtime_state(runtime: Any) -> RunState | None: if callable(try_loader): for mode in ("trees", "main"): try: - state = try_loader(mode=mode, strict_mode_match=True) + state = call_state_loader( + try_loader, + mode=mode, + strict_mode_match=True, + project_names=project_names, + ) except Exception: state = None if isinstance(state, RunState): diff --git a/python/envctl_engine/planning/worktree_provenance.py b/python/envctl_engine/planning/worktree_provenance.py index aaf59fce2..837aa1420 100644 --- a/python/envctl_engine/planning/worktree_provenance.py +++ b/python/envctl_engine/planning/worktree_provenance.py @@ -8,6 +8,7 @@ from envctl_engine.planning.worktree_code_intelligence_files import ensure_worktree_git_excludes from envctl_engine.runtime.codex_tmux_support import _tmux_session_exists from envctl_engine.runtime.runtime_context import resolve_process_runtime +from envctl_engine.shared.git_branch_support import detect_preferred_default_branch from envctl_engine.shared.parsing import parse_bool @@ -165,13 +166,10 @@ def resolve_branch_ref(self: Any, *, source_branch: str) -> str: def detect_default_branch(self: Any) -> str: - ref = git_command_output(self, ["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]).strip() - if ref.startswith("origin/"): - return ref.split("origin/", 1)[1] - for candidate in ("main", "master"): - if git_command_output(self, ["rev-parse", "--verify", candidate]).strip(): - return candidate - return "main" + return detect_preferred_default_branch( + lambda args: git_command_output(self, list(args)), + probe_remote_heads=True, + ) def git_command_output(self: Any, args: list[str]) -> str: @@ -179,11 +177,16 @@ def git_command_output(self: Any, args: list[str]) -> str: def git_command_output_at(self: Any, cwd: Path, args: list[str]) -> str: + command_env = dict(self._command_env(port=0)) + timeout = 30.0 + if args[:1] == ["ls-remote"]: + command_env["GIT_TERMINAL_PROMPT"] = "0" + timeout = 10.0 result = resolve_process_runtime(self).run( ["git", "-C", str(cwd), *args], cwd=cwd, - env=self._command_env(port=0), - timeout=30.0, + env=command_env, + timeout=timeout, ) if getattr(result, "returncode", 1) != 0: return "" diff --git a/python/envctl_engine/runtime/engine_runtime_artifacts.py b/python/envctl_engine/runtime/engine_runtime_artifacts.py index aaf7d9310..349d2801d 100644 --- a/python/envctl_engine/runtime/engine_runtime_artifacts.py +++ b/python/envctl_engine/runtime/engine_runtime_artifacts.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Any, Protocol +from envctl_engine.runtime.runtime_context import optional_state_repository from envctl_engine.runtime.runtime_readiness import ( FEATURE_MATRIX_RELATIVE_PATH, GAP_REPORT_RELATIVE_PATH, @@ -60,36 +61,37 @@ def write_runtime_readiness_report( *, run_dir: Path | None = None, readiness_result: RuntimeReadinessResult | None = None, -) -> None: +) -> bool: started = time.monotonic() emit = getattr(runtime, "_emit", None) cached = _cached_runtime_readiness_payload(runtime) if readiness_result is None and cached is not None: - _write_runtime_readiness_payload(runtime, report_text=cached, run_dir=run_dir) + persisted = _write_runtime_readiness_payload(runtime, report_text=cached, run_dir=run_dir) if callable(emit): emit( "artifacts.runtime_readiness_report", duration_ms=round((time.monotonic() - started) * 1000.0, 2), used_cached_contract=True, run_dir=str(run_dir) if run_dir is not None else None, + persisted=persisted, ) - return + return persisted readiness = ( - readiness_result - if readiness_result is not None - else evaluate_runtime_readiness(runtime.config.base_dir) + readiness_result if readiness_result is not None else evaluate_runtime_readiness(runtime.config.base_dir) ) report_payload = build_runtime_readiness_report(readiness) report_text = json.dumps(report_payload, indent=2, sort_keys=True) - _write_runtime_readiness_payload(runtime, report_text=report_text, run_dir=run_dir) + persisted = _write_runtime_readiness_payload(runtime, report_text=report_text, run_dir=run_dir) if callable(emit): emit( "artifacts.runtime_readiness_report", duration_ms=round((time.monotonic() - started) * 1000.0, 2), used_cached_contract=readiness_result is not None, run_dir=str(run_dir) if run_dir is not None else None, + persisted=persisted, ) + return persisted def _write_runtime_readiness_payload( @@ -97,12 +99,24 @@ def _write_runtime_readiness_payload( *, report_text: str, run_dir: Path | None, -) -> None: +) -> bool: + repository = optional_state_repository(runtime) + repository_writer = getattr(repository, "write_runtime_artifact", None) + if callable(repository_writer): + return bool( + repository_writer( + run_id=run_dir.name if run_dir is not None else None, + artifact_name="runtime_readiness_report.json", + text=report_text, + ) + ) + (runtime.runtime_root / "runtime_readiness_report.json").write_text(report_text, encoding="utf-8") runtime.runtime_legacy_root.mkdir(parents=True, exist_ok=True) (runtime.runtime_legacy_root / "runtime_readiness_report.json").write_text(report_text, encoding="utf-8") if run_dir is not None: (run_dir / "runtime_readiness_report.json").write_text(report_text, encoding="utf-8") + return True def _write_cached_runtime_readiness_payload( @@ -110,9 +124,9 @@ def _write_cached_runtime_readiness_payload( *, run_dir: Path, cached: str, -) -> None: +) -> bool: started = time.monotonic() - _write_runtime_readiness_payload(runtime, report_text=cached, run_dir=run_dir) + persisted = _write_runtime_readiness_payload(runtime, report_text=cached, run_dir=run_dir) emit = getattr(runtime, "_emit", None) if callable(emit): emit( @@ -120,10 +134,12 @@ def _write_cached_runtime_readiness_payload( duration_ms=round((time.monotonic() - started) * 1000.0, 2), used_cached_contract=True, run_dir=str(run_dir), + persisted=persisted, ) + return persisted -def _write_pending_runtime_readiness_payload(runtime: Any, *, run_dir: Path) -> None: +def _write_pending_runtime_readiness_payload(runtime: Any, *, run_dir: Path) -> bool: report = { "passed": None, "pending": True, @@ -139,23 +155,29 @@ def _write_pending_runtime_readiness_payload(runtime: Any, *, run_dir: Path) -> "errors": [], "warnings": [], } - _write_runtime_readiness_payload( + persisted = _write_runtime_readiness_payload( runtime, report_text=json.dumps(report, indent=2, sort_keys=True), run_dir=run_dir, ) emit = getattr(runtime, "_emit", None) if callable(emit): - emit("artifacts.runtime_readiness_report.pending", run_dir=str(run_dir)) + emit("artifacts.runtime_readiness_report.pending", run_dir=str(run_dir), persisted=persisted) + return persisted def _start_background_runtime_readiness_report(runtime: Any, *, run_dir: Path) -> None: def _worker() -> None: try: - write_runtime_readiness_report(runtime, run_dir=run_dir) + persisted = write_runtime_readiness_report(runtime, run_dir=run_dir) emit = getattr(runtime, "_emit", None) if callable(emit): - emit("artifacts.runtime_readiness_report.complete", run_dir=str(run_dir)) + event = ( + "artifacts.runtime_readiness_report.complete" + if persisted + else "artifacts.runtime_readiness_report.discarded" + ) + emit(event, run_dir=str(run_dir)) except Exception as exc: # pragma: no cover emit = getattr(runtime, "_emit", None) if callable(emit): diff --git a/python/envctl_engine/runtime/engine_runtime_construction.py b/python/envctl_engine/runtime/engine_runtime_construction.py index 70531d228..881773307 100644 --- a/python/envctl_engine/runtime/engine_runtime_construction.py +++ b/python/envctl_engine/runtime/engine_runtime_construction.py @@ -30,9 +30,23 @@ def initialize_runtime_construction(runtime: Any, config: EngineConfig, *, env: runtime.env = dict(env or {}) runtime.runtime_legacy_root = config.runtime_dir / "python-engine" runtime.runtime_root = config.runtime_scope_dir - runtime.runtime_legacy_root.mkdir(parents=True, exist_ok=True) - runtime.runtime_root.mkdir(parents=True, exist_ok=True) - runtime._ensure_legacy_lock_view() + compat_mode = runtime._state_compat_mode() + runtime.state_repository = RuntimeStateRepository( + runtime_root=runtime.runtime_root, + runtime_legacy_root=runtime.runtime_legacy_root, + runtime_dir=runtime.config.runtime_dir, + runtime_scope_id=runtime.config.runtime_scope_id, + compat_mode=compat_mode, + ) + # The repository canonicalizes ancestor symlinks before applying its + # containment and ownership checks. Keep every runtime consumer on that + # same canonical identity so persisted artifact paths and caller-visible + # paths never disagree (for example, macOS /var versus /private/var). + runtime.runtime_root = runtime.state_repository.runtime_root + runtime.runtime_legacy_root = runtime.state_repository.runtime_legacy_root + runtime.state_repository.ensure_runtime_roots() + if compat_mode == RuntimeStateRepository.COMPAT_READ_WRITE: + runtime._ensure_legacy_lock_view() runtime.port_planner = PortPlanner( backend_base=config.backend_port_base, frontend_base=config.frontend_port_base, @@ -61,7 +75,6 @@ def initialize_runtime_construction(runtime: Any, config: EngineConfig, *, env: ), ) runtime.requirements = RequirementsOrchestrator() - runtime.services = ServiceManager() runtime.events = [] runtime._emit_lock = threading.Lock() runtime._emit_listeners = [] @@ -69,9 +82,11 @@ def initialize_runtime_construction(runtime: Any, config: EngineConfig, *, env: runtime._startup_warnings_by_project = {} runtime._debug_hash_salt = uuid.uuid4().hex runtime._debug_recorder = None + runtime._debug_recorder_failure = None runtime._active_command_id = None runtime._last_debug_bundle_path = None runtime.process_runner = ProcessRunner(emit=runtime._emit) + runtime.services = ServiceManager(process_runner=runtime.process_runner) probe_backend_name = "psutil" if runtime._probe_psutil_enabled() else "shell" probe_backend = runtime._build_process_probe_backend() runtime.process_probe = ProcessProbe(probe_backend) @@ -89,13 +104,6 @@ def initialize_runtime_construction(runtime: Any, config: EngineConfig, *, env: "backend": runtime._conflict_count("BACKEND"), "frontend": runtime._conflict_count("FRONTEND"), } - runtime.state_repository = RuntimeStateRepository( - runtime_root=runtime.runtime_root, - runtime_legacy_root=runtime.runtime_legacy_root, - runtime_dir=runtime.config.runtime_dir, - runtime_scope_id=runtime.config.runtime_scope_id, - compat_mode=runtime._state_compat_mode(), - ) runtime.runtime_context = RuntimeContext( config=runtime.config, env=runtime.env, diff --git a/python/envctl_engine/runtime/engine_runtime_dispatch.py b/python/envctl_engine/runtime/engine_runtime_dispatch.py index 3254549a3..7d36cf34a 100644 --- a/python/envctl_engine/runtime/engine_runtime_dispatch.py +++ b/python/envctl_engine/runtime/engine_runtime_dispatch.py @@ -2,13 +2,35 @@ from typing import Any -from .command_policy import dispatch_family_for_command +from .command_policy import ( + LIFECYCLE_CLEANUP_COMMANDS, + STARTUP_COMMANDS, + dispatch_family_for_command, +) from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.inspection_support import dispatch_direct_inspection +from envctl_engine.runtime.lifecycle_operation_lease import run_exclusive_lifecycle_operation from envctl_engine.runtime.utility_command_support import dispatch_utility_command from envctl_engine.shared.process_probe import ProcessProbe +_EXCLUSIVE_OPERATION_COMMANDS = frozenset( + { + *LIFECYCLE_CLEANUP_COMMANDS, + *STARTUP_COMMANDS, + "blast-worktree", + "delete-worktree", + "ensure-worktree", + "resume", + "self-destruct-worktree", + } +) + + +def _requires_exclusive_operation(command: str) -> bool: + return command in _EXCLUSIVE_OPERATION_COMMANDS + + def dispatch(runtime: Any, route: Route) -> int: runtime.process_probe = ProcessProbe(runtime._build_process_probe_backend()) effective_mode = getattr(route, "mode", "") @@ -28,7 +50,14 @@ def dispatch(runtime: Any, route: Route) -> int: effective_mode=effective_mode, command=getattr(route, "command", ""), ) - return dispatch_command(runtime, route) + command = str(getattr(route, "command", "")).strip() + if not _requires_exclusive_operation(command): + return dispatch_command(runtime, route) + return run_exclusive_lifecycle_operation( + runtime, + command=command, + operation=lambda: dispatch_command(runtime, route), + ) def dispatch_command(runtime: Any, route: Route) -> int: diff --git a/python/envctl_engine/runtime/engine_runtime_event_support.py b/python/envctl_engine/runtime/engine_runtime_event_support.py index 53e2a9685..7c8bbce0b 100644 --- a/python/envctl_engine/runtime/engine_runtime_event_support.py +++ b/python/envctl_engine/runtime/engine_runtime_event_support.py @@ -6,7 +6,7 @@ from typing import Any, Mapping from envctl_engine.runtime.command_router import Route -from envctl_engine.runtime.runtime_context import run_dir_path +from envctl_engine.runtime.runtime_context import optional_state_repository, run_dir_path from envctl_engine.debug.debug_contract import apply_debug_event_contract from envctl_engine.debug.debug_utils import debug_env_value, hash_command, hash_value from envctl_engine.shared.parsing import parse_bool, parse_int @@ -70,7 +70,14 @@ def emit(runtime: Any, event_name: str, **payload: object) -> None: recorder_payload = dict(safe_payload) component_value = recorder_payload.pop("component", None) component: str | None = component_value if isinstance(component_value, str) else None - runtime._debug_recorder.record(event_name, component=component, **recorder_payload) + try: + runtime._debug_recorder.record(event_name, component=component, **recorder_payload) + except Exception as exc: # noqa: BLE001 + # Debug telemetry must never interrupt lifecycle ownership changes. + # Disable a failed recorder so a read-only diagnostic sink cannot + # repeatedly destabilize the command that it is observing. + runtime._debug_recorder_failure = str(exc) + runtime._debug_recorder = None for listener in list(runtime._emit_listeners): try: listener(event_name, dict(safe_payload)) @@ -112,6 +119,16 @@ def auto_debug_pack(runtime: Any, *, reason: str) -> None: def persist_events_snapshot(runtime: Any) -> None: events_text = "".join(json.dumps(event, sort_keys=True) + "\n" for event in runtime.events) + repository = optional_state_repository(runtime) + repository_writer = getattr(repository, "write_runtime_artifact", None) + if callable(repository_writer): + repository_writer( + run_id=_current_run_id(runtime), + artifact_name="events.jsonl", + text=events_text, + ) + return + (runtime.runtime_root / "events.jsonl").write_text(events_text, encoding="utf-8") runtime.runtime_legacy_root.mkdir(parents=True, exist_ok=True) (runtime.runtime_legacy_root / "events.jsonl").write_text(events_text, encoding="utf-8") @@ -121,6 +138,17 @@ def persist_events_snapshot(runtime: Any) -> None: def _current_run_dir(runtime: Any) -> Path | None: + run_id = _current_run_id(runtime) + if run_id is None: + return None + try: + candidate = run_dir_path(runtime, run_id) + except Exception: + return None + return candidate + + +def _current_run_id(runtime: Any) -> str | None: env = getattr(runtime, "env", None) if not isinstance(env, dict): return None @@ -130,11 +158,7 @@ def _current_run_dir(runtime: Any) -> Path | None: run_id = raw_run_id.strip() if not run_id: return None - try: - candidate = run_dir_path(runtime, run_id) - except Exception: - return None - return candidate + return run_id def current_session_id(runtime: Any) -> str | None: diff --git a/python/envctl_engine/runtime/engine_runtime_lifecycle_facade.py b/python/envctl_engine/runtime/engine_runtime_lifecycle_facade.py index 2764266c0..44f0984a1 100644 --- a/python/envctl_engine/runtime/engine_runtime_lifecycle_facade.py +++ b/python/envctl_engine/runtime/engine_runtime_lifecycle_facade.py @@ -27,8 +27,8 @@ class RuntimeLifecycleFacadeMixin: def _release_port_session(self) -> None: runtime_release_port_session(self) - def _clear_runtime_state(self, *, command: str, aggressive: bool = False, route: Route | None = None) -> None: - self.lifecycle_cleanup_orchestrator.clear_runtime_state( + def _clear_runtime_state(self, *, command: str, aggressive: bool = False, route: Route | None = None) -> bool: + return self.lifecycle_cleanup_orchestrator.clear_runtime_state( command=command, aggressive=aggressive, route=route, @@ -54,8 +54,8 @@ def _blast_all_kill_orchestrator_processes(self) -> None: def _blast_all_docker_cleanup(self, *, route: Route | None) -> int: return self.lifecycle_cleanup_orchestrator.blast_all_docker_cleanup(route=route) - def _terminate_started_services(self, services: dict[str, object]) -> None: - runtime_terminate_started_services(self, services) + def _terminate_started_services(self, services: dict[str, object]) -> set[str]: + return runtime_terminate_started_services(self, services) def _terminate_services_from_state( self, @@ -64,8 +64,8 @@ def _terminate_services_from_state( selected_services: set[str] | None, aggressive: bool, verify_ownership: bool, - ) -> None: - runtime_terminate_services_from_state( + ) -> set[str]: + return runtime_terminate_services_from_state( self, state, selected_services=selected_services, diff --git a/python/envctl_engine/runtime/engine_runtime_startup_facade.py b/python/envctl_engine/runtime/engine_runtime_startup_facade.py index bd84ea62d..2ed209e17 100644 --- a/python/envctl_engine/runtime/engine_runtime_startup_facade.py +++ b/python/envctl_engine/runtime/engine_runtime_startup_facade.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Sequence from pathlib import Path from typing import Any, Mapping, cast @@ -208,10 +209,21 @@ def _write_runtime_readiness_report( ) -> None: runtime_write_runtime_readiness_report(self, run_dir=run_dir, readiness_result=readiness_result) - def _try_load_existing_state(self, *, mode: str | None = None, strict_mode_match: bool = False) -> RunState | None: + def _try_load_existing_state( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, + ) -> RunState | None: return cast( RunState | None, - runtime_try_load_existing_state(self, mode=mode, strict_mode_match=strict_mode_match), + runtime_try_load_existing_state( + self, + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ), ) def _state_matches_scope(self, state: RunState) -> bool: diff --git a/python/envctl_engine/runtime/engine_runtime_startup_support.py b/python/envctl_engine/runtime/engine_runtime_startup_support.py index 9a4abb529..e118ce4aa 100644 --- a/python/envctl_engine/runtime/engine_runtime_startup_support.py +++ b/python/envctl_engine/runtime/engine_runtime_startup_support.py @@ -1,8 +1,10 @@ from __future__ import annotations +import threading from pathlib import Path from typing import Any, Callable, Mapping +from envctl_engine.requirements.core import dependency_definitions from envctl_engine.runtime.command_router import Route from envctl_engine.startup.run_reuse_support import ( build_startup_identity_metadata as build_startup_identity_metadata_impl, @@ -11,11 +13,17 @@ state_has_resumable_services as state_has_resumable_services_impl, ) from envctl_engine.startup.protocols import ProjectContextLike +from envctl_engine.startup.service_execution_environment import configured_service_types_for_mode +from envctl_engine.startup.service_execution_policy import additional_service_enabled_for_context +from envctl_engine.startup.startup_selection_support import _restart_service_types_for_project from envctl_engine.state.models import PortPlan, RunState from envctl_engine.shared.parsing import parse_bool, parse_int from envctl_engine.planning import discover_tree_projects +_PORT_REAP_LOCK = threading.Lock() + + def effective_start_mode(runtime: Any, route: Route) -> str: if route.mode == "main" and runtime._setup_worktree_requested(route): return "trees" @@ -197,7 +205,22 @@ def discover_projects( def reserve_project_ports(runtime: Any, context: ProjectContextLike, route: Route | None = None) -> None: + _reap_stale_port_locks_once(runtime) + selected_app_services = _selected_app_services_for_port_reservation(runtime, context=context, route=route) for service_name, plan in context.ports.items(): + if not _port_reservation_enabled( + runtime, + service_name=service_name, + route=route, + selected_app_services=selected_app_services, + ): + runtime._emit( + "port.reservation.skipped", + project=context.name, + service=service_name, + reason="startup_surface_disabled", + ) + continue owner = f"{context.name}:{service_name}" restart_preferred_port = plan.assigned if getattr(plan, "source", "") == "restart" else None restart_conflict_detail = None @@ -245,6 +268,80 @@ def reserve_project_ports(runtime: Any, context: ProjectContextLike, route: Rout runtime._emit("port.reserved", project=context.name, service=service_name, port=reserved) +def _reap_stale_port_locks_once(runtime: Any) -> None: + reaper = getattr(runtime.port_planner, "reap_stale_locks", None) + if not callable(reaper) or bool(getattr(runtime, "_startup_stale_port_locks_reaped", False)): + return + with _PORT_REAP_LOCK: + if bool(getattr(runtime, "_startup_stale_port_locks_reaped", False)): + return + reaped = int(reaper() or 0) + setattr(runtime, "_startup_stale_port_locks_reaped", True) + runtime._emit("port.lock.reap_stale", count=reaped) + + +def _port_reservation_enabled( + runtime: Any, + *, + service_name: str, + route: Route | None, + selected_app_services: set[str] | None, +) -> bool: + if route is None: + return True + + dependency_ids = { + definition.id + for definition in dependency_definitions() + if any(resource.legacy_port_key == service_name for resource in definition.resources) + } + if dependency_ids: + if route.flags.get("launch_dependencies") is False: + return False + enabled = getattr(runtime, "_requirement_enabled", None) + if not callable(enabled): + return True + return any(bool(enabled(dependency_id, mode=route.mode, route=route)) for dependency_id in dependency_ids) + + return selected_app_services is None or service_name in selected_app_services + + +def _selected_app_services_for_port_reservation( + runtime: Any, + *, + context: ProjectContextLike, + route: Route | None, +) -> set[str] | None: + if route is None: + return None + try: + configured_service_types = configured_service_types_for_mode( + runtime, + route.mode, + Path(context.root), + ) + except AttributeError: + dependency_port_names = { + resource.legacy_port_key for definition in dependency_definitions() for resource in definition.resources + } + configured_service_types = set(context.ports).difference(dependency_port_names) + configured_additional_services = tuple( + service + for service in getattr(getattr(runtime, "config", None), "additional_services", ()) + if additional_service_enabled_for_context( + service, + mode=route.mode, + project_root=Path(context.root), + ) + ) + return _restart_service_types_for_project( + route=route, + project_name=context.name, + default_service_types=configured_service_types, + additional_services=configured_additional_services, + ) + + def _release_restart_preferred_lock(port_planner: Any, port: int, *, owner: str) -> None: release = getattr(port_planner, "release", None) if not callable(release): diff --git a/python/envctl_engine/runtime/engine_runtime_state_lookup.py b/python/envctl_engine/runtime/engine_runtime_state_lookup.py index 6aa0a3713..5f1f82198 100644 --- a/python/envctl_engine/runtime/engine_runtime_state_lookup.py +++ b/python/envctl_engine/runtime/engine_runtime_state_lookup.py @@ -1,15 +1,24 @@ from __future__ import annotations +from collections.abc import Sequence from typing import Any +from envctl_engine.state.lookup import call_state_loader + def try_load_existing_state( runtime: Any, *, mode: str | None = None, strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, ) -> object | None: - state = runtime.state_repository.load_latest(mode=mode, strict_mode_match=strict_mode_match) + state = call_state_loader( + runtime.state_repository.load_latest, + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ) if state is not None: runtime._emit( "state.fingerprint.after_reload", diff --git a/python/envctl_engine/runtime/help_topic_actions.py b/python/envctl_engine/runtime/help_topic_actions.py index 84b358a6c..74e8f7e3b 100644 --- a/python/envctl_engine/runtime/help_topic_actions.py +++ b/python/envctl_engine/runtime/help_topic_actions.py @@ -68,6 +68,7 @@ usage=("envctl pr [--project ] [--pr-base ]",), what_it_does=( "resolves the current branch, checks for an existing PR, and creates one with gh or repo helper scripts", + "defaults the PR base to dev, then main, then master, while --pr-base remains an explicit override", "commits and pushes dirty worktree changes first using envctl commit behavior when needed", "uses MAIN_TASK.md or commit history to build a useful PR title/body unless explicit env is provided", ), @@ -76,7 +77,7 @@ "--pr-base set the target base branch", "--interactive opt into prompts/selection instead of default headless action mode", ), - examples=("envctl pr --project feature-a-1 --pr-base main", "envctl pr --main --pr-base main"), + examples=("envctl pr --project feature-a-1", "envctl pr --main --pr-base main"), aliases=("prs", "p", "--pr", "--prs"), related=("commit", "review", "test"), ), diff --git a/python/envctl_engine/runtime/help_topic_lifecycle.py b/python/envctl_engine/runtime/help_topic_lifecycle.py index 4de800366..9d6b2c8a6 100644 --- a/python/envctl_engine/runtime/help_topic_lifecycle.py +++ b/python/envctl_engine/runtime/help_topic_lifecycle.py @@ -39,9 +39,7 @@ "restart": CommandHelpTopic( command="restart", summary="restart selected envctl-managed services", - usage=( - "envctl restart [--project |--service |--all] [runtime scope] [--headless]", - ), + usage=("envctl restart [--project |--service |--all] [runtime scope] [--headless]",), what_it_does=( "loads saved runtime state, stops the selected services, and starts them again", "keeps the same target model as startup while using existing envctl state as the source of truth", @@ -95,12 +93,17 @@ "stop": CommandHelpTopic( command="stop", summary="stop selected envctl-managed services and update runtime state", - usage=("envctl stop [--project |--service |--all] [runtime scope]",), + usage=("envctl stop [--main|--trees] [--project |--service |--all] [runtime scope]",), what_it_does=( "loads saved runtime state and terminates selected services", - "supports service-scope flags so you can stop backend, frontend, dependencies, or the entire system", + ( + "dependency scope releases saved dependency records and port locks; " + "managed Docker stacks and storage remain reusable" + ), + "use stop-all --remove-volumes or blast-all when managed Docker containers must be removed", ), flags=( + "--main | --trees choose the saved runtime mode; an explicit mode leaves the other mode active", "--project stop services for selected project/worktree", "--service stop one saved service", "--all stop all matching saved services", @@ -117,14 +120,19 @@ "stop-all": CommandHelpTopic( command="stop-all", summary="stop every saved envctl-managed service in the selected scope", - usage=("envctl stop-all [runtime scope]", "envctl kill-all [runtime scope]"), + usage=( + "envctl stop-all [--main|--trees] [runtime scope]", + "envctl kill-all [--main|--trees] [runtime scope]", + ), what_it_does=( - "terminates all saved services for the selected runtime state", - "can optionally remove volumes when supported by the selected cleanup flags", + "terminates all saved app services and releases dependency records and port locks", + "keeps managed Docker dependency stacks and storage reusable unless explicit removal cleanup is requested", + "use --remove-volumes for supported Docker removal, or blast-all for aggressive cleanup", ), flags=( + "--main | --trees choose the saved runtime mode; an explicit mode leaves the other mode active", "--backend|--frontend|--fullstack|--dependencies|--entire-system choose cleanup scope", - "--stop-all-remove-volumes / --remove-volumes remove supported volumes too", + "--stop-all-remove-volumes / --remove-volumes remove supported managed containers and volumes", ), examples=("envctl stop-all", "envctl kill-all", "envctl stop-all --entire-system --remove-volumes"), aliases=("kill-all", "killall", "stopall", "--stop-all", "--kill-all"), diff --git a/python/envctl_engine/runtime/inspection_support.py b/python/envctl_engine/runtime/inspection_support.py index a83f6c337..08d6f88b1 100644 --- a/python/envctl_engine/runtime/inspection_support.py +++ b/python/envctl_engine/runtime/inspection_support.py @@ -10,6 +10,11 @@ from envctl_engine.config.persistence import managed_values_from_local_state, managed_values_to_payload from envctl_engine.planning import PlanProjectPrediction from envctl_engine.runtime.command_router import Route, list_supported_commands +from envctl_engine.runtime.state_inspection_support import ( + emit_cwd_runtime_mismatch_warnings as _emit_cwd_runtime_mismatch_warnings_impl, + print_state as _print_state_impl, + try_load_existing_state as _try_load_existing_state_impl, +) from envctl_engine.startup.startup_selection_support import ( _state_project_names as _state_project_names_impl, _tree_preselected_projects_from_state as _tree_preselected_projects_from_state_impl, @@ -20,15 +25,7 @@ print_attach_command, kill_session, ) -from envctl_engine.state import state_to_dict from envctl_engine.state.models import RunState -from envctl_engine.state.project_runtime import ( - cwd_runtime_warnings, - dependency_mode_summary, - project_resolution_event_payload, - resolve_requested_project_state, -) -from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.runtime.endpoints_command_support import run_endpoints_command from envctl_engine.runtime.startup_inspection_support import ( additional_service_contexts as _additional_service_contexts_impl, @@ -87,14 +84,35 @@ def _print_targets(runtime: Any, route: Route, *, trees_only: bool) -> int: preselected: set[str] = set() state_present: set[str] = set() services_running: set[str] = set() + state_error: dict[str, str] | None = None if mode == "trees": - startup = getattr(runtime, "startup_orchestrator", None) - if startup is not None: - preselected = set(_tree_preselected_projects_from_state_impl(runtime=runtime, project_contexts=projects)) - state = _try_load_existing_state(runtime, mode="trees", strict_mode_match=True) - if state is not None: - state_present = set(_state_project_names_impl(runtime=runtime, state=state)) - services_running = _running_tree_project_names(runtime=runtime, state=state) + states, load_error = _load_all_existing_states(runtime, mode="trees") + if load_error is not None: + state_error = { + "code": "runtime_state_integrity_error", + "message": load_error, + } + else: + startup = getattr(runtime, "startup_orchestrator", None) + if startup is not None: + try: + preselected = set( + _tree_preselected_projects_from_state_impl(runtime=runtime, project_contexts=projects) + ) + except RuntimeError as exc: + # The selection helper performs its own strict state load. If + # the repository changes between the two reads, keep this + # inspection command useful without presenting partial state + # as trustworthy. + states = [] + preselected.clear() + state_error = { + "code": "runtime_state_integrity_error", + "message": str(exc), + } + for state in states: + state_present.update(_state_project_names_impl(runtime=runtime, state=state)) + services_running.update(_running_tree_project_names(runtime=runtime, state=state)) payload = { "mode": mode, @@ -112,6 +130,8 @@ def _print_targets(runtime: Any, route: Route, *, trees_only: bool) -> int: "running": project.name in services_running, }, ), + "state_error": state_error, + "warnings": [state_error] if state_error is not None else [], } print(json.dumps(payload, indent=2, sort_keys=True)) return 0 @@ -317,87 +337,42 @@ def _apply_runtime_effective_config_values(values: Any, config: Any) -> None: def _print_state(runtime: Any, route: Route, *, json_output: bool) -> int: - strict = runtime._state_lookup_strict_mode_match(route) - state = _try_load_existing_state(runtime, mode=route.mode, strict_mode_match=strict) - if state is None: - if json_output: - print(json.dumps({"found": False, "mode": getattr(route, "mode", None)}, indent=2, sort_keys=True)) - else: - print("No previous state found.") - return 1 - requested_projects = list(getattr(route, "projects", []) or []) - if requested_projects: - resolution = resolve_requested_project_state( - state, - requested_projects, - command="show-state", - runtime=runtime, - allow_multi=False, - ) - if not resolution.ok: - emitter = getattr(runtime, "_emit", None) - if callable(emitter): - emitter( - "state.project_resolution.failed", - **project_resolution_event_payload(resolution, state, runtime=runtime), - ) - if json_output: - print(json.dumps(resolution.payload(), indent=2, sort_keys=True)) - else: - print(f"Requested project is not running: {resolution.requested_project}") - print("Active runtime projects: " + (", ".join(resolution.active_projects) or "none")) - return 1 - emitter = getattr(runtime, "_emit", None) - if callable(emitter): - emitter( - "state.project_resolution.ok", - **project_resolution_event_payload(resolution, state, runtime=runtime), - ) - if resolution.state is not None and not bool(getattr(route, "flags", {}).get("all")): - state = resolution.state - - dependency_summary = dependency_mode_summary(state) - cwd_project, warnings = cwd_runtime_warnings( - state, - runtime=runtime, - requested_projects=list(getattr(route, "projects", []) or []), + return _print_state_impl( + runtime, + route, + json_output=json_output, + state_loader=_try_load_existing_state, + warning_emitter=_emit_cwd_runtime_mismatch_warnings, ) - _emit_cwd_runtime_mismatch_warnings(runtime, command="show-state", state=state, warnings=warnings) - payload: dict[str, Any] = { - "found": True, - "runtime_root": str(runtime.runtime_root), - "run_state_path": str(runtime._run_state_path()), - "state": state_to_dict(state), - "runtime_map": build_runtime_map(state), - "dependency_mode": dependency_summary["dependency_mode"], - "shared_dependencies": dependency_summary["shared_dependencies"], - "cwd_project": cwd_project, - "warnings": warnings, - } - if json_output: - print(json.dumps(payload, indent=2, sort_keys=True)) - return 0 - - state_payload = payload["state"] - print(f"run_id: {state_payload['run_id']}") - print(f"mode: {state_payload['mode']}") - print(f"services: {len(state_payload['services'])}") - print(f"requirements: {len(state_payload['requirements'])}") - for warning in warnings: - print(f"Warning: {warning['message']}") - print("run_state_path:") - print(render_path_for_terminal(payload["run_state_path"], env=getattr(runtime, "env", {}))) - return 0 +def _try_load_existing_state( + runtime: Any, + *, + mode: str | None, + strict_mode_match: bool, + project_names: list[str] | None = None, +) -> RunState | None: + return _try_load_existing_state_impl( + runtime, + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ) -def _try_load_existing_state(runtime: Any, *, mode: str | None, strict_mode_match: bool) -> RunState | None: - loader = getattr(runtime, "_try_load_existing_state", None) - if not callable(loader): - return None - state = loader(mode=mode, strict_mode_match=strict_mode_match) - return state if isinstance(state, RunState) else None +def _load_all_existing_states(runtime: Any, *, mode: str) -> tuple[list[RunState], str | None]: + repository = getattr(runtime, "state_repository", None) + load_all = getattr(repository, "load_all", None) + try: + if callable(load_all): + states = [state for state in load_all(mode=mode) if isinstance(state, RunState)] + if states: + return states, None + state = _try_load_existing_state(runtime, mode=mode, strict_mode_match=True) + except RuntimeError as exc: + return [], str(exc) + return ([state] if state is not None else []), None def _emit_cwd_runtime_mismatch_warnings( @@ -407,20 +382,12 @@ def _emit_cwd_runtime_mismatch_warnings( state: RunState, warnings: list[dict[str, object]], ) -> None: - emitter = getattr(runtime, "_emit", None) - if not callable(emitter): - return - for warning in warnings: - if str(warning.get("code") or "") != "cwd_runtime_mismatch": - continue - emitter( - "state.cwd_runtime_mismatch", - run_id=getattr(state, "run_id", ""), - mode=getattr(state, "mode", ""), - command=command, - cwd_project=warning.get("cwd_project"), - active_projects=warning.get("active_projects"), - ) + _emit_cwd_runtime_mismatch_warnings_impl( + runtime, + command=command, + state=state, + warnings=warnings, + ) def _additional_service_enabled_for_context(runtime: Any, mode: str, service: object, context: object) -> bool: diff --git a/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py b/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py index b2680e985..17cdac4a6 100644 --- a/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py +++ b/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py @@ -11,6 +11,7 @@ requirements_have_enabled_components, select_dependency_components_for_stop, ) +from envctl_engine.runtime.lifecycle_requirement_ports import requirement_component_port_owners from envctl_engine.runtime.lifecycle_service_stop_selection import ( select_services_for_stop, service_matches_runtime_scope, @@ -23,15 +24,17 @@ service_type_from_stopped_service, should_preserve_stopped_dashboard_state, ) +from envctl_engine.runtime.lifecycle_targeted_state import LifecycleTargetedStateSupport from envctl_engine.runtime.runtime_context import ( resolve_port_allocator, resolve_process_runtime, resolve_state_repository, ) from envctl_engine.shared.protocols import PortAllocator, ProcessRuntime, StateRepository -from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.runtime.command_router import Route +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RequirementsResult, RunState +from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.ui.selection_support import ( interactive_selection_allowed, project_names_from_state, @@ -43,7 +46,7 @@ from envctl_engine.ui.spinner_service import emit_spinner_policy, resolve_spinner_policy -class LifecycleCleanupOrchestrator(LifecycleBlastCleanupSupport): +class LifecycleCleanupOrchestrator(LifecycleTargetedStateSupport, LifecycleBlastCleanupSupport): def __init__(self, runtime: Any) -> None: self.runtime = runtime @@ -54,13 +57,16 @@ def execute(self, route: Route) -> int: def _execute_full_cleanup(self, route: Route) -> int: def operation() -> int: - self.clear_runtime_state( + completed = self.clear_runtime_state( command=route.command, aggressive=(route.command == "blast-all"), route=route, ) - print("Stopped runtime state.") - return 0 + if completed: + print("Stopped runtime state.") + return 0 + print("Runtime cleanup is incomplete; failed services remain tracked.") + return 1 message = ( "Stopping all services and runtime state..." @@ -75,77 +81,197 @@ def operation() -> int: ) def _execute_stop(self, route: Route) -> int: - state = self.runtime._try_load_existing_state( # type: ignore[attr-defined] + targeted_states = self._targeted_active_states(route) + if targeted_states: + result = 0 + for state in targeted_states: + result = max(result, self._execute_stop_state(route, state)) + return result + + state = call_state_loader( + self.runtime._try_load_existing_state, # type: ignore[attr-defined] mode=route.mode, strict_mode_match=self.runtime._state_lookup_strict_mode_match(route), # type: ignore[attr-defined] + project_names=route.projects or None, ) if state is None: print("No active runtime state found.") return 0 + return self._execute_stop_state(route, state) + + def _targeted_active_states(self, route: Route) -> list[RunState]: + selected_projects = {str(project).strip().casefold() for project in route.projects if str(project).strip()} + if not selected_projects: + return [] + repository = self._state_repository(self.runtime) + load_all = getattr(repository, "load_all", None) + if not callable(load_all): + return [] + matching_states = self._states_matching_projects( + [state for state in load_all(mode=route.mode) if isinstance(state, RunState)], + selected_projects, + ) + if matching_states: + return matching_states + if self.runtime._state_lookup_strict_mode_match(route): # type: ignore[attr-defined] + return [] + + fallback_state = call_state_loader( + self.runtime._try_load_existing_state, # type: ignore[attr-defined] + mode=route.mode, + strict_mode_match=False, + project_names=route.projects, + ) + if not isinstance(fallback_state, RunState) or fallback_state.mode == route.mode: + return [] + return self._states_matching_projects( + [state for state in load_all(mode=fallback_state.mode) if isinstance(state, RunState)], + selected_projects, + ) + + def _states_matching_projects( + self, + states: Iterable[RunState], + selected_projects: set[str], + ) -> list[RunState]: + return [ + state + for state in states + if selected_projects.intersection( + str(getattr(project, "name", project)).strip().casefold() + for project in self._project_names_from_state(state) + ) + ] + + def _execute_stop_state(self, route: Route, state: RunState) -> int: + if route.flags.get("runtime_scope") == "dependencies": return self._execute_stop_dependencies(route, state) + owned_projects = self._owned_project_names(state) + targeted_project_keys = frozenset( + str(project).strip().casefold() + for project in route.projects + if str(project).strip().casefold() in owned_projects + ) preserve_stopped_dashboard_state = self._should_preserve_stopped_dashboard_state(route) selected_dependencies = self._select_dependency_components_for_stop(state, route) selected = self._select_services_for_stop(state, route) if not selected and not selected_dependencies: - if not state.services and not state.requirements: - self.clear_runtime_state(command="stop", aggressive=False) + if targeted_project_keys and self._targeted_projects_have_no_managed_resources( + state, + targeted_project_keys, + ): + self._remove_targeted_requirements(state, targeted_project_keys) + state_remains = self._persist_targeted_project_state( + state, + owned_projects=owned_projects, + targeted_project_keys=targeted_project_keys, + ) + print("Stopped selected project state." if state_remains else "Stopped runtime state.") + return 0 + has_enabled_requirements = any( + self._requirements_have_enabled_components(requirements) for requirements in state.requirements.values() + ) + if not state.services and not has_enabled_requirements: + self._deactivate_runtime_state(state) print("Stopped runtime state.") return 0 print("No matching services selected for stop.") return 1 def operation() -> int: - if selected and preserve_stopped_dashboard_state: - self._remember_dashboard_stopped_services(state, selected) - + self.runtime._emit( # type: ignore[attr-defined] + "cleanup.stop", + aggressive=False, + run_id=state.run_id, + ) + failed_services: set[str] = set() if selected: - self.runtime._terminate_services_from_state( # type: ignore[attr-defined] + termination_result = self.runtime._terminate_services_from_state( # type: ignore[attr-defined] state, selected_services=selected, aggressive=False, verify_ownership=True, ) + if isinstance(termination_result, set): + failed_services = {str(name) for name in termination_result} - for service_name in list(selected): + stopped_services = selected.difference(failed_services) + if stopped_services and preserve_stopped_dashboard_state: + self._remember_dashboard_stopped_services(state, stopped_services) + + for service_name in stopped_services: state.services.pop(service_name, None) - if selected_dependencies: + if selected_dependencies and not failed_services: self._release_selected_dependency_components(state, selected_dependencies) - if not bool(route.flags.get("stop_preserve_requirements")): + if not failed_services and not bool(route.flags.get("stop_preserve_requirements")): remaining_projects = { - self.runtime._project_name_from_service(name) # type: ignore[attr-defined] - for name in state.services + self._project_name_for_service(name, service) for name, service in state.services.items() } - removed_projects = [project for project in state.requirements if project not in remaining_projects] + removed_projects = [ + project + for project, requirements in state.requirements.items() + if self._requirement_project_key(project, requirements) not in remaining_projects + and ( + not targeted_project_keys + or self._requirement_project_key(project, requirements) in targeted_project_keys + ) + ] for project in removed_projects: self.runtime._release_requirement_ports(state.requirements[project]) # type: ignore[attr-defined] state.requirements.pop(project, None) + if targeted_project_keys: + self._persist_targeted_project_state( + state, + owned_projects=owned_projects, + targeted_project_keys=targeted_project_keys, + ) + if failed_services: + print("Could not safely stop: " + ", ".join(sorted(failed_services))) + return 1 + if stopped_services and selected_dependencies: + print("Stopped selected services and dependencies.") + elif selected_dependencies: + print("Stopped selected dependencies.") + elif stopped_services: + print("Stopped selected services.") + else: + print("Stopped selected project state.") + return 0 + if not state.services and not state.requirements: if preserve_stopped_dashboard_state and self._has_dashboard_stopped_services(state): self._save_selected_stop_state(state) - if selected and selected_dependencies: + if stopped_services and selected_dependencies: print("Stopped selected services and dependencies.") elif selected_dependencies: print("Stopped selected dependencies.") - else: + elif stopped_services: print("Stopped selected services.") + else: + print("No selected service could be stopped.") return 0 - self.clear_runtime_state(command="stop", aggressive=False) + self._deactivate_runtime_state(state) print("Stopped runtime state.") return 0 self._save_selected_stop_state(state) - if selected and selected_dependencies: + if failed_services: + print("Could not safely stop: " + ", ".join(sorted(failed_services))) + return 1 + if stopped_services and selected_dependencies: print("Stopped selected services and dependencies.") elif selected_dependencies: print("Stopped selected dependencies.") - else: + elif stopped_services: print("Stopped selected services.") + else: + print("Stopped selected dependencies.") return 0 selected_count = len(selected) + sum(len(items) for items in selected_dependencies.values()) @@ -218,7 +344,7 @@ def _execute_stop_dependencies(self, route: Route, state: RunState) -> int: def operation() -> int: if not state.requirements: if not state.services: - self._purge_runtime_state(aggressive=False) + self._deactivate_runtime_state(state) print("No dependencies found.") return 0 @@ -229,7 +355,7 @@ def operation() -> int: if state.services: self._save_selected_stop_state(state) else: - self._purge_runtime_state(aggressive=False) + self._deactivate_runtime_state(state) print("Stopped dependencies.") return 0 @@ -255,9 +381,18 @@ def _release_selected_dependency_components( release_component_ports_fn=self._release_requirement_component_ports, ) - def _release_requirement_component_ports(self, component: Mapping[str, object]) -> None: + def _release_requirement_component_ports( + self, + requirements: RequirementsResult, + dependency_id: str, + component: Mapping[str, object], + ) -> None: port_allocator = self._port_allocator(self.runtime) - release_requirement_component_ports(component, release_port_fn=port_allocator.release) + release_requirement_component_ports( + component, + port_planner=port_allocator, + owner_candidates=requirement_component_port_owners(requirements, dependency_id), + ) @staticmethod def _requirements_have_enabled_components(requirements: object) -> bool: @@ -296,7 +431,7 @@ def _project_names_from_state(self, state: RunState) -> list[object]: def _services_from_selection(self, selection: TargetSelection, state: RunState) -> set[str]: return services_from_selection(self.runtime, selection, state) - def clear_runtime_state(self, *, command: str, aggressive: bool = False, route: Route | None = None) -> None: + def clear_runtime_state(self, *, command: str, aggressive: bool = False, route: Route | None = None) -> bool: rt = self.runtime if command == "stop-all": rt._emit("cleanup.stop_all", aggressive=aggressive) # type: ignore[attr-defined] @@ -304,14 +439,71 @@ def clear_runtime_state(self, *, command: str, aggressive: bool = False, route: rt._emit("cleanup.blast", aggressive=aggressive) # type: ignore[attr-defined] else: rt._emit("cleanup.stop", aggressive=aggressive) # type: ignore[attr-defined] - state = rt._try_load_existing_state() # type: ignore[attr-defined] - if state is not None: - rt._terminate_services_from_state( # type: ignore[attr-defined] + state_repository = self._state_repository(rt) + load_all = getattr(state_repository, "load_all", None) + mode_filter = route.mode if command == "stop-all" and route is not None else None + if callable(load_all): + states = [state for state in load_all(mode=mode_filter) if isinstance(state, RunState)] + else: + states = [] + if not states: + state = call_state_loader( + rt._try_load_existing_state, # type: ignore[attr-defined] + mode=mode_filter, + strict_mode_match=mode_filter is not None, + ) + states = [state] if isinstance(state, RunState) else [] + seen_run_ids: set[str] = set() + failed_by_run: dict[str, set[str]] = {} + for state in states: + if state.run_id in seen_run_ids: + continue + seen_run_ids.add(state.run_id) + termination_result = rt._terminate_services_from_state( # type: ignore[attr-defined] state, selected_services=None, aggressive=aggressive, verify_ownership=not aggressive, ) + failed_services = ( + {str(name) for name in termination_result} if isinstance(termination_result, set) else set() + ) + if not failed_services: + requirements_to_release = list(state.requirements.values()) + self._deactivate_runtime_state(state, release_session_when_empty=False) + for requirements in requirements_to_release: + self._release_requirement_ports(requirements) + continue + failed_by_run[state.run_id] = failed_services + state.services = {name: service for name, service in state.services.items() if name in failed_services} + failed_projects = { + project_name + for name, service in state.services.items() + if (project_name := self._project_name_for_service(name, service)) + } + requirements_to_release: list[RequirementsResult] = [] + retained_requirements: dict[str, RequirementsResult] = {} + for project, requirements in state.requirements.items(): + requirement_project = str(getattr(requirements, "project", "") or project).strip().casefold() + if not failed_projects or requirement_project in failed_projects: + retained_requirements[project] = requirements + else: + requirements_to_release.append(requirements) + state.requirements = {project: requirements for project, requirements in retained_requirements.items()} + state_repository.save_selected_stop_state( + state=state, + emit=rt._emit, # type: ignore[attr-defined] + runtime_map_builder=lambda raw_state: build_runtime_map(cast(RunState, raw_state)), + ) + for requirements in requirements_to_release: + self._release_requirement_ports(requirements) + + if failed_by_run: + rt._emit( # type: ignore[attr-defined] + "cleanup.incomplete", + failed_services={run_id: sorted(names) for run_id, names in failed_by_run.items()}, + ) + return False if aggressive and self.blast_all_ecosystem_enabled(): self.blast_all_ecosystem_cleanup(route=route) @@ -332,25 +524,68 @@ def clear_runtime_state(self, *, command: str, aggressive: bool = False, route: removed = self.blast_all_docker_cleanup(route=volume_route) rt._emit("cleanup.stop_all.remove_volumes.finish", removed=removed) # type: ignore[attr-defined] - self._purge_runtime_state(aggressive=aggressive) - if aggressive: + if aggressive or command != "stop-all" or not self._has_active_runs(state_repository): + self._purge_runtime_state(aggressive=aggressive) + repository = self._state_repository(rt) + if aggressive and getattr(repository, "compat_mode", None) == getattr( + repository, + "COMPAT_READ_WRITE", + "compat_read_write", + ): self.blast_all_purge_legacy_state_artifacts() + return True - def _save_selected_stop_state(self, state: RunState) -> None: - self._state_repository(self.runtime).save_selected_stop_state( - state=state, - emit=self.runtime._emit, # type: ignore[attr-defined] - runtime_map_builder=lambda raw_state: build_runtime_map(cast(RunState, raw_state)), - ) + @staticmethod + def _has_active_runs(repository: object) -> bool: + has_active_runs = getattr(repository, "has_active_runs", None) + if callable(has_active_runs): + return bool(has_active_runs()) + load_all = getattr(repository, "load_all", None) + return bool(load_all()) if callable(load_all) else True def _purge_runtime_state(self, *, aggressive: bool) -> None: - if aggressive: - self.release_all_runtime_ports() - else: - release_port_session = getattr(self.runtime, "_release_port_session", None) - if callable(release_port_session): - release_port_session() self._state_repository(self.runtime).purge(aggressive=aggressive) + # Purging means no tracked run remains in this repository scope. Port + # locks were created by the original startup process (and therefore a + # different PortPlanner session), so releasing only this command's + # session leaves unused/disabled stack locks behind indefinitely. The + # registry commit happens first so a failed purge cannot erase locks + # that still protect an authoritative active state. + self.release_all_runtime_ports() + + def _deactivate_runtime_state( + self, + state: RunState, + *, + release_session_when_empty: bool = True, + ) -> None: + repository = self._state_repository(self.runtime) + source_run_ids = [ + str(run_id).strip() + for run_id in ( + state.metadata.get("state_source_run_ids") + if isinstance(state.metadata.get("state_source_run_ids"), list) + else [] + ) + if str(run_id).strip() + ] + run_ids = sorted({state.run_id, *source_run_ids}) + deactivate_many = getattr(repository, "deactivate_runs", None) + deactivate = getattr(repository, "deactivate_run", None) + if callable(deactivate_many) or callable(deactivate): + if callable(deactivate_many): + deactivate_many(run_ids) + elif callable(deactivate): + for run_id in run_ids: + deactivate(run_id) + if release_session_when_empty and not self._has_active_runs(repository): + # The final run may have been started by an earlier process and + # therefore a different planner session. At this point the + # repository proves the whole scope inactive, so every scoped + # lock is stale ownership rather than another live run's lock. + self.release_all_runtime_ports() + return + self._purge_runtime_state(aggressive=False) def _run_spinner_operation( self, diff --git a/python/envctl_engine/runtime/lifecycle_dependency_stop.py b/python/envctl_engine/runtime/lifecycle_dependency_stop.py index aedc97432..909958e9a 100644 --- a/python/envctl_engine/runtime/lifecycle_dependency_stop.py +++ b/python/envctl_engine/runtime/lifecycle_dependency_stop.py @@ -1,11 +1,12 @@ from __future__ import annotations from collections.abc import Callable, Mapping +from typing import Any from envctl_engine.requirements.core import dependency_definitions from envctl_engine.runtime.command_router import Route -from envctl_engine.runtime.lifecycle_requirement_ports import component_port_values -from envctl_engine.state.models import RunState +from envctl_engine.runtime.lifecycle_requirement_ports import release_requirement_component_port_values +from envctl_engine.state.models import RequirementsResult, RunState def select_dependency_components_for_stop(state: RunState, route: Route) -> dict[str, set[str]]: @@ -37,7 +38,7 @@ def release_selected_dependency_components( state: RunState, selected_dependencies: dict[str, set[str]], *, - release_component_ports_fn: Callable[[Mapping[str, object]], None], + release_component_ports_fn: Callable[[RequirementsResult, str, Mapping[str, object]], None], ) -> None: for project_name, dependency_ids in selected_dependencies.items(): requirements = state.requirements.get(project_name) @@ -48,7 +49,7 @@ def release_selected_dependency_components( if not bool(component.get("enabled", False)): continue if not bool(component.get("external")): - release_component_ports_fn(component) + release_component_ports_fn(requirements, dependency_id, component) requirements.components[dependency_id] = {} if not requirements_have_enabled_components(requirements): state.requirements.pop(project_name, None) @@ -57,10 +58,14 @@ def release_selected_dependency_components( def release_requirement_component_ports( component: Mapping[str, object], *, - release_port_fn: Callable[[int], None], + port_planner: Any, + owner_candidates: tuple[str, ...] = (), ) -> None: - for port in sorted(component_port_values(component)): - release_port_fn(port) + release_requirement_component_port_values( + port_planner, + component, + owner_candidates=owner_candidates, + ) def requirements_have_enabled_components(requirements: object) -> bool: @@ -68,7 +73,5 @@ def requirements_have_enabled_components(requirements: object) -> bool: if not isinstance(components, Mapping): return False return any( - bool(component.get("enabled", False)) - for component in components.values() - if isinstance(component, Mapping) + bool(component.get("enabled", False)) for component in components.values() if isinstance(component, Mapping) ) diff --git a/python/envctl_engine/runtime/lifecycle_operation_lease.py b/python/envctl_engine/runtime/lifecycle_operation_lease.py new file mode 100644 index 000000000..8992961d5 --- /dev/null +++ b/python/envctl_engine/runtime/lifecycle_operation_lease.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import os +import threading +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from envctl_engine.state.persistence import advisory_file_lock + + +_OPERATION_OWNER_ATTR = "_lifecycle_operation_owner" +_OPERATION_DEPTH_ATTR = "_lifecycle_operation_depth" +_OPERATION_LEASE_ATTR = "_lifecycle_operation_lease" + + +class _LifecycleOperationLease: + def __init__(self, manager: Any) -> None: + self._manager = manager + self.released = False + + def release(self) -> None: + if self.released: + return + self.released = True + self._manager.__exit__(None, None, None) + + +def run_exclusive_lifecycle_operation( + runtime: Any, + *, + command: str, + operation: Callable[[], int], +) -> int: + owner = os.getpid(), threading.get_ident() + current_owner = getattr(runtime, _OPERATION_OWNER_ATTR, None) + current_depth = int(getattr(runtime, _OPERATION_DEPTH_ATTR, 0) or 0) + if current_owner == owner and current_depth > 0: + setattr(runtime, _OPERATION_DEPTH_ATTR, current_depth + 1) + try: + return operation() + finally: + if getattr(runtime, _OPERATION_OWNER_ATTR, None) == owner: + setattr(runtime, _OPERATION_DEPTH_ATTR, current_depth) + + lock_path = _operation_lock_path(runtime) + manager = advisory_file_lock(lock_path, exclusive=True, timeout_seconds=0.0) + try: + manager.__enter__() + except TimeoutError: + runtime._emit("lifecycle.operation.busy", command=command, lock_path=str(lock_path)) + print("Another envctl lifecycle operation is already active for this repository.") + return 1 + + lease = _LifecycleOperationLease(manager) + setattr(runtime, _OPERATION_OWNER_ATTR, owner) + setattr(runtime, _OPERATION_DEPTH_ATTR, 1) + setattr(runtime, _OPERATION_LEASE_ATTR, lease) + try: + return operation() + finally: + if getattr(runtime, _OPERATION_LEASE_ATTR, None) is lease: + _clear_runtime_lease(runtime) + lease.release() + + +def release_lifecycle_operation(runtime: Any) -> bool: + """Release a committed lifecycle mutation before entering an interactive wait.""" + + owner = os.getpid(), threading.get_ident() + if getattr(runtime, _OPERATION_OWNER_ATTR, None) != owner: + return False + lease = getattr(runtime, _OPERATION_LEASE_ATTR, None) + if not isinstance(lease, _LifecycleOperationLease): + return False + _clear_runtime_lease(runtime) + lease.release() + return True + + +def lifecycle_operation_active(runtime: Any) -> bool: + """Return whether this thread still owns an unreleased lifecycle lease.""" + + owner = os.getpid(), threading.get_ident() + lease = getattr(runtime, _OPERATION_LEASE_ATTR, None) + return ( + getattr(runtime, _OPERATION_OWNER_ATTR, None) == owner + and int(getattr(runtime, _OPERATION_DEPTH_ATTR, 0) or 0) > 0 + and isinstance(lease, _LifecycleOperationLease) + and not lease.released + ) + + +def _clear_runtime_lease(runtime: Any) -> None: + setattr(runtime, _OPERATION_DEPTH_ATTR, 0) + setattr(runtime, _OPERATION_OWNER_ATTR, None) + setattr(runtime, _OPERATION_LEASE_ATTR, None) + + +def _operation_lock_path(runtime: Any) -> Path: + repository = getattr(runtime, "state_repository", None) + repository_lock_path = getattr(repository, "lifecycle_operation_lock_path", None) + if callable(repository_lock_path): + return Path(repository_lock_path()) + root = getattr(repository, "runtime_root", None) + if root is None: + root = getattr(runtime, "runtime_root") + root_path = Path(root).expanduser().absolute() + if root_path.is_symlink(): + raise RuntimeError(f"runtime_root changed identity or became a symlink: {root_path}") + return root_path.resolve(strict=False) / ".lifecycle_operation.lock" + + +__all__ = [ + "lifecycle_operation_active", + "release_lifecycle_operation", + "run_exclusive_lifecycle_operation", +] diff --git a/python/envctl_engine/runtime/lifecycle_requirement_ports.py b/python/envctl_engine/runtime/lifecycle_requirement_ports.py index 213ef6157..5d51d01b5 100644 --- a/python/envctl_engine/runtime/lifecycle_requirement_ports.py +++ b/python/envctl_engine/runtime/lifecycle_requirement_ports.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Mapping +import inspect from typing import Any from envctl_engine.requirements.core import dependency_definitions @@ -8,8 +9,122 @@ def release_requirement_ports(runtime: Any, requirements: RequirementsResult) -> None: - for port in sorted(requirement_port_values(requirements)): - runtime.port_planner.release(port) + reservations: list[tuple[int, tuple[str, ...], str | None]] = [] + for definition in dependency_definitions(): + component = requirements.component(definition.id) + if not bool(component.get("enabled", False)) or bool(component.get("external")): + continue + owners = requirement_component_port_owners(requirements, definition.id) + expected_session = _port_lock_session(component.get("port_lock_session")) + reservations.extend((port, owners, expected_session) for port in component_port_values(component)) + for port, owners, expected_session in sorted(reservations, key=lambda item: item[0]): + release_port_reservation( + runtime.port_planner, + port, + owner_candidates=owners, + expected_session=expected_session, + ) + + +def release_requirement_component_port_values( + port_planner: Any, + component: Mapping[str, object], + *, + owner_candidates: tuple[str, ...], +) -> None: + expected_session = _port_lock_session(component.get("port_lock_session")) + for port in sorted(component_port_values(component)): + release_port_reservation( + port_planner, + port, + owner_candidates=owner_candidates, + expected_session=expected_session, + ) + + +def release_port_reservation( + port_planner: Any, + port: int, + *, + owner_candidates: tuple[str, ...], + expected_session: str | None, +) -> bool: + """Release one persisted reservation without crossing planner sessions.""" + + owners = tuple(dict.fromkeys(owner for owner in owner_candidates if owner)) + normalized_session = _port_lock_session(expected_session) + release_owned = getattr(port_planner, "release_owned", None) + if normalized_session and callable(release_owned): + for owner in owners: + if bool(release_owned(port, owner, expected_session=normalized_session)): + return True + + # A legacy record has no session identity. Its only safe eager release is + # through the current planner session, whose release method checks its own + # session before unlinking. For a persisted session, use this fallback only + # when that exact planner session is still active. + current_session = _port_lock_session(getattr(port_planner, "session_id", None)) + if normalized_session is None or current_session == normalized_session: + _release_current_session(port_planner, port, owners) + + # Reaping is the compatibility path for dead foreign sessions. It is + # deliberately attempted only through the planner's liveness proof. + reap_stale = getattr(port_planner, "reap_stale", None) + if callable(reap_stale): + if owners: + for owner in owners: + if bool(reap_stale(port, owner=owner)): + return True + elif bool(reap_stale(port)): + return True + return False + + +def _release_current_session(port_planner: Any, port: int, owners: tuple[str, ...]) -> None: + release = getattr(port_planner, "release", None) + if not callable(release): + return + if not owners: + release(port) + return + try: + parameters = inspect.signature(release).parameters.values() + except (TypeError, ValueError): + return + supports_owner = any( + parameter.name == "owner" or parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters + ) + if not supports_owner: + # Minimal test doubles and legacy adapters may expose release(port). + # Production PortPlanner always takes owner and enforces its session. + release(port) + return + for owner in owners: + release(port, owner=owner) + + +def _port_lock_session(value: object) -> str | None: + normalized = value.strip() if isinstance(value, str) else "" + return normalized or None + + +def requirement_component_port_owners( + requirements: RequirementsResult, + dependency_id: str, +) -> tuple[str, ...]: + """Return every owner form used while reserving one dependency's ports.""" + + project = str(requirements.project).strip() + normalized_dependency = str(dependency_id).strip().lower() + if not project or not normalized_dependency: + return () + owner_suffixes = {"requirements"} + for definition in dependency_definitions(): + if definition.id != normalized_dependency: + continue + owner_suffixes.update(resource.legacy_port_key for resource in definition.resources) + break + return tuple(f"{project}:{suffix}" for suffix in sorted(owner_suffixes)) def requirement_port_values(requirements: RequirementsResult) -> set[int]: @@ -20,9 +135,7 @@ def requirement_port_values(requirements: RequirementsResult) -> set[int]: continue if bool(component.get("external")): continue - final = component.get("final") - if isinstance(final, int) and final > 0: - ports.add(final) + ports.update(component_port_values(component)) return ports diff --git a/python/envctl_engine/runtime/lifecycle_service_stop_selection.py b/python/envctl_engine/runtime/lifecycle_service_stop_selection.py index 917c59505..273c58233 100644 --- a/python/envctl_engine/runtime/lifecycle_service_stop_selection.py +++ b/python/envctl_engine/runtime/lifecycle_service_stop_selection.py @@ -22,24 +22,26 @@ def select_services_for_stop( if bool(route.flags.get("all")): return set(state.services.keys()) + selected, has_selectors = _selected_services_from_route( + state, + route, + project_name_from_service_fn=project_name_from_service_fn, + selectors_from_passthrough_fn=selectors_from_passthrough_fn, + ) + runtime_scope = route.flags.get("runtime_scope") if runtime_scope in {"backend", "frontend"}: - return { + scoped = { name for name, service in state.services.items() if service_matches_runtime_scope(name, service, str(runtime_scope)) } + return scoped.intersection(selected) if has_selectors else scoped if runtime_scope in {"fullstack", "entire-system"}: - return set(state.services.keys()) + return selected if has_selectors else set(state.services.keys()) if runtime_scope == "dependencies": return set() - selected, has_selectors = _selected_services_from_route( - state, - route, - project_name_from_service_fn=project_name_from_service_fn, - selectors_from_passthrough_fn=selectors_from_passthrough_fn, - ) if not selected and has_selectors: return set() if not selected and select_dependency_components_for_stop(state, route): diff --git a/python/envctl_engine/runtime/lifecycle_service_termination.py b/python/envctl_engine/runtime/lifecycle_service_termination.py index 0c8ff9841..ad1a87e35 100644 --- a/python/envctl_engine/runtime/lifecycle_service_termination.py +++ b/python/envctl_engine/runtime/lifecycle_service_termination.py @@ -2,13 +2,39 @@ import concurrent.futures import os -import signal +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path from typing import Any +from envctl_engine.shared.process_termination import ( + pid_is_running, + terminate_pid, + wait_for_pid_exit, +) +from envctl_engine.shared.services import service_project_name +from envctl_engine.runtime.lifecycle_requirement_ports import release_port_reservation -def terminate_started_services(runtime: Any, services: dict[str, object]) -> None: - for service in services.values(): - runtime._terminate_service_record(service, aggressive=False, verify_ownership=False) + +def terminate_started_services(runtime: Any, services: dict[str, object]) -> set[str]: + """Terminate startup-owned services and report every unconfirmed exit.""" + + failed: set[str] = set() + for name, service in services.items(): + try: + terminated = runtime._terminate_service_record( + service, + aggressive=False, + verify_ownership=False, + ) + except Exception as exc: # noqa: BLE001 + _emit_cleanup_error(runtime, service=name, error=exc) + terminated = False + if not terminated: + failed.add(name) + else: + _release_service_ports(runtime, name=name, service=service) + return failed def terminate_services_from_state( @@ -18,19 +44,25 @@ def terminate_services_from_state( selected_services: set[str] | None, aggressive: bool, verify_ownership: bool, -) -> None: +) -> set[str]: work_items: list[tuple[str, object]] = [] for name, service in getattr(state, "services", {}).items(): if selected_services is not None and name not in selected_services: continue work_items.append((name, service)) - def terminate_one(item: tuple[str, object]) -> tuple[str, bool, int | None]: + def terminate_one(item: tuple[str, object]) -> tuple[str, object, bool, int | None]: name, service = item - terminated = runtime._terminate_service_record( - service, aggressive=aggressive, verify_ownership=verify_ownership - ) - return name, terminated, service_port(service) + try: + terminated = runtime._terminate_service_record( + service, + aggressive=aggressive, + verify_ownership=verify_ownership, + ) + except Exception as exc: # noqa: BLE001 + _emit_cleanup_error(runtime, service=name, error=exc) + terminated = False + return name, service, terminated, service_port(service) if len(work_items) <= 1: results = [terminate_one(item) for item in work_items] @@ -39,47 +71,120 @@ def terminate_one(item: tuple[str, object]) -> tuple[str, bool, int | None]: with concurrent.futures.ThreadPoolExecutor(max_workers=worker_count) as pool: results = list(pool.map(terminate_one, work_items)) - for _name, terminated, port in results: - if terminated and port is not None: - runtime.port_planner.release(port) + for name, service, terminated, _port in results: + if terminated: + _release_service_ports(runtime, name=name, service=service) + return {name for name, _service, terminated, _port in results if not terminated} + + +def _release_service_ports(runtime: Any, *, name: str, service: object) -> None: + port_planner = getattr(runtime, "port_planner", None) + if port_planner is None: + return + project = str(service_project_name(service)).strip() + if not project: + project_resolver = getattr(runtime, "_project_name_from_service", None) + if callable(project_resolver): + project = str(project_resolver(name) or "").strip() + service_type = str(getattr(service, "type", "") or "").strip().lower() + owners: tuple[str, ...] = () + if project: + owner_candidates = [f"{project}:services"] + if service_type: + owner_candidates = [ + f"{project}:{service_type}", + f"{project}:services:{service_type}-launch", + *owner_candidates, + ] + owners = tuple(owner_candidates) + + raw_expected_session = getattr(service, "port_lock_session", None) + expected_session = raw_expected_session.strip() if isinstance(raw_expected_session, str) else None + for port in service_ports(service): + release_port_reservation( + port_planner, + port, + owner_candidates=owners, + expected_session=expected_session, + ) + + +def service_ports(service: object) -> tuple[int, ...]: + """Return every distinct port reservation represented by a service record.""" + + ports: list[int] = [] + for attribute in ("requested_port", "actual_port"): + value = getattr(service, attribute, None) + if isinstance(value, int) and not isinstance(value, bool) and value > 0 and value not in ports: + ports.append(value) + return tuple(ports) def terminate_service_record(runtime: Any, service: object, *, aggressive: bool, verify_ownership: bool) -> bool: - pid = getattr(service, "pid", None) - if not isinstance(pid, int) or pid <= 0: - return True - if pid in {os.getpid(), os.getppid()}: - runtime._emit("cleanup.skip", service=getattr(service, "name", "unknown"), pid=pid, reason="self_or_parent") + recorded_pids = _recorded_service_pids(service) + if not recorded_pids and str(getattr(service, "status", "") or "") == "termination_failed": + return False + protected_pids = {os.getpid(), os.getppid()} + protected = [pid for pid in recorded_pids if pid in protected_pids] + if protected: + runtime._emit( + "cleanup.skip", + service=getattr(service, "name", "unknown"), + pid=protected[0], + reason="self_or_parent", + ) return False + live_pids = [pid for pid in recorded_pids if _pid_is_running(runtime, pid)] port = service_port(service) + listener_probe = _live_listener_pids_for_port(runtime, port) if port is not None else _ListenerProbe() + if not live_pids: + return not listener_probe.failed and not listener_probe.pids if verify_ownership: if port is None: - runtime._emit( - "cleanup.skip", - service=getattr(service, "name", "unknown"), - pid=pid, - reason="missing_port_for_ownership", - ) - return False - is_owner = _service_pid_or_listener_owns_port(runtime, service, pid=pid, port=port) - if not is_owner: - runtime._emit("cleanup.skip", service=getattr(service, "name", "unknown"), pid=pid, port=port) - return False + if not all(_non_listener_service_matches_recorded_cwd(runtime, service, pid=pid) for pid in live_pids): + runtime._emit( + "cleanup.skip", + service=getattr(service, "name", "unknown"), + pid=live_pids[0], + reason="missing_port_for_ownership", + ) + return False + termination_pids = live_pids + else: + owners = [pid for pid in live_pids if _pid_owns_port(runtime, pid=pid, port=port)] + if not owners: + runtime._emit( + "cleanup.skip", + service=getattr(service, "name", "unknown"), + pid=live_pids[0], + port=port, + reason="ownership_mismatch", + ) + return False + primary_pid = getattr(service, "pid", None) + termination_pids = list(owners) + if isinstance(primary_pid, int) and primary_pid in live_pids and primary_pid not in termination_pids: + termination_pids.insert(0, primary_pid) + else: + termination_pids = live_pids - try: - terminate_group = getattr(runtime.process_runner, "terminate_process_group", None) - if callable(terminate_group): - return bool(terminate_group(pid, term_timeout=0.5 if aggressive else 2.0, kill_timeout=1.0)) - except Exception: # noqa: BLE001 - pass - try: - return bool(runtime.process_runner.terminate(pid, term_timeout=0.5 if aggressive else 2.0, kill_timeout=1.0)) - except Exception: # noqa: BLE001 - pass - try: - os.kill(pid, signal.SIGTERM) - except OSError: - return True + failed = False + for pid in termination_pids: + if not _pid_is_running(runtime, pid): + continue + if not terminate_pid( + pid, + process_runner=runtime.process_runner, + term_timeout=0.5 if aggressive else 2.0, + kill_timeout=1.0, + ): + failed = True + if failed: + return False + if port is not None: + listener_probe = _live_listener_pids_for_port(runtime, port) + if listener_probe.failed or listener_probe.pids: + return False return True @@ -93,21 +198,92 @@ def service_port(service: object) -> int | None: return None -def _service_pid_or_listener_owns_port(runtime: Any, service: object, *, pid: int, port: int) -> bool: +def _emit_cleanup_error(runtime: Any, *, service: str, error: BaseException) -> None: + emit = getattr(runtime, "_emit", None) + if callable(emit): + emit("cleanup.error", service=service, error=str(error)) + + +def _pid_owns_port(runtime: Any, *, pid: int, port: int) -> bool: try: - if bool(runtime.process_runner.pid_owns_port(pid, port)): - return True + return bool(runtime.process_runner.pid_owns_port(pid, port)) except Exception: # noqa: BLE001 - pass - listener_pids = getattr(service, "listener_pids", None) - if not isinstance(listener_pids, list): return False - for listener_pid in listener_pids: - if not isinstance(listener_pid, int) or listener_pid <= 0: - continue - try: - if bool(runtime.process_runner.pid_owns_port(listener_pid, port)): - return True - except Exception: # noqa: BLE001 - continue - return False + + +def _recorded_service_pids(service: object) -> list[int]: + candidates = [getattr(service, "pid", None), *(getattr(service, "listener_pids", None) or [])] + return list( + dict.fromkeys(pid for pid in candidates if isinstance(pid, int) and not isinstance(pid, bool) and pid > 0) + ) + + +@dataclass(frozen=True, slots=True) +class _ListenerProbe: + available: bool = False + failed: bool = False + pids: frozenset[int] = frozenset() + + +def _live_listener_pids_for_port(runtime: Any, port: int) -> _ListenerProbe: + listener_reader = getattr(runtime, "_listener_pids_for_port", None) + if not callable(listener_reader): + return _ListenerProbe() + try: + listeners = listener_reader(port) + except Exception: # noqa: BLE001 + return _ListenerProbe(available=True, failed=True) + if listeners is None or isinstance(listeners, (str, bytes, Mapping)): + return _ListenerProbe(available=True, failed=True) + try: + listener_items = iter(listeners) + except TypeError: + return _ListenerProbe(available=True, failed=True) + return _ListenerProbe( + available=True, + pids=frozenset( + pid + for pid in listener_items + if isinstance(pid, int) and not isinstance(pid, bool) and pid > 0 and _pid_is_running(runtime, pid) + ), + ) + + +def _non_listener_service_matches_recorded_cwd(runtime: Any, service: object, *, pid: int) -> bool: + if bool(getattr(service, "listener_expected", True)): + return False + recorded_cwd = str(getattr(service, "cwd", "") or "").strip() + if not recorded_cwd: + return False + cwd_reader = getattr(runtime.process_runner, "process_cwd", None) + if not callable(cwd_reader): + return False + try: + actual_cwd = str(cwd_reader(pid) or "").strip() + except Exception: # noqa: BLE001 + return False + if not actual_cwd: + return False + try: + return Path(actual_cwd).resolve(strict=False) == Path(recorded_cwd).resolve(strict=False) + except OSError: + return actual_cwd == recorded_cwd + + +def _wait_for_pid_exit( + runtime: Any, + pid: int, + *, + timeout: float, + initial_identity: str | None, +) -> bool: + return wait_for_pid_exit( + runtime.process_runner, + pid, + timeout=timeout, + initial_identity=initial_identity, + ) + + +def _pid_is_running(runtime: Any, pid: int) -> bool: + return pid_is_running(runtime.process_runner, pid) diff --git a/python/envctl_engine/runtime/lifecycle_targeted_state.py b/python/envctl_engine/runtime/lifecycle_targeted_state.py new file mode 100644 index 000000000..1d7cf805f --- /dev/null +++ b/python/envctl_engine/runtime/lifecycle_targeted_state.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from typing import Any, cast + +from envctl_engine.state.models import RequirementsResult, RunState +from envctl_engine.state.runtime_map import build_runtime_map +from envctl_engine.state.state_aggregation import filter_state_to_owned_projects + + +class LifecycleTargetedStateSupport: + """Own project-scoped state reduction after targeted lifecycle stops.""" + + runtime: Any = cast(Any, None) + + def _project_name_for_service(self, name: str, service: object) -> str: + explicit_project = str(getattr(service, "project", "") or "").strip() + if explicit_project: + return explicit_project.casefold() + return str(self.runtime._project_name_from_service(name) or "").strip().casefold() + + def _save_selected_stop_state( + self, + state: RunState, + *, + authoritative_project_names: list[str] | None = None, + ) -> None: + owner = cast(Any, self) + repository = owner._state_repository(self.runtime) + + def runtime_map_builder(raw_state: object) -> dict[str, object]: + return build_runtime_map(cast(RunState, raw_state)) + + if authoritative_project_names is None: + repository.save_selected_stop_state( + state=state, + emit=self.runtime._emit, + runtime_map_builder=runtime_map_builder, + ) + return + repository.save_selected_stop_state( + state=state, + emit=self.runtime._emit, + runtime_map_builder=runtime_map_builder, + authoritative_project_names=authoritative_project_names, + ) + + def _persist_targeted_project_state( + self, + state: RunState, + *, + owned_projects: dict[str, str], + targeted_project_keys: frozenset[str], + ) -> bool: + owner = cast(Any, self) + active_project_keys = self._active_project_keys(state) + remaining_keys = (set(owned_projects).difference(targeted_project_keys)).union( + targeted_project_keys.intersection(active_project_keys) + ) + if not remaining_keys: + for requirements in state.requirements.values(): + owner._release_requirement_ports(requirements) + owner._deactivate_runtime_state(state) + return False + + remaining_names = [owned_projects[key] for key in sorted(remaining_keys)] + filtered = filter_state_to_owned_projects( + state, + frozenset(remaining_keys), + service_project_name=self._project_name_for_service, + ) + filtered.metadata["project_names"] = remaining_names + self._save_selected_stop_state( + filtered, + authoritative_project_names=remaining_names, + ) + return True + + def _owned_project_names(self, state: RunState) -> dict[str, str]: + projects: dict[str, str] = {} + + def add(raw_name: object) -> None: + name = str(raw_name or "").strip() + if name: + projects.setdefault(name.casefold(), name) + + metadata_names = state.metadata.get("project_names") + if isinstance(metadata_names, Iterable) and not isinstance(metadata_names, (str, bytes, Mapping)): + for project in metadata_names: + add(project) + metadata_roots = state.metadata.get("project_roots") + if isinstance(metadata_roots, Mapping): + for project in metadata_roots: + add(project) + for project, requirements in state.requirements.items(): + add(getattr(requirements, "project", "") or project) + for name, service in state.services.items(): + add(self._project_name_for_service(name, service)) + if state.mode == "trees" and any(project != "main" for project in projects): + projects.pop("main", None) + return projects + + def _active_project_keys(self, state: RunState) -> set[str]: + active = { + self._project_name_for_service(name, service) + for name, service in state.services.items() + if self._project_name_for_service(name, service) + } + active.update( + self._requirement_project_key(project, requirements) for project, requirements in state.requirements.items() + ) + raw_stopped = state.metadata.get("dashboard_stopped_services") + if isinstance(raw_stopped, list): + active.update( + str(item.get("project", "")).strip().casefold() + for item in raw_stopped + if isinstance(item, Mapping) and str(item.get("project", "")).strip() + ) + return {project for project in active if project} + + def _targeted_projects_have_no_managed_resources( + self, + state: RunState, + targeted_project_keys: frozenset[str], + ) -> bool: + if any( + self._project_name_for_service(name, service) in targeted_project_keys + for name, service in state.services.items() + ): + return False + return not any( + self._requirement_project_key(project, requirements) in targeted_project_keys + and cast(Any, self)._requirements_have_enabled_components(requirements) + for project, requirements in state.requirements.items() + ) + + def _remove_targeted_requirements( + self, + state: RunState, + targeted_project_keys: frozenset[str], + ) -> None: + owner = cast(Any, self) + for project, requirements in list(state.requirements.items()): + if self._requirement_project_key(project, requirements) not in targeted_project_keys: + continue + owner._release_requirement_ports(requirements) + state.requirements.pop(project, None) + + @staticmethod + def _requirement_project_key(project: str, requirements: RequirementsResult) -> str: + return str(getattr(requirements, "project", "") or project).strip().casefold() diff --git a/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py b/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py index 771073dbb..a7e1b1ffa 100644 --- a/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py +++ b/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py @@ -30,6 +30,7 @@ from envctl_engine.runtime.lifecycle_worktree_processes import ( blast_tree_listener_ports as _blast_tree_listener_ports, ) +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RequirementsResult, RunState from envctl_engine.state.runtime_map import build_runtime_map @@ -136,7 +137,12 @@ def blast_worktree_before_delete( ) for mode in ("trees", "main"): - state = runtime._try_load_existing_state(mode=mode, strict_mode_match=True) + state = call_state_loader( + runtime._try_load_existing_state, + mode=mode, + strict_mode_match=True, + project_names=[normalized_project], + ) if state is None: continue @@ -159,33 +165,39 @@ def blast_worktree_before_delete( if not selected_services and requirement_key is None: continue - runtime._terminate_services_from_state( + termination_result = runtime._terminate_services_from_state( state, selected_services=selected_services, aggressive=True, verify_ownership=False, ) - for service_name in selected_services: + failed_services = ( + {str(name) for name in termination_result} if isinstance(termination_result, (set, frozenset)) else set() + ) + stopped_services = selected_services.difference(failed_services) + for service_name in stopped_services: state.services.pop(service_name, None) - if requirement_key is not None: + if requirement_key is not None and not failed_services: requirement_entry = state.requirements.pop(requirement_key, None) if requirement_entry is not None: target_ports.update(cleanup.collect_requirement_ports(requirement_entry)) release_requirement_ports(runtime, requirement_entry) - remaining_projects: set[str] = set() - for service_name in state.services: - project = runtime._project_name_from_service(service_name) - if project: - remaining_projects.add(project) - for project in list(state.requirements.keys()): - if project in remaining_projects: - continue - requirement_entry = state.requirements.pop(project) - release_requirement_ports(runtime, requirement_entry) + if not failed_services: + remaining_projects: set[str] = set() + for service_name in state.services: + project = runtime._project_name_from_service(service_name) + if project: + remaining_projects.add(project) + for project in list(state.requirements.keys()): + if project in remaining_projects: + continue + requirement_entry = state.requirements.pop(project) + release_requirement_ports(runtime, requirement_entry) - artifact_paths.update(cleanup.prune_project_metadata(state, project_name=normalized_project)) + if not failed_services: + artifact_paths.update(cleanup.prune_project_metadata(state, project_name=normalized_project)) try: runtime.state_repository.save_selected_stop_state( @@ -202,15 +214,27 @@ def blast_worktree_before_delete( mode=mode, warning=warning, ) - continue + raise RuntimeError(warning) from exc runtime._emit( "cleanup.worktree.state.updated", project=normalized_project, mode=mode, - services_removed=len(selected_services), - requirements_removed=(requirement_key is not None), + services_removed=len(stopped_services), + requirements_removed=(requirement_key is not None and not failed_services), ) + if failed_services: + warning = ( + f"worktree cleanup could not confirm service exit for {normalized_project} ({mode}): " + + ", ".join(sorted(failed_services)) + ) + runtime._emit( + "cleanup.worktree.warning", + project=normalized_project, + mode=mode, + warning=warning, + ) + raise RuntimeError(warning) if blast_mode: cleanup.blast_tree_listener_ports( diff --git a/python/envctl_engine/runtime/service_manager.py b/python/envctl_engine/runtime/service_manager.py index a26833372..20384feb9 100644 --- a/python/envctl_engine/runtime/service_manager.py +++ b/python/envctl_engine/runtime/service_manager.py @@ -1,13 +1,13 @@ from __future__ import annotations import concurrent.futures -import os -import signal import time +from collections.abc import Iterable from dataclasses import dataclass from typing import Callable from envctl_engine.requirements.common_contracts import is_bind_conflict +from envctl_engine.shared.process_termination import terminate_pid from envctl_engine.shared.services import service_display_name from envctl_engine.state.models import ServiceRecord @@ -16,6 +16,14 @@ class ServiceStartError(RuntimeError): """Raised when a service fails to start after retry policy.""" +class ServiceCleanupError(ServiceStartError): + """Raised when a failed startup leaves processes whose exit is unconfirmed.""" + + def __init__(self, message: str, unterminated_services: dict[str, ServiceRecord]) -> None: + super().__init__(message) + self.unterminated_services = dict(unterminated_services) + + @dataclass(slots=True) class ServiceStartOutcome: record: ServiceRecord @@ -38,6 +46,9 @@ class ServiceStartDescriptor: class ServiceManager: + def __init__(self, *, process_runner: object | None = None) -> None: + self.process_runner = process_runner + def start_service_with_retry( self, *, @@ -58,6 +69,20 @@ def start_service_with_retry( while True: success, error, pid = start(current_port) if success: + if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0: + record = _failed_start_record( + project=project, + service_type=service_type, + cwd=cwd, + pid=None, + requested_port=current_port if listener_expected else None, + listener_expected=listener_expected, + failure_detail="service launcher reported success without a valid PID", + ) + raise ServiceCleanupError( + f"Service launcher for {record.name} reported success without a valid PID", + {record.name: record}, + ) try: actual_port: int | None = current_port if listener_expected else None if detect_actual is not None: @@ -73,9 +98,37 @@ def start_service_with_retry( started_at=time.time(), listener_expected=listener_expected, ) - except RuntimeError as exc: + except Exception as exc: # noqa: BLE001 error = str(exc) - _terminate_pid(pid, process_runner=self) + if not _terminate_pid(pid, process_runner=self.process_runner): + record = _failed_start_record( + project=project, + service_type=service_type, + cwd=cwd, + pid=pid, + requested_port=current_port if listener_expected else None, + listener_expected=listener_expected, + failure_detail=error, + ) + raise ServiceCleanupError( + f"{error}; cleanup could not confirm exit of PID {pid}", + {record.name: record}, + ) from exc + elif isinstance(pid, int) and pid > 0: + if not _terminate_pid(pid, process_runner=self.process_runner): + record = _failed_start_record( + project=project, + service_type=service_type, + cwd=cwd, + pid=pid, + requested_port=current_port if listener_expected else None, + listener_expected=listener_expected, + failure_detail=error or "service launcher reported failure", + ) + raise ServiceCleanupError( + f"{error or 'service launcher reported failure'}; cleanup could not confirm exit of PID {pid}", + {record.name: record}, + ) if retries >= max_retries or not _is_retryable_error(error): raise ServiceStartError( @@ -160,6 +213,8 @@ def start_record(descriptor: ServiceStartDescriptor) -> ServiceRecord: on_retry=on_retry, ) except Exception as exc: + if isinstance(exc, ServiceCleanupError): + raise if descriptor.critical: raise record = ServiceRecord( @@ -189,8 +244,7 @@ def start_record(descriptor: ServiceStartDescriptor) -> ServiceRecord: worker_count = max_workers or len(descriptors) with concurrent.futures.ThreadPoolExecutor(max_workers=min(len(descriptors), worker_count)) as executor: future_map = { - executor.submit(start_record, descriptor): index - for index, descriptor in enumerate(descriptors) + executor.submit(start_record, descriptor): index for index, descriptor in enumerate(descriptors) } records_by_index: dict[int, ServiceRecord] = {} failures_by_index: dict[int, Exception] = {} @@ -201,17 +255,27 @@ def start_record(descriptor: ServiceStartDescriptor) -> ServiceRecord: except Exception as exc: # noqa: BLE001 failures_by_index[index] = exc - partial_records.extend( - records_by_index[index] for index in sorted(records_by_index) - ) + partial_records.extend(records_by_index[index] for index in sorted(records_by_index)) if failures_by_index: - raise failures_by_index[min(failures_by_index)] + primary_failure = failures_by_index[min(failures_by_index)] + unterminated = _unterminated_services_from_failures(failures_by_index.values()) + if unterminated: + raise ServiceCleanupError(str(primary_failure), unterminated) from primary_failure + raise primary_failure else: for descriptor in descriptors: partial_records.append(start_record(descriptor)) - except Exception: + except Exception as exc: + unterminated = _unterminated_services_from_failures((exc,)) for record in partial_records: - _terminate_pid(record.pid, process_runner=self) + if not _terminate_pid(record.pid, process_runner=self.process_runner): + unterminated[record.name] = record + if unterminated: + names = ", ".join(sorted(unterminated)) + raise ServiceCleanupError( + f"{exc}; cleanup could not confirm exit for: {names}", + unterminated, + ) from exc raise return {record.name: record for record in partial_records} @@ -225,17 +289,52 @@ def _is_retryable_error(error: str | None) -> bool: return False -def _terminate_pid(pid: int | None, *, process_runner: object | None = None) -> None: - if not isinstance(pid, int) or pid <= 0: - return - terminator = getattr(process_runner, "terminate_process_group", None) if process_runner is not None else None - if callable(terminator): - try: - terminator(pid, term_timeout=2.0, kill_timeout=1.0) - return - except Exception: # noqa: BLE001 - pass - try: - os.kill(pid, signal.SIGTERM) - except OSError: - return +def _terminate_pid(pid: int | None, *, process_runner: object | None = None) -> bool: + return terminate_pid( + pid, + process_runner=process_runner, + term_timeout=2.0, + kill_timeout=1.0, + ) + + +def _failed_start_record( + *, + project: str, + service_type: str, + cwd: str, + pid: int | None, + requested_port: int | None, + listener_expected: bool, + failure_detail: str, +) -> ServiceRecord: + record = ServiceRecord( + name=f"{project} {service_display_name(service_type)}", + type=service_type, + cwd=cwd, + pid=pid, + requested_port=requested_port, + actual_port=None, + status="termination_failed", + started_at=time.time(), + listener_expected=listener_expected, + failure_detail=failure_detail, + degraded=True, + ) + record.project = project + record.service_slug = service_type + return record + + +def _unterminated_services_from_failures( + failures: Iterable[object], +) -> dict[str, ServiceRecord]: + unterminated: dict[str, ServiceRecord] = {} + for failure in failures: + services = getattr(failure, "unterminated_services", None) + if not isinstance(services, dict): + continue + for name, service in services.items(): + if isinstance(name, str) and isinstance(service, ServiceRecord): + unterminated[name] = service + return unterminated diff --git a/python/envctl_engine/runtime/state_fingerprint_support.py b/python/envctl_engine/runtime/state_fingerprint_support.py index 3c16058bd..cb06bea1b 100644 --- a/python/envctl_engine/runtime/state_fingerprint_support.py +++ b/python/envctl_engine/runtime/state_fingerprint_support.py @@ -1,12 +1,8 @@ from __future__ import annotations -import hashlib -import json - -from envctl_engine.state import state_to_dict +from envctl_engine.state.fingerprints import state_fingerprint as _state_fingerprint from envctl_engine.state.models import RunState def state_fingerprint(state: RunState) -> str: - payload = json.dumps(state_to_dict(state), sort_keys=True) - return hashlib.sha256(payload.encode("utf-8")).hexdigest() + return _state_fingerprint(state) diff --git a/python/envctl_engine/runtime/state_inspection_support.py b/python/envctl_engine/runtime/state_inspection_support.py new file mode 100644 index 000000000..260989fcf --- /dev/null +++ b/python/envctl_engine/runtime/state_inspection_support.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +from envctl_engine.runtime.command_router import Route +from envctl_engine.state import state_to_dict +from envctl_engine.state.lookup import call_state_loader +from envctl_engine.state.models import RunState +from envctl_engine.state.project_runtime import ( + cwd_runtime_warnings, + dependency_mode_summary, + project_resolution_event_payload, + resolve_requested_project_state, +) +from envctl_engine.state.runtime_map import build_runtime_map +from envctl_engine.ui.path_links import render_path_for_terminal + +StateLoader = Callable[..., RunState | None] +WarningEmitter = Callable[..., None] + + +def print_state( + runtime: Any, + route: Route, + *, + json_output: bool, + state_loader: StateLoader, + warning_emitter: WarningEmitter, +) -> int: + strict = runtime._state_lookup_strict_mode_match(route) + requested_projects = list(getattr(route, "projects", []) or []) + state = state_loader( + runtime, + mode=route.mode, + strict_mode_match=strict, + project_names=requested_projects or None, + ) + if state is None and requested_projects: + # A targeted repository lookup correctly returns no owner for an + # unknown project. Load the mode-wide aggregate once so the command can + # distinguish that case from an entirely missing runtime and return the + # structured requested_project_not_running contract. + state = state_loader( + runtime, + mode=route.mode, + strict_mode_match=strict, + project_names=None, + ) + if state is None: + if json_output: + print(json.dumps({"found": False, "mode": getattr(route, "mode", None)}, indent=2, sort_keys=True)) + else: + print("No previous state found.") + return 1 + if requested_projects: + resolution = resolve_requested_project_state( + state, + requested_projects, + command="show-state", + runtime=runtime, + allow_multi=False, + ) + if not resolution.ok: + _emit_project_resolution(runtime, "state.project_resolution.failed", resolution, state) + if json_output: + print(json.dumps(resolution.payload(), indent=2, sort_keys=True)) + else: + print(f"Requested project is not running: {resolution.requested_project}") + print("Active runtime projects: " + (", ".join(resolution.active_projects) or "none")) + return 1 + _emit_project_resolution(runtime, "state.project_resolution.ok", resolution, state) + if resolution.state is not None and not bool(getattr(route, "flags", {}).get("all")): + state = resolution.state + + dependency_summary = dependency_mode_summary(state) + cwd_project, warnings = cwd_runtime_warnings( + state, + runtime=runtime, + requested_projects=list(getattr(route, "projects", []) or []), + ) + warning_emitter(runtime, command="show-state", state=state, warnings=warnings) + payload: dict[str, Any] = { + "found": True, + "runtime_root": str(runtime.runtime_root), + "run_state_path": str(runtime._run_state_path()), + "state": state_to_dict(state), + "runtime_map": build_runtime_map(state), + "dependency_mode": dependency_summary["dependency_mode"], + "shared_dependencies": dependency_summary["shared_dependencies"], + "cwd_project": cwd_project, + "warnings": warnings, + } + if json_output: + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + + state_payload = payload["state"] + print(f"run_id: {state_payload['run_id']}") + print(f"mode: {state_payload['mode']}") + print(f"services: {len(state_payload['services'])}") + print(f"requirements: {len(state_payload['requirements'])}") + for warning in warnings: + print(f"Warning: {warning['message']}") + print("run_state_path:") + print(render_path_for_terminal(payload["run_state_path"], env=getattr(runtime, "env", {}))) + return 0 + + +def try_load_existing_state( + runtime: Any, + *, + mode: str | None, + strict_mode_match: bool, + project_names: list[str] | None = None, +) -> RunState | None: + loader = getattr(runtime, "_try_load_existing_state", None) + if not callable(loader): + return None + state = call_state_loader( + loader, + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ) + return state if isinstance(state, RunState) else None + + +def emit_cwd_runtime_mismatch_warnings( + runtime: Any, + *, + command: str, + state: RunState, + warnings: list[dict[str, object]], +) -> None: + emitter = getattr(runtime, "_emit", None) + if not callable(emitter): + return + for warning in warnings: + if str(warning.get("code") or "") != "cwd_runtime_mismatch": + continue + emitter( + "state.cwd_runtime_mismatch", + run_id=getattr(state, "run_id", ""), + mode=getattr(state, "mode", ""), + command=command, + cwd_project=warning.get("cwd_project"), + active_projects=warning.get("active_projects"), + ) + + +def _emit_project_resolution(runtime: Any, event: str, resolution: Any, state: RunState) -> None: + emitter = getattr(runtime, "_emit", None) + if callable(emitter): + emitter( + event, + **project_resolution_event_payload(resolution, state, runtime=runtime), + ) diff --git a/python/envctl_engine/shared/git_branch_support.py b/python/envctl_engine/shared/git_branch_support.py new file mode 100644 index 000000000..0b4efe602 --- /dev/null +++ b/python/envctl_engine/shared/git_branch_support.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import dataclass + + +PREFERRED_PR_BASE_BRANCHES = ("dev", "main", "master") + + +@dataclass(frozen=True, slots=True) +class _RemoteBranchInventory: + preferred_heads: frozenset[str] + default_branch: str | None + reachable: bool + + +def detect_preferred_default_branch( + git_output: Callable[[Sequence[str]], str], + *, + probe_remote_heads: bool = False, +) -> str: + """Resolve the PR base contract before consulting a repository's remote HEAD.""" + + tracked_remote_branches: set[str] = set() + for branch in PREFERRED_PR_BASE_BRANCHES: + if git_output(("rev-parse", "--verify", f"refs/remotes/origin/{branch}")).strip(): + tracked_remote_branches.add(branch) + + remote_inventory = _remote_branch_inventory(git_output) if probe_remote_heads else None + # A successful live inventory is authoritative over local tracking refs, + # including when none of the preferred branches exist. When the probe is + # unavailable or times out, retain the offline tracking-ref fallback. + available_branches = ( + remote_inventory.preferred_heads + if remote_inventory is not None and remote_inventory.reachable + else tracked_remote_branches + ) + for branch in PREFERRED_PR_BASE_BRANCHES: + if branch in available_branches: + return branch + if remote_inventory is not None and remote_inventory.reachable and remote_inventory.default_branch: + return remote_inventory.default_branch + + remote_head = git_output(("symbolic-ref", "--short", "refs/remotes/origin/HEAD")).strip() + if remote_head.startswith("origin/"): + return remote_head.removeprefix("origin/") + for branch in PREFERRED_PR_BASE_BRANCHES: + if git_output(("rev-parse", "--verify", f"refs/heads/{branch}")).strip(): + return branch + return "main" + + +def _remote_branch_inventory( + git_output: Callable[[Sequence[str]], str], +) -> _RemoteBranchInventory: + output = git_output( + ( + "ls-remote", + "--symref", + "origin", + "HEAD", + *(f"refs/heads/{branch}" for branch in PREFERRED_PR_BASE_BRANCHES), + ) + ) + heads: set[str] = set() + default_branch: str | None = None + for raw_line in output.splitlines(): + fields = raw_line.split() + if len(fields) == 3 and fields[0] == "ref:" and fields[2] == "HEAD": + ref = fields[1] + if ref.startswith("refs/heads/"): + default_branch = ref.removeprefix("refs/heads/") or None + continue + if len(fields) != 2 or not fields[1].startswith("refs/heads/"): + continue + branch = fields[1].removeprefix("refs/heads/") + if branch in PREFERRED_PR_BASE_BRANCHES: + heads.add(branch) + return _RemoteBranchInventory( + preferred_heads=frozenset(heads), + default_branch=default_branch, + reachable=bool(output.strip()), + ) + + +__all__ = ["PREFERRED_PR_BASE_BRANCHES", "detect_preferred_default_branch"] diff --git a/python/envctl_engine/shared/ports.py b/python/envctl_engine/shared/ports.py index 76f5f4fa0..36552d9ef 100644 --- a/python/envctl_engine/shared/ports.py +++ b/python/envctl_engine/shared/ports.py @@ -1,18 +1,24 @@ from __future__ import annotations +import errno import hashlib import json +import math import os import shutil import socket import subprocess +import threading import time import uuid +from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path from typing import Final from typing import Callable from envctl_engine.state.models import PortPlan +from envctl_engine.state.persistence import advisory_file_lock, fsync_directory _CORE_SERVICE_NAMES: Final[frozenset[str]] = frozenset( @@ -26,6 +32,12 @@ } ) +_PORT_GUARD_SHARDS: Final[int] = 64 +# ``flock`` behavior for separate descriptors in one process differs across +# platforms. A bounded in-process stripe set makes the contract explicit while +# the on-disk guard shards serialize independent processes. +_PORT_THREAD_GUARDS: Final[tuple[threading.RLock, ...]] = tuple(threading.RLock() for _ in range(_PORT_GUARD_SHARDS)) + class PortPlanner: def __init__( @@ -41,6 +53,7 @@ def __init__( lock_dir: str | None = None, session_id: str | None = None, stale_lock_seconds: int = 3600, + corrupt_lock_grace_seconds: float = 1.0, availability_checker: Callable[[int], bool] | None = None, pid_checker: Callable[[int], bool] | None = None, time_provider: Callable[[], float] | None = None, @@ -59,10 +72,11 @@ def __init__( self.supabase_api_base = supabase_api_base self.additional_service_bases = self._normalized_additional_service_bases(additional_service_bases or {}) self.lock_dir = Path(lock_dir or "/tmp/envctl-python-locks") - self.lock_dir.mkdir(parents=True, exist_ok=True) + self._ensure_real_directory(self.lock_dir) self.session_id = session_id or f"session-{uuid.uuid4().hex[:8]}" self.session_pid = os.getpid() self.stale_lock_seconds = max(stale_lock_seconds, 0) + self.corrupt_lock_grace_seconds = max(float(corrupt_lock_grace_seconds), 0.0) self.availability_checker = availability_checker self.pid_checker = pid_checker self.time_provider = time_provider or time.time @@ -75,6 +89,8 @@ def __init__( self.scope_key = str(scope_key or "global").strip() or "global" self.dynamic_main_dependency_ports = bool(dynamic_main_dependency_ports) self.max_port: Final[int] = 65000 + self._guard_dir = self.lock_dir / ".port-guards" + self._thread_guard_namespace = str(self.lock_dir.resolve(strict=False)) def plan_project( self, @@ -196,57 +212,83 @@ def attach_existing_port(self, plan: PortPlan, existing_port: int) -> PortPlan: def release(self, port: int, owner: str | None = None) -> None: lock_path = self._lock_path(port) - if not lock_path.exists(): - return - if owner is None: - lock_path.unlink(missing_ok=True) - self._emit("port.lock.release", port=port, owner=None, session=self.session_id) - return - try: - payload = json.loads(lock_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): + with self._port_guard(port): + payload = self._read_lock_payload(lock_path) + if payload is None or payload.get("session") != self.session_id: + return + if owner is not None and payload.get("owner") != owner: + return lock_path.unlink(missing_ok=True) self._emit("port.lock.release", port=port, owner=owner, session=self.session_id) - return - if payload.get("owner") == owner: + + def release_owned(self, port: int, owner: str, *, expected_session: str) -> bool: + """Release an exact logical owner from one persisted planner session.""" + + normalized_session = expected_session.strip() if isinstance(expected_session, str) else "" + if not normalized_session: + return False + + lock_path = self._lock_path(port) + with self._port_guard(port): + payload = self._read_lock_payload(lock_path) + if payload is None or payload.get("owner") != owner or payload.get("session") != normalized_session: + return False lock_path.unlink(missing_ok=True) - self._emit("port.lock.release", port=port, owner=owner, session=self.session_id) + self._emit("port.lock.release", port=port, owner=owner, session=normalized_session) + return True + + def reap_stale(self, port: int, owner: str | None = None) -> bool: + """Remove one proven-stale reservation without touching a live foreign session.""" + + lock_path = self._lock_path(port) + with self._port_guard(port): + if not lock_path.exists() or not self._lock_is_stale(lock_path): + return False + payload = self._read_lock_payload(lock_path) + if owner is not None and payload is not None and payload.get("owner") != owner: + return False + lock_path.unlink(missing_ok=True) + self._emit_reclaim(port=port, owner=owner, payload=payload) + return True + + def reap_stale_locks(self) -> int: + """Reap every proven-stale numeric reservation in this repository scope.""" + + reaped = 0 + for lock_path in tuple(self.lock_dir.glob("*.lock")): + port = self._port_from_lock_path(lock_path) + if port is not None and self.reap_stale(port): + reaped += 1 + return reaped def release_session(self) -> None: for lock_path in self.lock_dir.glob("*.lock"): - try: - payload = json.loads(lock_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): + port = self._port_from_lock_path(lock_path) + if port is None: + continue + with self._port_guard(port): + payload = self._read_lock_payload(lock_path) + if payload is None or payload.get("session") != self.session_id: + continue + lock_path.unlink(missing_ok=True) + self._emit("port.lock.release", port=port, owner=payload.get("owner"), session=self.session_id) + + def release_all(self) -> None: + for lock_path in self.lock_dir.glob("*.lock"): + port = self._port_from_lock_path(lock_path) + if port is None: lock_path.unlink(missing_ok=True) - self._emit( - "port.lock.release", port=self._port_from_lock_path(lock_path), owner=None, session=self.session_id - ) continue - if payload.get("session") == self.session_id: + with self._port_guard(port): + payload = self._read_lock_payload(lock_path) lock_path.unlink(missing_ok=True) self._emit( "port.lock.release", - port=self._port_from_lock_path(lock_path), - owner=payload.get("owner"), + port=port, + owner=payload.get("owner") if payload is not None else None, session=self.session_id, ) - def release_all(self) -> None: - for lock_path in self.lock_dir.glob("*.lock"): - owner = None - try: - payload = json.loads(lock_path.read_text(encoding="utf-8")) - owner = payload.get("owner") - except (OSError, json.JSONDecodeError): - owner = None - lock_path.unlink(missing_ok=True) - self._emit( - "port.lock.release", - port=self._port_from_lock_path(lock_path), - owner=owner, - session=self.session_id, - ) - def _lock_path(self, port: int) -> Path: return self.lock_dir / f"{port}.lock" @@ -370,87 +412,147 @@ def _normalize_project_identity(project: str) -> str: def _reserve_port(self, port: int, owner: str) -> bool: lock_path = self._lock_path(port) - if lock_path.exists() and self._lock_is_stale(lock_path): - stale_owner = None - stale_session = None - stale_pid = None - try: - stale_payload = json.loads(lock_path.read_text(encoding="utf-8")) - if isinstance(stale_payload, dict): - stale_owner = stale_payload.get("owner") - stale_session = stale_payload.get("session") - stale_pid = stale_payload.get("pid") - except (OSError, json.JSONDecodeError): - stale_owner = None - stale_session = None - stale_pid = None - lock_path.unlink(missing_ok=True) - self._emit( - "port.lock.reclaim", - port=port, - owner=owner, - session=self.session_id, - reclaimed_owner=stale_owner, - reclaimed_session=stale_session, - reclaimed_pid=stale_pid, - ) - if lock_path.exists(): - return False + with self._port_guard(port): + if lock_path.exists(): + if not self._lock_is_stale(lock_path): + return False + stale_payload = self._read_lock_payload(lock_path) + lock_path.unlink(missing_ok=True) + self._emit_reclaim(port=port, owner=owner, payload=stale_payload) - if not self._is_port_available(port): - return False + if not self._is_port_available(port): + return False - payload = { - "owner": owner, - "session": self.session_id, - "pid": self.session_pid, - "created_at": self.time_provider(), - } - try: - fd = lock_path.open("x", encoding="utf-8") - except FileExistsError: - return False - with fd: - fd.write(json.dumps(payload, sort_keys=True)) - self._emit("port.lock.acquire", port=port, owner=owner, session=self.session_id) - return True + payload = { + "owner": owner, + "session": self.session_id, + "pid": self.session_pid, + "created_at": self.time_provider(), + } + if not self._publish_lock(lock_path, payload, port=port): + return False + self._emit("port.lock.acquire", port=port, owner=owner, session=self.session_id) + return True def _lock_is_stale(self, lock_path: Path) -> bool: - try: - payload = json.loads(lock_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return True + payload = self._read_lock_payload(lock_path) + if payload is None: + return self._corrupt_lock_is_stale(lock_path) pid = payload.get("pid") - if isinstance(pid, int) and pid > 0: + if isinstance(pid, int) and not isinstance(pid, bool) and pid > 0: if self._pid_is_running(pid): return False return True created_at_raw = payload.get("created_at") if created_at_raw is None: - return False + return self._corrupt_lock_is_stale(lock_path) created_at = self._parse_created_at(created_at_raw) if created_at is None: - return True + return self._corrupt_lock_is_stale(lock_path) return (self.time_provider() - created_at) > self.stale_lock_seconds def _pid_is_running(self, pid: int) -> bool: if self.pid_checker is not None: - return self.pid_checker(pid) + try: + return bool(self.pid_checker(pid)) + except Exception: # noqa: BLE001 - an inconclusive probe must preserve a foreign lock + return True try: os.kill(pid, 0) - except OSError: + except ProcessLookupError: return False + except PermissionError: + return True + except OSError as exc: + return exc.errno != errno.ESRCH return True + def _corrupt_lock_is_stale(self, lock_path: Path) -> bool: + try: + modified_at = lock_path.stat().st_mtime + except FileNotFoundError: + return True + except OSError: + return False + return (self.time_provider() - modified_at) >= self.corrupt_lock_grace_seconds + + @staticmethod + def _read_lock_payload(lock_path: Path) -> dict[str, object] | None: + try: + payload = json.loads(lock_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + return payload if isinstance(payload, dict) else None + + def _publish_lock(self, lock_path: Path, payload: dict[str, object], *, port: int) -> bool: + """Publish a complete payload without ever exposing an empty lock path.""" + + self._ensure_real_directory(self._guard_dir) + pending_path = self._guard_dir / f"{self._guard_shard(port):02x}.pending" + pending_path.unlink(missing_ok=True) + flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(pending_path, flags, 0o600) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as pending_file: + pending_file.write(json.dumps(payload, sort_keys=True)) + pending_file.flush() + os.fsync(pending_file.fileno()) + try: + os.link(pending_path, lock_path) + except FileExistsError: + return False + fsync_directory(self.lock_dir) + return True + finally: + pending_path.unlink(missing_ok=True) + + def _emit_reclaim(self, *, port: int, owner: str | None, payload: dict[str, object] | None) -> None: + payload = payload or {} + self._emit( + "port.lock.reclaim", + port=port, + owner=owner, + session=self.session_id, + reclaimed_owner=payload.get("owner"), + reclaimed_session=payload.get("session"), + reclaimed_pid=payload.get("pid"), + ) + + @contextmanager + def _port_guard(self, port: int) -> Iterator[None]: + shard = self._guard_shard(port) + self._ensure_real_directory(self._guard_dir) + guard_path = self._guard_dir / f"{shard:02x}.guard" + thread_key = f"{self._thread_guard_namespace}:{shard}".encode("utf-8") + thread_index = int(hashlib.sha1(thread_key).hexdigest()[:8], 16) % len(_PORT_THREAD_GUARDS) + with _PORT_THREAD_GUARDS[thread_index]: + with advisory_file_lock(guard_path, exclusive=True): + yield + + @staticmethod + def _guard_shard(port: int) -> int: + return int(port) % _PORT_GUARD_SHARDS + + @staticmethod + def _ensure_real_directory(path: Path) -> None: + try: + path.mkdir(parents=True, exist_ok=False) + except FileExistsError: + pass + if path.is_symlink() or not path.is_dir(): + raise OSError(f"Port reservation directory is not a real directory: {path}") + def _parse_created_at(self, value: object) -> float | None: - if isinstance(value, (int, float)): - return float(value) + if isinstance(value, (int, float)) and not isinstance(value, bool): + parsed = float(value) + return parsed if math.isfinite(parsed) else None if not isinstance(value, str): return None try: - return float(value) + parsed = float(value) + return parsed if math.isfinite(parsed) else None except ValueError: pass iso = value @@ -459,8 +561,9 @@ def _parse_created_at(self, value: object) -> float | None: try: from datetime import datetime - return datetime.fromisoformat(iso).timestamp() - except ValueError: + parsed = datetime.fromisoformat(iso).timestamp() + return parsed if math.isfinite(parsed) else None + except (OverflowError, ValueError): return None def _is_port_available(self, port: int) -> bool: @@ -545,7 +648,10 @@ def _is_port_available_via_docker_publish_filter(self, port: int) -> bool: def _emit(self, event_name: str, **payload: object) -> None: if self.event_handler is None: return - self.event_handler(event_name, payload) + try: + self.event_handler(event_name, payload) + except Exception: # noqa: BLE001 - telemetry must never change lock ownership + return @staticmethod def _port_from_lock_path(lock_path: Path) -> int | None: diff --git a/python/envctl_engine/shared/process_cwd.py b/python/envctl_engine/shared/process_cwd.py new file mode 100644 index 000000000..5b2668a47 --- /dev/null +++ b/python/envctl_engine/shared/process_cwd.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import os +import subprocess +from collections.abc import Callable, Sequence +from typing import Any + + +def process_cwd( + pid: int, + *, + run_probe: Callable[[Sequence[str]], Any] | None = None, +) -> str | None: + """Read a process CWD portably, returning only explicit probe evidence.""" + + if pid <= 0: + return None + try: + return os.readlink(f"/proc/{pid}/cwd") + except OSError: + pass + + command = ["lsof", "-a", "-p", str(pid), "-d", "cwd", "-Fn"] + try: + completed = ( + run_probe(command) + if run_probe is not None + else subprocess.run( # noqa: S603 + command, + text=True, + encoding="utf-8", + errors="replace", + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=1.0, + check=False, + ) + ) + except (OSError, subprocess.TimeoutExpired): + return None + try: + returncode = int(getattr(completed, "returncode", 1)) + except (TypeError, ValueError): + return None + if returncode != 0: + return None + return parse_lsof_cwd(str(getattr(completed, "stdout", "") or "")) + + +def parse_lsof_cwd(output: str) -> str | None: + for line in output.splitlines(): + if line.startswith("n") and len(line) > 1: + return line[1:] + return None diff --git a/python/envctl_engine/shared/process_lifecycle_probe.py b/python/envctl_engine/shared/process_lifecycle_probe.py index f72a7c4ed..018d5a238 100644 --- a/python/envctl_engine/shared/process_lifecycle_probe.py +++ b/python/envctl_engine/shared/process_lifecycle_probe.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable +import errno import os import re import shutil @@ -8,6 +9,8 @@ import socket import time +from envctl_engine.shared.process_cwd import process_cwd as read_process_cwd + class ProcessLifecycleProbeMixin: def is_pid_running(self, pid: int) -> bool: @@ -15,8 +18,12 @@ def is_pid_running(self, pid: int) -> bool: return False try: os.kill(pid, 0) - except OSError: + except ProcessLookupError: return False + except PermissionError: + return True + except OSError as exc: + return exc.errno != errno.ESRCH identity = self._pid_identity(pid) if identity is not None: try: @@ -29,6 +36,9 @@ def is_pid_running(self, pid: int) -> bool: return False return True + def process_cwd(self, pid: int) -> str | None: + return read_process_cwd(pid, run_probe=self.run_probe) + def wait_for_port(self, port: int, *, host: str = "127.0.0.1", timeout: float = 30.0) -> bool: if port <= 0: return False @@ -311,7 +321,7 @@ def _terminate_with_signal_sender( try: signal_sender(pid, signal.SIGTERM) except OSError: - return True + return not self.is_pid_running(pid) deadline = time.monotonic() + max(term_timeout, 0.0) while time.monotonic() < deadline: @@ -330,7 +340,7 @@ def _terminate_with_signal_sender( return True signal_sender(pid, signal.SIGKILL) except OSError: - return True + return not self.is_pid_running(pid) kill_deadline = time.monotonic() + max(kill_timeout, 0.0) while time.monotonic() < kill_deadline: @@ -356,12 +366,100 @@ def terminate(self, pid: int, *, term_timeout: float = 2.0, kill_timeout: float ) def terminate_process_group(self, pid: int, *, term_timeout: float = 2.0, kill_timeout: float = 1.0) -> bool: - return self._terminate_with_signal_sender( - pid, - term_timeout=term_timeout, - kill_timeout=kill_timeout, - signal_sender=os.killpg, - ) + if pid <= 0: + return True + try: + process_group_id = os.getpgid(pid) + except ProcessLookupError: + return not self.is_pid_running(pid) + except (PermissionError, OSError): + return False + if process_group_id <= 0: + return False + if process_group_id == os.getpgrp(): + return False + + initial_members = self._process_group_member_pids(process_group_id) + initial_identities = {member_pid: self._pid_identity(member_pid) for member_pid in initial_members or ()} + try: + os.killpg(process_group_id, signal.SIGTERM) + except OSError: + return not self._process_group_is_running(process_group_id) and not self.is_pid_running(pid) + + if self._wait_for_process_group_exit(process_group_id, timeout=term_timeout): + return True + if not self._process_group_escalation_is_safe( + process_group_id, + initial_members=initial_members, + initial_identities=initial_identities, + ): + return False + try: + os.killpg(process_group_id, signal.SIGKILL) + except OSError: + return not self._process_group_is_running(process_group_id) and not self.is_pid_running(pid) + return self._wait_for_process_group_exit(process_group_id, timeout=kill_timeout) + + def _process_group_member_pids(self, process_group_id: int) -> set[int] | None: + try: + completed = self.run_probe(["ps", "-axo", "pid=,pgid=,stat="]) + except OSError: + return None + if completed.returncode != 0: + return None + + members: set[int] = set() + for line in completed.stdout.splitlines(): + parts = line.strip().split(maxsplit=2) + if len(parts) != 3 or not parts[0].isdigit() or not parts[1].isdigit(): + continue + if int(parts[1]) != process_group_id or parts[2].upper().startswith("Z"): + continue + members.add(int(parts[0])) + return members + + def _process_group_is_running(self, process_group_id: int) -> bool: + members = self._process_group_member_pids(process_group_id) + if members is not None: + return bool(members) + try: + os.killpg(process_group_id, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError as exc: + return exc.errno != errno.ESRCH + return True + + def _wait_for_process_group_exit(self, process_group_id: int, *, timeout: float) -> bool: + deadline = time.monotonic() + max(timeout, 0.0) + while True: + if not self._process_group_is_running(process_group_id): + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + time.sleep(min(0.05, remaining)) + + def _process_group_escalation_is_safe( + self, + process_group_id: int, + *, + initial_members: set[int] | None, + initial_identities: dict[int, str | None], + ) -> bool: + if process_group_id == os.getpgrp(): + return False + current_members = self._process_group_member_pids(process_group_id) + if current_members is None or initial_members is None: + return self._process_group_is_running(process_group_id) + for member_pid in initial_members.intersection(current_members): + initial_identity = initial_identities.get(member_pid) + current_identity = self._pid_identity(member_pid) + if initial_identity is None or current_identity is None or current_identity == initial_identity: + return True + return False def _pid_identity(self, pid: int) -> str | None: if pid <= 0: diff --git a/python/envctl_engine/shared/process_termination.py b/python/envctl_engine/shared/process_termination.py new file mode 100644 index 000000000..646e2d561 --- /dev/null +++ b/python/envctl_engine/shared/process_termination.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import errno +import os +import signal +import time +from collections.abc import Callable +from typing import Any + +_IDENTITY_UNSET = object() + + +def terminate_pid( + pid: int | None, + *, + process_runner: object | None, + term_timeout: float, + kill_timeout: float, +) -> bool: + """Terminate a PID without treating an attempted signal as proof of exit. + + Process-runner implementations get the first opportunity to terminate the + process group and then the individual process. A false result (or an + exception) is not success: the raw-signal fallback confirms exit and + escalates to SIGKILL when necessary. + """ + + if not isinstance(pid, int) or pid <= 0: + return True + if pid in {os.getpid(), os.getppid()}: + return False + if not pid_is_running(process_runner, pid): + return True + initial_identity = pid_identity(process_runner, pid) + + group_terminator = getattr(process_runner, "terminate_process_group", None) + if callable(group_terminator): + try: + terminated = bool(group_terminator(pid, term_timeout=term_timeout, kill_timeout=kill_timeout)) + except Exception: # noqa: BLE001 + if callable(getattr(process_runner, "_process_group_is_running", None)): + return False + if _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity): + return False + else: + if not terminated: + # Killing only the recorded leader after group cleanup failed + # can orphan descendants and must never be reported as success. + return False + return _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity) + + terminator = getattr(process_runner, "terminate", None) + if callable(terminator): + try: + terminated = bool(terminator(pid, term_timeout=term_timeout, kill_timeout=kill_timeout)) + except Exception: # noqa: BLE001 + if _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity): + return True + else: + if _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity): + return True + if terminated: + # A capable runner that claims success while the same PID remains + # alive is not proof. Do not signal again: the PID may have raced + # with reuse between the runner and this verification. + return False + + return terminate_with_raw_signals( + process_runner, + pid, + term_timeout=term_timeout, + kill_timeout=kill_timeout, + initial_identity=initial_identity, + ) + + +def terminate_with_raw_signals( + process_runner: object | None, + pid: int, + *, + term_timeout: float, + kill_timeout: float, + initial_identity: str | None | object = _IDENTITY_UNSET, +) -> bool: + """Last-resort termination that proves the recorded process exited.""" + + if initial_identity is _IDENTITY_UNSET: + initial_identity = pid_identity(process_runner, pid) + assert isinstance(initial_identity, str) or initial_identity is None + if _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity): + return True + term_sent = _send_signal(pid, signal.SIGTERM) + if term_sent is None: + return True + if not term_sent: + return not pid_is_running(process_runner, pid) + if wait_for_pid_exit( + process_runner, + pid, + timeout=term_timeout, + initial_identity=initial_identity, + ): + return True + kill_sent = _send_signal(pid, signal.SIGKILL) + if kill_sent is None: + return True + if not kill_sent: + return not pid_is_running(process_runner, pid) + return wait_for_pid_exit( + process_runner, + pid, + timeout=kill_timeout, + initial_identity=initial_identity, + ) + + +def _pid_gone_or_replaced( + process_runner: object | None, + pid: int, + *, + initial_identity: str | None, +) -> bool: + if not pid_is_running(process_runner, pid): + return True + if initial_identity is None: + return False + current_identity = pid_identity(process_runner, pid) + return current_identity is None or current_identity != initial_identity + + +def wait_for_pid_exit( + process_runner: object | None, + pid: int, + *, + timeout: float, + initial_identity: str | None, +) -> bool: + deadline = time.monotonic() + max(timeout, 0.0) + while True: + if not pid_is_running(process_runner, pid): + return True + if initial_identity is not None: + current_identity = pid_identity(process_runner, pid) + if current_identity is None or current_identity != initial_identity: + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + time.sleep(min(0.05, remaining)) + + +def pid_is_running(process_runner: object | None, pid: int) -> bool: + checker = getattr(process_runner, "is_pid_running", None) + if callable(checker): + try: + return bool(checker(pid)) + except Exception: # noqa: BLE001 + pass + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError as exc: + return exc.errno != errno.ESRCH + return True + + +def pid_identity(process_runner: object | None, pid: int) -> str | None: + identity_reader = getattr(process_runner, "_pid_identity", None) + if not callable(identity_reader): + return None + try: + identity = identity_reader(pid) + except Exception: # noqa: BLE001 + return None + if not isinstance(identity, str): + return None + normalized = identity.strip() + return normalized or None + + +def _send_signal(pid: int, requested_signal: signal.Signals) -> bool | None: + sender: Callable[[int, signal.Signals], Any] = os.kill + try: + sender(pid, requested_signal) + except ProcessLookupError: + return None + except OSError: + return False + return True diff --git a/python/envctl_engine/shared/protocols.py b/python/envctl_engine/shared/protocols.py index 07facf1b7..fd2d8fa0b 100644 --- a/python/envctl_engine/shared/protocols.py +++ b/python/envctl_engine/shared/protocols.py @@ -72,6 +72,8 @@ def update_final_port(self, plan: object, final_port: int, source: str = "retry" def release(self, port: int, owner: str | None = None) -> None: ... + def release_owned(self, port: int, owner: str, *, expected_session: str) -> bool: ... + def release_session(self) -> None: ... def release_all(self) -> None: ... @@ -108,11 +110,32 @@ def save_selected_stop_state( state: object, emit: Callable[..., None], runtime_map_builder: Callable[[object], dict[str, object]], + authoritative_project_names: Sequence[str] | None = None, ) -> dict[str, object]: ... - def load_latest(self, *, mode: str | None = None, strict_mode_match: bool = False) -> object | None: ... + def load_latest( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, + ) -> object | None: ... + + def load_all(self, *, mode: str | None = None) -> Sequence[object]: ... + + def has_active_runs(self) -> bool: ... + + def write_runtime_artifact( + self, + *, + run_id: str | None, + artifact_name: str, + text: str, + ) -> bool: ... + + def deactivate_run(self, run_id: str) -> bool: ... - def load_by_pointer(self, pointer_path: str) -> object: ... + def deactivate_runs(self, run_ids: Sequence[str]) -> bool: ... def purge(self, *, aggressive: bool = False) -> None: ... diff --git a/python/envctl_engine/startup/disabled_startup_resolution.py b/python/envctl_engine/startup/disabled_startup_resolution.py index 6f4801f52..ccd03fdfd 100644 --- a/python/envctl_engine/startup/disabled_startup_resolution.py +++ b/python/envctl_engine/startup/disabled_startup_resolution.py @@ -6,6 +6,7 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_env import route_is_implicit_start +from envctl_engine.runtime.lifecycle_operation_lease import release_lifecycle_operation from envctl_engine.startup.finalization import ( build_planning_dashboard_state, headless_plan_output_only, @@ -68,6 +69,7 @@ def resolve_disabled_startup_mode( artifacts_started = time.monotonic() runtime._write_artifacts(run_state, session.selected_contexts, errors=[]) emit_phase(session, "artifacts_write", artifacts_started, status="ok") + release_lifecycle_operation(runtime) if route.command == "plan": validate_plan_agent_handoff(session, phase="disabled_startup_finalization") print_plan_dry_run_preview(session) @@ -139,13 +141,11 @@ def validate_plan_agent_handoff(session: StartupSession, *, phase: str) -> None: session=session, validate_plan_agent_handoff=validate_plan_agent_handoff, attach_plan_agent_terminal=attach_plan_agent_terminal, - print_headless_plan_session_summary=lambda session, *, attach_target: ( - print_headless_plan_session_summary( - session, - validate_plan_agent_handoff=validate_plan_agent_handoff, - print_fn=print_fn, - attach_target=attach_target, - ) + print_headless_plan_session_summary=lambda session, *, attach_target: print_headless_plan_session_summary( + session, + validate_plan_agent_handoff=validate_plan_agent_handoff, + print_fn=print_fn, + attach_target=attach_target, ), ), print_fn=print_fn, diff --git a/python/envctl_engine/startup/finalization.py b/python/envctl_engine/startup/finalization.py index 77ce8611d..0796de926 100644 --- a/python/envctl_engine/startup/finalization.py +++ b/python/envctl_engine/startup/finalization.py @@ -6,6 +6,7 @@ from envctl_engine.planning.plan_agent.tmux_transport import attach_plan_agent_terminal from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.lifecycle_operation_lease import release_lifecycle_operation from envctl_engine.startup.finalization_plan_output import ( format_degraded_handoff_text_for_terminal, headless_plan_output_only, @@ -106,6 +107,7 @@ def finalize_successful_startup( artifacts_started = time.monotonic() runtime._write_artifacts(run_state, session.selected_contexts, errors=session.errors) emit_phase(session, "artifacts_write", artifacts_started, status="ok") + release_lifecycle_operation(runtime) if requirements_timing_enabled(session.effective_route) and not suppress_timing_output(session.effective_route): runtime._emit( "startup.debug_tty_group", @@ -213,13 +215,11 @@ def finalize_successful_startup_with_runtime( session=session, validate_plan_agent_handoff=validate_plan_agent_handoff, attach_plan_agent_terminal=attach_plan_agent_terminal, - print_headless_plan_session_summary=lambda session, *, attach_target: ( - print_headless_plan_session_summary( - session, - validate_plan_agent_handoff=validate_plan_agent_handoff, - print_fn=print_fn, - attach_target=attach_target, - ) + print_headless_plan_session_summary=lambda session, *, attach_target: print_headless_plan_session_summary( + session, + validate_plan_agent_handoff=validate_plan_agent_handoff, + print_fn=print_fn, + attach_target=attach_target, ), ), finalize_plan_agent_degraded_handoff=lambda session: finalize_plan_agent_degraded_handoff_with_runtime( @@ -257,13 +257,11 @@ def finalize_plan_agent_degraded_handoff_with_runtime( session=session, validate_plan_agent_handoff=validate_plan_agent_handoff, attach_plan_agent_terminal=attach_plan_agent_terminal, - print_headless_plan_session_summary=lambda session, *, attach_target: ( - print_headless_plan_session_summary( - session, - validate_plan_agent_handoff=validate_plan_agent_handoff, - print_fn=print_fn, - attach_target=attach_target, - ) + print_headless_plan_session_summary=lambda session, *, attach_target: print_headless_plan_session_summary( + session, + validate_plan_agent_handoff=validate_plan_agent_handoff, + print_fn=print_fn, + attach_target=attach_target, ), ), ) @@ -289,6 +287,7 @@ def finalize_plan_agent_degraded_handoff( build_success_run_state=build_success_run_state, emit_phase=emit_phase, ) + release_lifecycle_operation(runtime) render_plan_agent_degraded_handoff(session) if headless_plan_output_only(session): return 0 diff --git a/python/envctl_engine/startup/finalization_failure.py b/python/envctl_engine/startup/finalization_failure.py index d5d9928fc..75a14cf5c 100644 --- a/python/envctl_engine/startup/finalization_failure.py +++ b/python/envctl_engine/startup/finalization_failure.py @@ -8,8 +8,10 @@ from typing import Any from envctl_engine.startup.protocols import ProjectContextLike, StartupRuntime -from envctl_engine.startup.session import StartupSession +from envctl_engine.startup.session import StartupSession, unconfirmed_service_names from envctl_engine.startup.finalization_run_state import _build_run_state +from envctl_engine.requirements.core import dependency_definitions +from envctl_engine.runtime.lifecycle_requirement_ports import component_port_values from envctl_engine.state.models import RunState from envctl_engine.ui.color_policy import colors_enabled from envctl_engine.ui.path_links import local_paths_in_text, render_paths_in_terminal_text @@ -62,43 +64,105 @@ def _emit_failure(self, final_error: str) -> None: } if self.session.strict_truth_failed: failure_payload["services"] = sorted(self.session.merged_services) - self.runtime._emit("startup.failed", **failure_payload) + try: + self.runtime._emit("startup.failed", **failure_payload) + except Exception: # noqa: BLE001 + pass def _terminate_started_services(self) -> None: - started_services = { - service_name: service - for project_name in self.session.started_context_names - for service_name, service in self.session.services_by_project.get(project_name, {}).items() - } + started_services = dict(self.session.unterminated_services) + started_services.update( + { + service_name: service + for project_name in self.session.started_context_names + for service_name, service in self.session.services_by_project.get(project_name, {}).items() + } + ) if started_services: - self.runtime._terminate_started_services(started_services) + try: + failed_names = unconfirmed_service_names( + self.runtime._terminate_started_services(started_services), + started_services, + ) + except Exception: # noqa: BLE001 + failed_names = set(started_services) + for project_name in list(self.session.services_by_project): + retained = { + name: service + for name, service in self.session.services_by_project[project_name].items() + if name in failed_names + } + if retained: + self.session.services_by_project[project_name] = retained + else: + self.session.services_by_project.pop(project_name, None) + self.session.unterminated_services = { + name: service for name, service in started_services.items() if name in failed_names + } + for service in self.session.unterminated_services.values(): + service.status = "termination_failed" + service.degraded = True def _release_ports(self) -> None: - self.port_allocator(self.runtime).release_session() + if self.session.unterminated_services or _has_retained_managed_requirements(self.session): + try: + self.runtime._emit( + "port.session_release.skipped", + reason=( + "unterminated_services" + if self.session.unterminated_services + else "started_requirements_remain_tracked" + ), + services=sorted(self.session.unterminated_services), + ) + except Exception: # noqa: BLE001 + pass + return + try: + self.port_allocator(self.runtime).release_session() + except Exception: # noqa: BLE001 + pass def _write_failure_artifacts(self, final_error: str) -> None: - run_state = build_failure_run_state(self.runtime, self.session, final_error) artifacts_started = time.monotonic() + if self.session.preserve_existing_state_on_failure: + try: + self.runtime._emit( + "state.failure_write.skipped", + reason="existing_runtime_state_remains_authoritative", + run_id=self.session.run_id, + ) + self.emit_phase(self.session, "artifacts_write", artifacts_started, status="skipped") + except Exception: # noqa: BLE001 + pass + return + run_state = build_failure_run_state(self.runtime, self.session, final_error) self.runtime._write_artifacts(run_state, self.session.selected_contexts, errors=self.session.errors) - self.emit_phase(self.session, "artifacts_write", artifacts_started, status="error") + try: + self.emit_phase(self.session, "artifacts_write", artifacts_started, status="error") + except Exception: # noqa: BLE001 + pass def _print_final_status(self, final_error: str) -> None: - interactive_tty = failure_hyperlink_interactive_tty(self.runtime) - rendered_error = self.render_final_failure_status( - self.runtime, - self.session, - final_error, - interactive_tty=interactive_tty, - ) - self.print_fn( - render_paths_in_terminal_text( - rendered_error, - paths=local_paths_in_text(rendered_error), - env=self.runtime.env, - stream=self.output_stream, + try: + interactive_tty = failure_hyperlink_interactive_tty(self.runtime) + rendered_error = self.render_final_failure_status( + self.runtime, + self.session, + final_error, interactive_tty=interactive_tty, ) - ) + self.print_fn( + render_paths_in_terminal_text( + rendered_error, + paths=local_paths_in_text(rendered_error), + env=self.runtime.env, + stream=self.output_stream, + interactive_tty=interactive_tty, + ) + ) + except Exception: # noqa: BLE001 + pass def finalize_failed_startup( @@ -122,6 +186,21 @@ def finalize_failed_startup( ).finalize() +def _has_retained_managed_requirements(session: StartupSession) -> bool: + for requirements in session.requirements_by_project.values(): + for definition in dependency_definitions(): + component = requirements.component(definition.id) + if not bool(component.get("enabled", False)): + continue + if bool(component.get("external", False)) or bool(component.get("simulated", False)): + continue + if not bool(component.get("success", False)) and not str(component.get("container_name") or "").strip(): + continue + if component_port_values(component) or str(component.get("container_name") or "").strip(): + return True + return False + + def failure_hyperlink_interactive_tty(runtime: StartupRuntime) -> bool | None: link_mode = str(runtime.env.get("ENVCTL_UI_HYPERLINK_MODE", "")).strip().lower() return True if link_mode == "on" else None diff --git a/python/envctl_engine/startup/finalization_run_state.py b/python/envctl_engine/startup/finalization_run_state.py index 37a34cc90..8ca2f0c02 100644 --- a/python/envctl_engine/startup/finalization_run_state.py +++ b/python/envctl_engine/startup/finalization_run_state.py @@ -101,6 +101,8 @@ def _build_run_state(runtime: StartupRuntime, session: StartupSession, *, failed metadata["dashboard_shared_dependency_project"] = shared_dependency_project if session.warnings: metadata["warnings"] = list(session.warnings) + if session.unterminated_services: + metadata["termination_failed_services"] = sorted(session.unterminated_services) if session.plan_agent_launch_result is not None: launch_result = session.plan_agent_launch_result metadata["plan_agent_launch_status"] = str(getattr(launch_result, "status", "")).strip() @@ -197,9 +199,7 @@ def _build_run_state(runtime: StartupRuntime, session: StartupSession, *, failed return run_state -def _project_configured_services_metadata( - runtime: StartupRuntime, session: StartupSession -) -> dict[str, list[str]]: +def _project_configured_services_metadata(runtime: StartupRuntime, session: StartupSession) -> dict[str, list[str]]: configured: dict[str, list[str]] = {} for context in session.selected_contexts: service_types = [ diff --git a/python/envctl_engine/startup/protocols.py b/python/envctl_engine/startup/protocols.py index 29ff99624..61b3f05a0 100644 --- a/python/envctl_engine/startup/protocols.py +++ b/python/envctl_engine/startup/protocols.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from pathlib import Path from typing import Protocol @@ -40,8 +40,14 @@ def _requirements_ready(self, result: RequirementsResult) -> bool: ... def _assert_project_services_post_start_truth( self, *, context: ProjectContextLike, services: dict[str, ServiceRecord] ) -> None: ... - def _terminate_started_services(self, services: dict[str, ServiceRecord]) -> None: ... - def _try_load_existing_state(self, *, mode: str, strict_mode_match: bool = False) -> RunState | None: ... + def _terminate_started_services(self, services: dict[str, ServiceRecord]) -> set[str]: ... + def _try_load_existing_state( + self, + *, + mode: str, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, + ) -> RunState | None: ... def _requirement_enabled(self, requirement_name: str, *, mode: str, route: Route | None = None) -> bool: ... def _service_enabled_for_mode(self, mode: str, service_name: str) -> bool: ... def _set_plan_port_from_component(self, plan: PortPlan, component: dict[str, object]) -> None: ... @@ -86,7 +92,7 @@ def _terminate_services_from_state( selected_services: set[str] | None = None, aggressive: bool = False, verify_ownership: bool = False, - ) -> None: ... + ) -> set[str]: ... def _enforce_runtime_readiness_contract(self, *, scope: str) -> bool: ... def _new_run_id(self) -> str: ... def _current_session_id(self) -> str | None: ... diff --git a/python/envctl_engine/startup/requirements_component_startup.py b/python/envctl_engine/startup/requirements_component_startup.py index a251b2479..72e142912 100644 --- a/python/envctl_engine/startup/requirements_component_startup.py +++ b/python/envctl_engine/startup/requirements_component_startup.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Protocol +from envctl_engine.requirements.core import dependency_definitions from envctl_engine.requirements.orchestrator import FailureClass, RequirementOutcome from envctl_engine.requirements.supabase import ( evaluate_managed_supabase_reliability_contract, @@ -273,6 +274,8 @@ def _on_retry( failure_class: FailureClass, error: str | None, ) -> None: + if retry_port != failed_port: + self._release_failed_requirement_port(failed_port) reason_code = self.runtime.requirements.reason_code_for_failure(service_name, failure_class, error=error) self.runtime._emit( "requirements.retry", @@ -287,6 +290,23 @@ def _on_retry( error=(error or "").strip() or None, ) + def _release_failed_requirement_port(self, port: int) -> None: + release = getattr(self.runtime.port_planner, "release", None) + if not callable(release): + return + resource_keys = { + resource.legacy_port_key + for definition in dependency_definitions() + if definition.id == self.name + for resource in definition.resources + } + for owner_suffix in (*sorted(resource_keys), "requirements"): + try: + release(port, owner=f"{self.context.name}:{owner_suffix}") + except TypeError: + release(port) + return + def _finalize_outcome(self, outcome: RequirementOutcome) -> RequirementOutcome: if outcome.success and isinstance(self.native_effective_port, int) and self.native_effective_port > 0: outcome.final_port = self.native_effective_port diff --git a/python/envctl_engine/startup/requirements_execution.py b/python/envctl_engine/startup/requirements_execution.py index cf02b0f41..0845bcace 100644 --- a/python/envctl_engine/startup/requirements_execution.py +++ b/python/envctl_engine/startup/requirements_execution.py @@ -17,6 +17,7 @@ ) from envctl_engine.startup.startup_progress import suppress_timing_output from envctl_engine.startup.startup_selection_support import restart_include_requirements +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RequirementsResult _DOCKER_SOCKET_PATTERNS = ( @@ -61,7 +62,12 @@ def requirements_for_restart_context( if restart_include_requirements(route): return orchestrator.start_requirements_for_project(context, mode=mode, route=route) - previous = rt._try_load_existing_state(mode=mode, strict_mode_match=True) + previous = call_state_loader( + rt._try_load_existing_state, + mode=mode, + strict_mode_match=True, + project_names=[context.name], + ) if previous is not None: existing = previous.requirements.get(context.name) if isinstance(existing, RequirementsResult): diff --git a/python/envctl_engine/startup/requirements_project_startup.py b/python/envctl_engine/startup/requirements_project_startup.py index 9747253e8..914224d02 100644 --- a/python/envctl_engine/startup/requirements_project_startup.py +++ b/python/envctl_engine/startup/requirements_project_startup.py @@ -15,7 +15,7 @@ external_dependency_resources, external_dependency_url, ) -from envctl_engine.requirements.orchestrator import RequirementOutcome +from envctl_engine.requirements.orchestrator import FailureClass, RequirementOutcome from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.runtime_context import resolve_port_allocator from envctl_engine.shared.parsing import parse_bool, parse_int @@ -27,6 +27,12 @@ REQUIREMENTS_PROGRESS_PROJECT_FLAG = "_requirements_progress_project" +def _has_reserved_ports(final_port: object, resources: dict[str, int]) -> bool: + return (isinstance(final_port, int) and not isinstance(final_port, bool) and final_port > 0) or any( + isinstance(port, int) and not isinstance(port, bool) and port > 0 for port in resources.values() + ) + + def requirements_parallel_platform_default() -> bool: return sys.platform != "darwin" @@ -248,31 +254,62 @@ def _run_components_parallel(self, enabled_definitions: list[Any], *, worker_cou future_map[future] = definition.id for future in concurrent.futures.as_completed(future_map): definition_id = future_map[future] - self.outcomes[definition_id] = future.result() + try: + self.outcomes[definition_id] = future.result() + except Exception as exc: # noqa: BLE001 + self.outcomes[definition_id] = self._failed_outcome( + definition_id, + self._plan_for_dependency(definition_id), + exc, + ) def _run_component(self, component: str, plan: object, *, strict: bool = False) -> RequirementOutcome: started = time.monotonic() with self.progress_state_lock: self.pending_requirements.discard(component) self.active_requirements.add(component) - self._emit_progress() + self._emit_progress_safely() try: - outcome = self.orchestrator.runtime._start_requirement_component( - self.context, - component, - plan, - self._reserve_next, - strict=strict, - route=self.route, - ) + try: + outcome = self.orchestrator.runtime._start_requirement_component( + self.context, + component, + plan, + self._reserve_next, + strict=strict, + route=self.route, + ) + except Exception as exc: # noqa: BLE001 + outcome = self._failed_outcome(component, plan, exc) duration_ms = round((time.monotonic() - started) * 1000.0, 2) self.component_timings_ms[component] = duration_ms - self._emit_component_timing(component, outcome=outcome, enabled=True, duration_ms=duration_ms) + try: + self._emit_component_timing(component, outcome=outcome, enabled=True, duration_ms=duration_ms) + except Exception: # noqa: BLE001 + pass return outcome finally: with self.progress_state_lock: self.active_requirements.discard(component) - self._emit_progress() + self._emit_progress_safely() + + @staticmethod + def _failed_outcome(component: str, plan: object, error: BaseException) -> RequirementOutcome: + return RequirementOutcome( + service_name=component, + success=False, + requested_port=int(getattr(plan, "requested", 0) or 0), + final_port=int(getattr(plan, "final", 0) or 0), + retries=0, + failure_class=FailureClass.HARD_START_FAILURE, + error=str(error), + ) + + def _emit_progress_safely(self) -> None: + try: + self._emit_progress() + except Exception: # noqa: BLE001 + return def _build_components( self, @@ -301,6 +338,18 @@ def _build_components( "error": outcome.error, "container_name": outcome.container_name, } + if ( + enabled_lookup[definition.id] + and not external_lookup[definition.id] + and _has_reserved_ports( + outcome.final_port, + resources, + ) + ): + raw_session_id = getattr(resolve_port_allocator(rt), "session_id", None) + session_id = raw_session_id.strip() if isinstance(raw_session_id, str) else "" + if session_id: + components[definition.id]["port_lock_session"] = session_id if external_lookup[definition.id]: components[definition.id].update( { @@ -318,9 +367,7 @@ def _emit_progress(self) -> None: return progress_project_flag = self.route.flags.get(REQUIREMENTS_PROGRESS_PROJECT_FLAG) progress_project = ( - str(progress_project_flag).strip() - if progress_project_flag is not None - else self.context.name + str(progress_project_flag).strip() if progress_project_flag is not None else self.context.name ) progress_message = format_requirements_progress_message( active=self.active_requirements, diff --git a/python/envctl_engine/startup/restart_prestop_support.py b/python/envctl_engine/startup/restart_prestop_support.py index aa0d1e981..639387cbd 100644 --- a/python/envctl_engine/startup/restart_prestop_support.py +++ b/python/envctl_engine/startup/restart_prestop_support.py @@ -1,11 +1,13 @@ from __future__ import annotations -from dataclasses import dataclass -from pathlib import Path +from dataclasses import dataclass, field from typing import Any, Callable, Iterable from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.lifecycle_requirement_ports import release_port_reservation +from envctl_engine.state.lookup import call_state_loader from envctl_engine.runtime.runtime_context import resolve_port_allocator, resolve_process_runtime +from envctl_engine.shared.process_cwd import process_cwd from envctl_engine.shared.services import service_project_name, service_slug_from_record from envctl_engine.startup.startup_selection_support import ( _restart_selected_services, @@ -13,6 +15,8 @@ restart_target_projects, restart_target_projects_for_selected_services, ) +from envctl_engine.startup.run_reuse_dashboard_restore import metadata_without_dashboard_stopped_services +from envctl_engine.startup.session import metadata_with_state_sources @dataclass(frozen=True, slots=True) @@ -40,19 +44,16 @@ class RestartPrestopSelection: class RestartOrphanListenerScan: ports_by_type: dict[str, set[int]] selected_by_cwd: dict[str, set[str]] + owners_by_cwd_type: dict[tuple[str, str], tuple[str, ...]] = field(default_factory=dict) + port_lock_sessions_by_cwd_type: dict[tuple[str, str], tuple[str, ...]] = field(default_factory=dict) @dataclass(frozen=True, slots=True) class RestartOrphanListenerMatch: pid: int port: int - - -def process_cwd(pid: int) -> str | None: - try: - return str(Path(f"/proc/{pid}/cwd").resolve()) - except OSError: - return None + owner_candidates: tuple[str, ...] = () + port_lock_sessions: tuple[str, ...] = () def restart_fallback_start_route(route: Route, *, restart_lookup_mode: str) -> Route: @@ -68,7 +69,11 @@ def restart_fallback_start_route(route: Route, *, restart_lookup_mode: str) -> R def restart_prestop_state(*, route: Route, runtime: Any) -> RestartPrestopState: restart_lookup_mode = runtime._effective_start_mode(route) - resumed = runtime._try_load_existing_state(mode=restart_lookup_mode) + resumed = call_state_loader( + runtime._try_load_existing_state, + mode=restart_lookup_mode, + project_names=route.projects or None, + ) if resumed is not None and resumed.mode != restart_lookup_mode: runtime._emit( "restart.state_mode_mismatch", @@ -133,7 +138,7 @@ def handle_restart_prestop( runtime: Any, session: Any, suppress_progress_output: Callable[[Route], bool], - terminate_restart_orphan_listeners: Callable[..., None], + terminate_restart_orphan_listeners: Callable[..., set[int]], spinner_factory: Callable[..., Any], use_spinner_policy_fn: Callable[[Any], Any], resolve_spinner_policy_fn: Callable[[dict[str, str]], Any], @@ -181,17 +186,30 @@ def handle_restart_prestop( message="Restarting services...", ) try: - runtime._terminate_services_from_state( + termination_result = runtime._terminate_services_from_state( resumed, selected_services=selected_services, aggressive=False, verify_ownership=True, ) - terminate_restart_orphan_listeners( + failed_services = _failed_service_names(termination_result) + if failed_services: + session.preserve_existing_state_on_failure = True + raise RuntimeError( + "Restart aborted because existing services could not be stopped: " + + ", ".join(sorted(failed_services)) + ) + failed_orphan_pids = terminate_restart_orphan_listeners( state=resumed, selected_services=selected_services, aggressive=True, ) + if failed_orphan_pids: + session.preserve_existing_state_on_failure = True + raise RuntimeError( + "Restart aborted because orphan listeners could not be stopped: " + + ", ".join(str(pid) for pid in sorted(failed_orphan_pids)) + ) preservation = restart_prestop_preservation( resumed, selected_services=selected_services, @@ -202,6 +220,17 @@ def handle_restart_prestop( runtime._release_requirement_ports(requirements) session.preserved_requirements = dict(preservation.preserved_requirements) session.preserved_services = dict(preservation.preserved_services) + requested_service_names = { + str(name).strip() for name in route.flags.get("services", []) if str(name).strip() + } + inherited_metadata = metadata_without_dashboard_stopped_services( + {**dict(resumed.metadata), **dict(session.base_metadata)}, + restored_service_names=requested_service_names, + ) + session.base_metadata = metadata_with_state_sources( + inherited_metadata, + resumed, + ) if use_prestop_spinner: prestop_spinner.succeed("Restart pre-stop complete") runtime._emit( @@ -241,6 +270,12 @@ def handle_restart_prestop( return None +def _failed_service_names(result: object) -> set[str]: + if not isinstance(result, (set, frozenset)): + return set() + return {str(name).strip() for name in result if str(name).strip()} + + def restart_prestop_preservation( state: Any, *, @@ -342,6 +377,8 @@ def restart_orphan_listener_scan( "frontend": set(range(int(frontend_port_base), int(frontend_port_base) + span)), } selected_by_cwd: dict[str, set[str]] = {} + owners_by_cwd_type: dict[tuple[str, str], tuple[str, ...]] = {} + port_lock_sessions_by_cwd_type: dict[tuple[str, str], tuple[str, ...]] = {} for service_name, service in getattr(state, "services", {}).items(): if service_name not in selected_services: continue @@ -350,6 +387,20 @@ def restart_orphan_listener_scan( if service_type not in ports_by_type or not cwd: continue selected_by_cwd.setdefault(cwd, set()).add(service_type) + project = str(service_project_name(service) or service_name.rsplit(" ", 1)[0]).strip() + if project: + owners_by_cwd_type[(cwd, service_type)] = ( + f"{project}:{service_type}", + f"{project}:services:{service_type}-launch", + f"{project}:services", + ) + raw_port_lock_session = getattr(service, "port_lock_session", None) + port_lock_session = raw_port_lock_session.strip() if isinstance(raw_port_lock_session, str) else "" + if port_lock_session: + key = (cwd, service_type) + port_lock_sessions_by_cwd_type[key] = tuple( + dict.fromkeys((*port_lock_sessions_by_cwd_type.get(key, ()), port_lock_session)) + ) for attr_name in ("actual_port", "requested_port"): port = getattr(service, attr_name, None) if isinstance(port, int) and port > 0: @@ -357,6 +408,8 @@ def restart_orphan_listener_scan( return RestartOrphanListenerScan( ports_by_type=ports_by_type, selected_by_cwd=selected_by_cwd, + owners_by_cwd_type=owners_by_cwd_type, + port_lock_sessions_by_cwd_type=port_lock_sessions_by_cwd_type, ) @@ -377,7 +430,14 @@ def restart_matching_orphan_listeners( if process_cwd(pid) != cwd: continue seen_pids.add(pid) - matches.append(RestartOrphanListenerMatch(pid=pid, port=port)) + matches.append( + RestartOrphanListenerMatch( + pid=pid, + port=port, + owner_candidates=scan.owners_by_cwd_type.get((cwd, service_type), ()), + port_lock_sessions=scan.port_lock_sessions_by_cwd_type.get((cwd, service_type), ()), + ) + ) return matches @@ -392,8 +452,8 @@ def terminate_restart_orphan_listeners( listener_pids_for_port: Callable[[int], Iterable[int]] | None, process_cwd: Callable[[int], str | None], terminate_pid: Callable[..., bool] | None, - release_port: Callable[[int], None], -) -> None: + port_planner: Any, +) -> set[int]: scan = restart_orphan_listener_scan( state, selected_services=selected_services, @@ -402,17 +462,29 @@ def terminate_restart_orphan_listeners( port_spacing=port_spacing, ) if not scan.selected_by_cwd: - return + return set() if not callable(listener_pids_for_port) or not callable(terminate_pid): - return + return set() matches = restart_matching_orphan_listeners( scan, listener_pids_for_port=listener_pids_for_port, process_cwd=process_cwd, ) + failed_pids: set[int] = set() for match in matches: if terminate_pid(match.pid, term_timeout=0.5 if aggressive else 2.0, kill_timeout=1.0): - release_port(match.port) + expected_sessions = match.port_lock_sessions or (None,) + for expected_session in expected_sessions: + if release_port_reservation( + port_planner, + match.port, + owner_candidates=match.owner_candidates, + expected_session=expected_session, + ): + break + else: + failed_pids.add(match.pid) + return failed_pids def terminate_restart_orphan_listeners_with_runtime( @@ -421,10 +493,10 @@ def terminate_restart_orphan_listeners_with_runtime( state: Any, selected_services: set[str], aggressive: bool, -) -> None: +) -> set[int]: process_runtime = resolve_process_runtime(runtime) port_allocator = resolve_port_allocator(runtime) - terminate_restart_orphan_listeners( + return terminate_restart_orphan_listeners( state=state, selected_services=selected_services, aggressive=aggressive, @@ -432,7 +504,7 @@ def terminate_restart_orphan_listeners_with_runtime( frontend_port_base=int(runtime.config.frontend_port_base), port_spacing=int(getattr(runtime.config, "port_spacing", 20) or 20), listener_pids_for_port=getattr(runtime, "_listener_pids_for_port", None), - process_cwd=process_cwd, + process_cwd=getattr(process_runtime, "process_cwd", process_cwd), terminate_pid=getattr(process_runtime, "terminate", None), - release_port=port_allocator.release, + port_planner=port_allocator, ) diff --git a/python/envctl_engine/startup/resume_orchestrator.py b/python/envctl_engine/startup/resume_orchestrator.py index c9773809c..45e512fa6 100644 --- a/python/envctl_engine/startup/resume_orchestrator.py +++ b/python/envctl_engine/startup/resume_orchestrator.py @@ -6,7 +6,9 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_startup_support import mark_run_reused +from envctl_engine.runtime.lifecycle_operation_lease import release_lifecycle_operation from envctl_engine.state.models import RunState +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.ui.spinner import spinner, spinner_enabled, use_spinner_policy from envctl_engine.ui.spinner_service import emit_spinner_policy, resolve_spinner_policy @@ -47,9 +49,17 @@ def emit_phase(phase: str, started_at: float, **extra: object) -> None: ) state_load_started = time.monotonic() - state = rt._try_load_existing_state( # type: ignore[attr-defined] + raw_state_projects = route.flags.get("_state_project_names") + state_projects = ( + [str(project).strip() for project in raw_state_projects if str(project).strip()] + if isinstance(raw_state_projects, list) + else list(route.projects) + ) + state = call_state_loader( + rt._try_load_existing_state, # type: ignore[attr-defined] mode=route.mode, strict_mode_match=rt._state_lookup_strict_mode_match(route), # type: ignore[attr-defined] + project_names=state_projects or None, ) emit_phase("state_load", state_load_started, found=state is not None) if state is None: @@ -156,6 +166,17 @@ def emit_phase(phase: str, started_at: float, **extra: object) -> None: ) enter_interactive = bool(rt._should_enter_resume_interactive(route)) # type: ignore[attr-defined] + if not rt._state_has_resumable_services(state): # type: ignore[attr-defined] + rt._emit( # type: ignore[attr-defined] + "resume.empty_state", + run_id=state.run_id, + reason="no_resumable_services", + ) + print("No active services to resume.") + return 1 + + release_lifecycle_operation(rt) + if not enter_interactive: current_session_id = getattr(rt, "_current_session_id", None) session_id = current_session_id() if callable(current_session_id) else None @@ -173,14 +194,6 @@ def emit_phase(phase: str, started_at: float, **extra: object) -> None: print("Warning: stale services detected during resume: " + ", ".join(missing_services)) if restore_errors: print("Warning: resume restore encountered errors: " + "; ".join(restore_errors)) - if not rt._state_has_resumable_services(state): # type: ignore[attr-defined] - rt._emit( # type: ignore[attr-defined] - "resume.empty_state", - run_id=state.run_id, - reason="no_resumable_services", - ) - print("No active services to resume.") - return 1 if enter_interactive: return rt._run_interactive_dashboard_loop(state) # type: ignore[attr-defined] return 0 diff --git a/python/envctl_engine/startup/run_reuse_dashboard_restore.py b/python/envctl_engine/startup/run_reuse_dashboard_restore.py index 05e23a34f..759efee4b 100644 --- a/python/envctl_engine/startup/run_reuse_dashboard_restore.py +++ b/python/envctl_engine/startup/run_reuse_dashboard_restore.py @@ -6,6 +6,7 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.startup.run_reuse_decision import mark_run_reused +from envctl_engine.startup.session import metadata_with_state_sources def dashboard_stopped_service_entries(state: object) -> list[dict[str, str]]: @@ -104,9 +105,12 @@ def _apply_restore(self, *, restore_entries: list[dict[str, str]], contexts_to_s stopped_service_names = sorted({entry["name"] for entry in restore_entries}) stopped_service_types = sorted({entry["type"] for entry in restore_entries}) target_project_names = sorted({entry["project"] for entry in restore_entries}, key=str.casefold) - self.session.base_metadata = metadata_without_dashboard_stopped_services( - mark_run_reused(self.candidate_state.metadata, reason="restore_stopped_services"), - restored_service_names=set(stopped_service_names), + self.session.base_metadata = metadata_with_state_sources( + metadata_without_dashboard_stopped_services( + mark_run_reused(self.candidate_state.metadata, reason="restore_stopped_services"), + restored_service_names=set(stopped_service_names), + ), + self.candidate_state, ) self.session.preserved_services = dict(self.candidate_state.services) self.session.preserved_requirements = dict(self.candidate_state.requirements) diff --git a/python/envctl_engine/startup/run_reuse_decision.py b/python/envctl_engine/startup/run_reuse_decision.py index c97bcfc65..b1470c2fd 100644 --- a/python/envctl_engine/startup/run_reuse_decision.py +++ b/python/envctl_engine/startup/run_reuse_decision.py @@ -5,6 +5,7 @@ from typing import Any, Mapping from envctl_engine.runtime.command_router import Route +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RunState from envctl_engine.startup.run_reuse_identity import ( _auto_resume_start_enabled, @@ -56,20 +57,37 @@ def evaluate(self) -> RunReuseDecision: selected_payload=selected_payload, startup_enabled=startup_enabled, ) - if not _auto_resume_start_enabled(self.route): + auto_resume_enabled = _auto_resume_start_enabled(self.route) + explicit_fresh_start = bool(self.route.flags.get("no_resume")) + if not auto_resume_enabled and not explicit_fresh_start: return self._fresh_without_state( reason="auto_resume_disabled", selected_payload=selected_payload, startup_enabled=startup_enabled, ) - candidate = self.runtime._try_load_existing_state(mode=self.runtime_mode, strict_mode_match=True) + candidate = call_state_loader( + self.runtime._try_load_existing_state, + mode=self.runtime_mode, + strict_mode_match=True, + project_names=[identity.name for identity in selected_identities], + ) if candidate is None: return self._fresh_without_state( reason="no_matching_state", selected_payload=selected_payload, startup_enabled=startup_enabled, ) + if explicit_fresh_start: + state_identities = project_identities_from_state(self.runtime, candidate) + return RunReuseDecision( + candidate_state=candidate, + decision_kind="fresh_run", + reason="explicit_no_resume", + selected_projects=selected_payload, + state_projects=identities_to_payload(state_identities), + startup_enabled=startup_enabled, + ) return self._evaluate_candidate( candidate=candidate, selected_identities=selected_identities, diff --git a/python/envctl_engine/startup/run_reuse_fresh_start.py b/python/envctl_engine/startup/run_reuse_fresh_start.py index 2cb75fc60..32ff7f334 100644 --- a/python/envctl_engine/startup/run_reuse_fresh_start.py +++ b/python/envctl_engine/startup/run_reuse_fresh_start.py @@ -5,43 +5,28 @@ from typing import Any from envctl_engine.runtime.command_router import Route -from envctl_engine.shared.services import service_project_name, service_slug_from_record -from envctl_engine.startup.startup_selection_support import _restart_service_types_for_project +from envctl_engine.shared.services import service_project_name +from envctl_engine.startup.session import metadata_with_state_sources def fresh_start_replacement_services( *, - route: Route, selected_contexts: list[object], candidate_state: object, - configured_service_types: set[str], - additional_services: tuple[object, ...], project_name_from_service: Callable[[str], str], ) -> set[str]: - target_projects = {str(context.name).strip().lower() for context in selected_contexts} + """Select every old service owned by a project that is starting fresh.""" + + target_projects = {str(context.name).strip().casefold() for context in selected_contexts} target_projects.discard("") if not target_projects: return set() - selected_by_project = { - str(context.name).strip().lower(): _restart_service_types_for_project( - route=route, - project_name=str(context.name), - default_service_types=configured_service_types, - additional_services=additional_services, - ) - for context in selected_contexts - if str(context.name).strip() + return { + service_name + for service_name, service in getattr(candidate_state, "services", {}).items() + if str(service_project_name(service) or project_name_from_service(service_name)).strip().casefold() + in target_projects } - selected: set[str] = set() - for service_name, service in getattr(candidate_state, "services", {}).items(): - project = service_project_name(service) or project_name_from_service(service_name) - project_key = str(project).strip().lower() - if project_key not in target_projects: - continue - service_type = service_slug_from_record(service) - if service_type and service_type in selected_by_project.get(project_key, set()): - selected.add(service_name) - return selected @dataclass(frozen=True, slots=True) @@ -50,25 +35,48 @@ class FreshStartServiceReplacer: session: Any candidate_state: Any reason: str - fresh_start_replacement_services: Callable[..., set[str]] announce_session_identifiers: Callable[[Any], None] report_progress: Callable[[Route, str], None] - terminate_restart_orphan_listeners: Callable[..., None] + terminate_restart_orphan_listeners: Callable[..., set[int]] def replace(self) -> None: - if self.reason != "startup_fingerprint_mismatch": - return route = self.session.effective_route - if route.flags.get("runtime_scope") == "dependencies": - return - selected_services = self.fresh_start_replacement_services(self.session, candidate_state=self.candidate_state) - if not selected_services: - return - self.announce_session_identifiers(self.session) - self.report_progress( - route, - f"Startup selection changed; replacing {len(selected_services)} existing service(s)...", + dependencies_only = route.flags.get("runtime_scope") == "dependencies" + selected_services = ( + set() + if dependencies_only + else fresh_start_replacement_services( + selected_contexts=list(self.session.selected_contexts), + candidate_state=self.candidate_state, + project_name_from_service=self.runtime._project_name_from_service, + ) + ) + target_projects = { + str(context.name).strip().casefold() + for context in self.session.selected_contexts + if str(context.name).strip() + } + preserved_services = ( + dict(self.candidate_state.services) + if dependencies_only + else { + name: service + for name, service in self.candidate_state.services.items() + if name not in selected_services + } ) + preserved_requirements, requirements_to_release = _split_requirements( + self.candidate_state, + target_projects=target_projects, + ) + if selected_services: + self.announce_session_identifiers(self.session) + progress_message = ( + f"Auto-resume disabled; replacing {len(selected_services)} existing service(s)..." + if self.reason == "explicit_no_resume" + else f"Startup selection changed; replacing {len(selected_services)} existing service(s)..." + ) + self.report_progress(route, progress_message) self.runtime._emit( "state.run_reuse.replace_existing_services", run_id=self.candidate_state.run_id, @@ -76,16 +84,41 @@ def replace(self) -> None: reason=self.reason, selected_services=sorted(selected_services), ) - self.runtime._terminate_services_from_state( + if selected_services: + termination_result = self.runtime._terminate_services_from_state( + self.candidate_state, + selected_services=selected_services, + aggressive=False, + verify_ownership=True, + ) + failed_services = _failed_service_names(termination_result) + if failed_services: + self.session.preserve_existing_state_on_failure = True + raise RuntimeError( + "Fresh start aborted because existing services could not be stopped: " + + ", ".join(sorted(failed_services)) + ) + failed_orphan_pids = self.terminate_restart_orphan_listeners( + state=self.candidate_state, + selected_services=selected_services, + aggressive=True, + ) + if failed_orphan_pids: + self.session.preserve_existing_state_on_failure = True + raise RuntimeError( + "Fresh start aborted because orphan listeners could not be stopped: " + + ", ".join(str(pid) for pid in sorted(failed_orphan_pids)) + ) + for requirements in requirements_to_release.values(): + self.runtime._release_requirement_ports(requirements) + self.session.preserved_services = preserved_services + self.session.preserved_requirements = preserved_requirements + self.session.base_metadata = metadata_with_state_sources( + { + **dict(self.candidate_state.metadata), + **dict(self.session.base_metadata), + }, self.candidate_state, - selected_services=selected_services, - aggressive=False, - verify_ownership=True, - ) - self.terminate_restart_orphan_listeners( - state=self.candidate_state, - selected_services=selected_services, - aggressive=True, ) @@ -95,17 +128,15 @@ def replace_existing_project_services_for_fresh_start( session: Any, candidate_state: Any, reason: str, - fresh_start_replacement_services: Callable[..., set[str]], announce_session_identifiers: Callable[[Any], None], report_progress: Callable[[Route, str], None], - terminate_restart_orphan_listeners: Callable[..., None], + terminate_restart_orphan_listeners: Callable[..., set[int]], ) -> None: FreshStartServiceReplacer( runtime=runtime, session=session, candidate_state=candidate_state, reason=reason, - fresh_start_replacement_services=fresh_start_replacement_services, announce_session_identifiers=announce_session_identifiers, report_progress=report_progress, terminate_restart_orphan_listeners=terminate_restart_orphan_listeners, @@ -118,26 +149,36 @@ def replace_existing_project_services_for_fresh_start_with_defaults( session: Any, candidate_state: Any, reason: str, - configured_service_types: set[str], - additional_services: tuple[object, ...], announce_session_identifiers: Callable[[Any], None], report_progress: Callable[[Route, str], None], - terminate_restart_orphan_listeners: Callable[..., None], + terminate_restart_orphan_listeners: Callable[..., set[int]], ) -> None: replace_existing_project_services_for_fresh_start( runtime=runtime, session=session, candidate_state=candidate_state, reason=reason, - fresh_start_replacement_services=lambda _, *, candidate_state: fresh_start_replacement_services( - route=session.effective_route, - selected_contexts=list(session.selected_contexts), - candidate_state=candidate_state, - configured_service_types=configured_service_types, - additional_services=additional_services, - project_name_from_service=runtime._project_name_from_service, - ), announce_session_identifiers=announce_session_identifiers, report_progress=report_progress, terminate_restart_orphan_listeners=terminate_restart_orphan_listeners, ) + + +def _split_requirements( + state: object, + *, + target_projects: set[str], +) -> tuple[dict[str, object], dict[str, object]]: + preserved: dict[str, object] = {} + replaced: dict[str, object] = {} + for name, requirements in getattr(state, "requirements", {}).items(): + project = str(getattr(requirements, "project", "") or name).strip().casefold() + destination = replaced if project in target_projects else preserved + destination[name] = requirements + return preserved, replaced + + +def _failed_service_names(result: object) -> set[str]: + if not isinstance(result, (set, frozenset)): + return set() + return {str(name).strip() for name in result if str(name).strip()} diff --git a/python/envctl_engine/startup/run_reuse_identity.py b/python/envctl_engine/startup/run_reuse_identity.py index bd8e66f43..c26f5a8db 100644 --- a/python/envctl_engine/startup/run_reuse_identity.py +++ b/python/envctl_engine/startup/run_reuse_identity.py @@ -27,19 +27,31 @@ def build_startup_identity_metadata( route: Route | None = None, ) -> dict[str, object]: metadata = dict(base_metadata or {}) - project_roots = { - str(getattr(context, "name", "")).strip(): root - for context, root in ( - (context, normalize_project_root(getattr(context, "root", None))) for context in project_contexts - ) - if str(getattr(context, "name", "")).strip() and root - } + roots_by_key: dict[str, tuple[str, str]] = {} + existing_roots = metadata.get("project_roots") + if isinstance(existing_roots, Mapping): + for raw_name, raw_root in existing_roots.items(): + name = str(raw_name).strip() + root = normalize_project_root(raw_root) + if name and root: + roots_by_key[name.casefold()] = (name, root) + for context in project_contexts: + name = str(getattr(context, "name", "")).strip() + root = normalize_project_root(getattr(context, "root", None)) + if name and root: + roots_by_key[name.casefold()] = (name, root) + project_roots = {name: root for name, root in (roots_by_key[key] for key in sorted(roots_by_key))} identity_payload = _startup_identity_payload( runtime, runtime_mode=runtime_mode, project_contexts=project_contexts, route=route, ) + identity_payload["projects"] = [{"name": name, "root": root} for name, root in project_roots.items()] + identity_payload.pop("fingerprint", None) + identity_payload["fingerprint"] = hashlib.sha256( + json.dumps(identity_payload, sort_keys=True).encode("utf-8") + ).hexdigest() metadata["project_roots"] = project_roots metadata["startup_identity"] = identity_payload return metadata diff --git a/python/envctl_engine/startup/run_reuse_resolution.py b/python/envctl_engine/startup/run_reuse_resolution.py index cb9beb3c2..7eb95778a 100644 --- a/python/envctl_engine/startup/run_reuse_resolution.py +++ b/python/envctl_engine/startup/run_reuse_resolution.py @@ -21,7 +21,7 @@ ) from envctl_engine.startup.run_reuse_fresh_start import replace_existing_project_services_for_fresh_start_with_defaults from envctl_engine.startup.run_reuse_resume import StartupRunReuseResumeHandler -from envctl_engine.startup.session import StartupSession +from envctl_engine.startup.session import StartupSession, metadata_with_state_sources from envctl_engine.startup.session_lifecycle import announce_session_identifiers, emit_startup_phase from envctl_engine.startup.service_bootstrap_domain import configured_service_types_for_mode from envctl_engine.startup.startup_progress import report_progress @@ -137,7 +137,10 @@ def _apply_reuse_expand(self, *, decision: RunReuseDecision, candidate_state: An self.session.run_id = None self.session.preserved_services = dict(candidate_state.services) self.session.preserved_requirements = dict(candidate_state.requirements) - self.session.base_metadata = mark_run_reused(candidate_state.metadata, reason="reuse_expand") + self.session.base_metadata = metadata_with_state_sources( + mark_run_reused(candidate_state.metadata, reason="reuse_expand"), + candidate_state, + ) state_project_names = { str(project.get("name", "")).strip().casefold() for project in decision.state_projects @@ -321,7 +324,7 @@ def resolve_startup_run_reuse_with_runtime( runtime: Any, session: StartupSession, *, - terminate_restart_orphan_listeners: Callable[..., None], + terminate_restart_orphan_listeners: Callable[..., set[int]], validate_attach_target_fn: Callable[..., Any], attach_plan_agent_terminal: Callable[..., Any], progress_lock: Any, @@ -362,13 +365,11 @@ def report_progress_fn(route: Route, message: str) -> None: session=session, validate_plan_agent_handoff=validate_plan_agent_handoff, attach_plan_agent_terminal=attach_plan_agent_terminal, - print_headless_plan_session_summary=lambda session, *, attach_target: ( - print_headless_plan_session_summary( - session, - validate_plan_agent_handoff=validate_plan_agent_handoff, - print_fn=print_fn, - attach_target=attach_target, - ) + print_headless_plan_session_summary=lambda session, *, attach_target: print_headless_plan_session_summary( + session, + validate_plan_agent_handoff=validate_plan_agent_handoff, + print_fn=print_fn, + attach_target=attach_target, ), ), print_headless_plan_session_summary=lambda session: print_headless_plan_session_summary( @@ -391,8 +392,6 @@ def report_progress_fn(route: Route, message: str) -> None: session=session, candidate_state=candidate_state, reason=reason, - configured_service_types=set(configured_service_types_for_mode(runtime.config, session.runtime_mode)), - additional_services=tuple(getattr(runtime.config, "additional_services", ()) or ()), announce_session_identifiers=lambda session: announce_session_identifiers( runtime, session, diff --git a/python/envctl_engine/startup/run_reuse_resume.py b/python/envctl_engine/startup/run_reuse_resume.py index bed5d1840..248ee463b 100644 --- a/python/envctl_engine/startup/run_reuse_resume.py +++ b/python/envctl_engine/startup/run_reuse_resume.py @@ -6,6 +6,10 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_startup_support import mark_run_reused +from envctl_engine.runtime.lifecycle_operation_lease import ( + lifecycle_operation_active, + release_lifecycle_operation, +) from envctl_engine.runtime.runtime_context import resolve_state_repository from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.startup.finalization import build_planning_dashboard_state @@ -40,21 +44,25 @@ def resume_matching_run(self) -> int | None: self._emit_resume_match_phase() self._emit_resume_match_events() + lifecycle_lease_was_active = lifecycle_operation_active(self.runtime) resume_code = self.runtime._resume(self._resume_route()) if int(resume_code) == 0: return self._resume_success_result() + lifecycle_lease_was_released = lifecycle_lease_was_active and not lifecycle_operation_active(self.runtime) self._handle_resume_failure( resume_code=int(resume_code), previous_run_id=previous_run_id, previous_identifiers_announced=previous_identifiers_announced, + allow_fresh_fallback=not lifecycle_lease_was_released, ) - return None + return int(resume_code) if lifecycle_lease_was_released else None def resume_dashboard_run(self) -> int | None: self.session.run_id = self.candidate_state.run_id self.announce_session_identifiers(self.session) self._persist_dashboard_resume_metadata() self._emit_dashboard_resume_events() + release_lifecycle_operation(self.runtime) if self.headless_plan_output_only(self.session): self.print_headless_plan_session_summary(self.session) @@ -119,6 +127,7 @@ def _resume_route(self) -> Route: **self.route.flags, "_resume_source_command": self.route.command, "_run_reuse_reason": self.decision.decision_kind, + "_state_project_names": [str(name) for name in self._selected_project_names], } if self._attach_plan_agent_after_resume(): resume_flags["batch"] = True @@ -150,6 +159,7 @@ def _handle_resume_failure( resume_code: int, previous_run_id: str | None, previous_identifiers_announced: bool, + allow_fresh_fallback: bool = True, ) -> None: self.session.run_id = previous_run_id self.session.identifiers_announced = previous_identifiers_announced @@ -170,11 +180,12 @@ def _handle_resume_failure( mode=self.runtime_mode, command=self.route.command, ) - self.replace_existing_project_services_for_fresh_start( - self.session, - candidate_state=self.candidate_state, - reason=self.decision.reason, - ) + if allow_fresh_fallback: + self.replace_existing_project_services_for_fresh_start( + self.session, + candidate_state=self.candidate_state, + reason=self.decision.reason, + ) def _persist_dashboard_resume_metadata(self) -> None: self.candidate_state.metadata = build_planning_dashboard_state( diff --git a/python/envctl_engine/startup/selected_context_startup.py b/python/envctl_engine/startup/selected_context_startup.py index 7be9b3132..b96fcf004 100644 --- a/python/envctl_engine/startup/selected_context_startup.py +++ b/python/envctl_engine/startup/selected_context_startup.py @@ -7,7 +7,11 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_env import effective_dependency_scope from envctl_engine.startup.protocols import ProjectContextLike, StartupRuntime -from envctl_engine.startup.session import ProjectStartupResult, StartupSession +from envctl_engine.startup.session import ( + ProjectStartupResult, + StartupSession, + track_startup_failure, +) from envctl_engine.startup.startup_progress import suppress_progress_output as default_suppress_progress_output @@ -138,8 +142,11 @@ def start_selected_contexts( project_spinner_group=project_spinner_group, use_project_spinner_group=use_project_spinner_group, ) - except RuntimeError: - _fail_single_spinner(runtime=runtime, active_spinner=active_spinner, enabled=use_single_spinner) + except Exception: + try: + _fail_single_spinner(runtime=runtime, active_spinner=active_spinner, enabled=use_single_spinner) + except Exception: # noqa: BLE001 + pass raise _succeed_single_spinner( runtime=runtime, @@ -228,6 +235,7 @@ def _start_contexts_parallel( ) -> None: completed: dict[str, ProjectStartupResult] = {} failures: list[str] = [] + ui_failures: list[Exception] = [] with concurrent.futures.ThreadPoolExecutor(max_workers=parallel_workers) as executor: future_map = { executor.submit( @@ -243,7 +251,50 @@ def _start_contexts_parallel( context = future_map[future] try: result = cast(ProjectStartupResult, future.result()) - completed[context.name] = result + except RuntimeError as exc: + track_startup_failure(session, exc) + try: + degrade = should_degrade_to_plan_agent_handoff(session, str(exc)) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + degrade = False + if degrade: + try: + record_plan_agent_handoff_local_startup_failure( + session, + project_name=context.name, + error=str(exc), + ) + if use_project_spinner_group: + project_spinner_group.mark_success( + context.name, + "AI session running; local startup failed", + ) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + continue + failures.append(str(exc)) + try: + runtime._emit("startup.project.failed", project=context.name, error=str(exc)) + if use_project_spinner_group: + project_spinner_group.mark_failure(context.name, str(exc)) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + continue + except Exception as exc: # noqa: BLE001 + track_startup_failure(session, exc) + failures.append(str(exc)) + try: + runtime._emit("startup.project.failed", project=context.name, error=str(exc)) + if use_project_spinner_group: + project_spinner_group.mark_failure(context.name, str(exc)) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + continue + + completed[context.name] = result + record_project_startup(session, context, result) + try: _record_parallel_success( runtime=runtime, session=session, @@ -257,22 +308,12 @@ def _start_contexts_parallel( use_project_spinner_group=use_project_spinner_group, completed_count=len(completed), ) - except RuntimeError as exc: - if should_degrade_to_plan_agent_handoff(session, str(exc)): - record_plan_agent_handoff_local_startup_failure(session, project_name=context.name, error=str(exc)) - if use_project_spinner_group: - project_spinner_group.mark_success(context.name, "AI session running; local startup failed") - continue - failures.append(str(exc)) - runtime._emit("startup.project.failed", project=context.name, error=str(exc)) - if use_project_spinner_group: - project_spinner_group.mark_failure(context.name, str(exc)) - for context in session.contexts_to_start: - result = completed.get(context.name) - if result is not None: - record_project_startup(session, context, result) + except Exception as exc: # noqa: BLE001 + ui_failures.append(exc) if failures: raise RuntimeError("; ".join(failures)) + if ui_failures: + raise ui_failures[0] def _record_parallel_success( @@ -335,6 +376,7 @@ def _start_contexts_sequential( ), ) except RuntimeError as exc: + track_startup_failure(session, exc) if should_degrade_to_plan_agent_handoff(session, str(exc)): record_plan_agent_handoff_local_startup_failure(session, project_name=context.name, error=str(exc)) if use_project_spinner_group: diff --git a/python/envctl_engine/startup/service_attach_execution.py b/python/envctl_engine/startup/service_attach_execution.py index 1421c745b..e15a202c8 100644 --- a/python/envctl_engine/startup/service_attach_execution.py +++ b/python/envctl_engine/startup/service_attach_execution.py @@ -1,15 +1,15 @@ from __future__ import annotations -from collections.abc import Callable, Mapping -from dataclasses import dataclass -import os +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field from pathlib import Path import time from typing import Any, cast -from envctl_engine.runtime.service_manager import ServiceStartDescriptor +from envctl_engine.runtime.service_manager import ServiceCleanupError, ServiceStartDescriptor from envctl_engine.startup.service_execution_policy import ordered_service_layers from envctl_engine.startup.service_execution_records import PreparedServiceLaunch +from envctl_engine.startup.session import unconfirmed_service_names from envctl_engine.state.models import PortPlan, ServiceRecord @@ -34,6 +34,8 @@ class ServiceAttachRunner: additional_services: tuple[Any, ...] backend_listener_expected: bool rebound_delta: int + _active_service_ports: dict[str, int] = field(default_factory=dict, init=False, repr=False) + _rebound_launch_ports: dict[str, int] = field(default_factory=dict, init=False, repr=False) def start( self, @@ -42,24 +44,94 @@ def start( on_service_retry: Callable[[str, int, int, int, str | None], None], ) -> dict[str, ServiceRecord]: descriptors = self._service_descriptors() - generic_attach = getattr(self.runtime.services, "start_services_with_attach", None) - if callable(generic_attach): - return self._start_with_generic_attach( - descriptors=descriptors, - attach_parallel=attach_parallel, - on_service_retry=on_service_retry, - generic_attach=cast(Callable[..., dict[str, ServiceRecord]], generic_attach), - ) - if self.selected_service_types <= {"backend", "frontend"}: - return self._start_core_services_with_legacy_attach( - attach_parallel=attach_parallel, - on_service_retry=on_service_retry, - ) - raise RuntimeError("Service manager does not support additional app services") + self._active_service_ports = { + descriptor.service_type: descriptor.requested_port + for descriptor in descriptors + if descriptor.requested_port > 0 + } + + def release_failed_port_then_report( + service_type: str, + failed_port: int, + retry_port: int, + attempt: int, + error: str | None, + ) -> None: + self._release_failed_service_port(service_type, failed_port) + self._release_rebound_launch_port(service_type) + if retry_port > 0: + self._active_service_ports[service_type] = retry_port + on_service_retry(service_type, failed_port, retry_port, attempt, error) + + try: + if hasattr(self.runtime.services, "process_runner"): + self.runtime.services.process_runner = self.process_runtime + generic_attach = getattr(self.runtime.services, "start_services_with_attach", None) + if callable(generic_attach): + records = self._start_with_generic_attach( + descriptors=descriptors, + attach_parallel=attach_parallel, + on_service_retry=release_failed_port_then_report, + generic_attach=cast(Callable[..., dict[str, ServiceRecord]], generic_attach), + ) + elif self.selected_service_types <= {"backend", "frontend"}: + records = self._start_core_services_with_legacy_attach( + attach_parallel=attach_parallel, + on_service_retry=release_failed_port_then_report, + ) + else: + raise RuntimeError("Service manager does not support additional app services") + self._record_port_lock_session(records.values()) + return records + except Exception as exc: + self._record_port_lock_session(dict(getattr(exc, "unterminated_services", {}) or {}).values()) + self._release_failed_start_reservations(exc) + raise + + def _record_port_lock_session(self, services: Iterable[object]) -> None: + raw_session_id = getattr(self.port_allocator, "session_id", None) + session_id = raw_session_id.strip() if isinstance(raw_session_id, str) else "" + if not session_id: + return + for service in services: + if isinstance(service, ServiceRecord): + service.port_lock_session = session_id def reserve_next(self, port: int) -> int: return self.port_allocator.reserve_next(port, owner=f"{self.project_name}:services") + def _release_failed_service_port(self, service_type: str, port: int) -> None: + release = getattr(self.port_allocator, "release", None) + if not callable(release): + return + for owner in ( + f"{self.project_name}:{service_type}", + f"{self.project_name}:services", + f"{self.project_name}:services:{service_type}-launch", + ): + release(port, owner=owner) + + def _release_rebound_launch_port(self, service_type: str) -> None: + port = self._rebound_launch_ports.pop(service_type, None) + if port is None: + return + release = getattr(self.port_allocator, "release", None) + if callable(release): + release(port, owner=f"{self.project_name}:services:{service_type}-launch") + + def _release_failed_start_reservations(self, error: BaseException) -> None: + preserved_service_types = { + str(getattr(service, "type", "") or "").strip().lower() + for service in dict(getattr(error, "unterminated_services", {}) or {}).values() + if _service_has_recorded_pid(service) + } + for service_type, port in tuple(self._active_service_ports.items()): + if service_type in preserved_service_types: + continue + self._release_failed_service_port(service_type, port) + self._active_service_ports.pop(service_type, None) + self._release_rebound_launch_port(service_type) + def _start_with_generic_attach( self, *, @@ -70,18 +142,42 @@ def _start_with_generic_attach( ) -> dict[str, ServiceRecord]: descriptor_by_type = {descriptor.service_type: descriptor for descriptor in descriptors} records: dict[str, ServiceRecord] = {} - for layer in ordered_service_layers( - tuple(descriptor.service_type for descriptor in descriptors), - self.additional_services, - ): - layer_records = generic_attach( - project=self.project_name, - descriptors=tuple(descriptor_by_type[name] for name in layer), - reserve_next=self.reserve_next, - on_retry=on_service_retry, - parallel_start=attach_parallel and len(layer) > 1, - ) - records.update(layer_records) + try: + for layer in ordered_service_layers( + tuple(descriptor.service_type for descriptor in descriptors), + self.additional_services, + ): + layer_records = generic_attach( + project=self.project_name, + descriptors=tuple(descriptor_by_type[name] for name in layer), + reserve_next=self.reserve_next, + on_retry=on_service_retry, + parallel_start=attach_parallel and len(layer) > 1, + ) + records.update(layer_records) + except Exception as exc: + unterminated = dict(getattr(exc, "unterminated_services", {}) or {}) + terminator = getattr(self.runtime, "_terminate_started_services", None) + if records: + if callable(terminator): + try: + failed_names = unconfirmed_service_names( + terminator(records), + records, + ) + except Exception as cleanup_exc: # noqa: BLE001 + failed_names = set(records) + exc = RuntimeError(f"{exc}; cleanup error: {cleanup_exc}") + else: + failed_names = set(records) + unterminated.update((name, service) for name, service in records.items() if name in failed_names) + if unterminated: + names = ", ".join(sorted(unterminated)) + raise ServiceCleanupError( + f"{exc}; cleanup could not confirm exit for: {names}", + unterminated, + ) from exc + raise return records def _start_core_services_with_legacy_attach( @@ -140,9 +236,7 @@ def _service_descriptors(self) -> list[ServiceStartDescriptor]: service_type=service.name, cwd=str(launch.cwd), requested_port=launch.requested_port, - start=lambda port, service_name=service.name: self.start_additional_service( - service_name, port - ), + start=lambda port, service_name=service.name: self.start_additional_service(service_name, port), detect_actual=lambda pid, requested, service_name=service.name: self.detect_additional_actual( service_name, pid, requested ), @@ -197,7 +291,6 @@ def _launch_prepared_process( launch_port: int, event_port: int, env_extra: dict[str, str], - pid_fallback: int, ) -> tuple[bool, str | None, int | None]: launch_started = time.monotonic() process = self._start_process( @@ -206,15 +299,19 @@ def _launch_prepared_process( env=self.command_env_builder(port=launch_port, extra=env_extra), log_path=launch.log_path, ) - self._emit_attach_phase(service_name, "process_launch", launch_started) - self.runtime._emit( - "service.start", - project=self.project_name, - service=service_name, - port=event_port, - retry=False, - ) - return True, None, getattr(process, "pid", pid_fallback) + pid = getattr(process, "pid", None) + try: + self._emit_attach_phase(service_name, "process_launch", launch_started) + self.runtime._emit( + "service.start", + project=self.project_name, + service=service_name, + port=event_port, + retry=False, + ) + except Exception: # noqa: BLE001 + pass + return True, None, pid if isinstance(pid, int) and pid > 0 else None def start_backend(self, port: int) -> tuple[bool, str | None, int | None]: conflict_result = self._conflicting_start_result("backend", port) @@ -234,7 +331,6 @@ def start_backend(self, port: int) -> tuple[bool, str | None, int | None]: launch_port=port, event_port=port, env_extra=self.backend_env_extra, - pid_fallback=os.getpid(), ) def start_frontend(self, port: int) -> tuple[bool, str | None, int | None]: @@ -247,22 +343,29 @@ def start_frontend(self, port: int) -> tuple[bool, str | None, int | None]: launch_port, owner=f"{self.project_name}:services:frontend-launch", ) - command_resolve_started = time.monotonic() - command, _resolved_source = self.runtime._service_start_command_resolved( - service_name="frontend", - project_root=self.project_root, - port=launch_port, - ) - self._emit_attach_phase("frontend", "command_resolution", command_resolve_started) - return self._launch_prepared_process( - service_name="frontend", - command=command, - launch=self.prepared_launches["frontend"], - launch_port=launch_port, - event_port=port, - env_extra=self.frontend_env_extra, - pid_fallback=os.getpid() + 1, - ) + self._rebound_launch_ports["frontend"] = launch_port + try: + command_resolve_started = time.monotonic() + command, _resolved_source = self.runtime._service_start_command_resolved( + service_name="frontend", + project_root=self.project_root, + port=launch_port, + ) + self._emit_attach_phase("frontend", "command_resolution", command_resolve_started) + result = self._launch_prepared_process( + service_name="frontend", + command=command, + launch=self.prepared_launches["frontend"], + launch_port=launch_port, + event_port=port, + env_extra=self.frontend_env_extra, + ) + except Exception: + self._release_rebound_launch_port("frontend") + raise + if not isinstance(result[2], int) or isinstance(result[2], bool) or result[2] <= 0: + self._release_rebound_launch_port("frontend") + return result def start_additional_service(self, service_name: str, port: int) -> tuple[bool, str | None, int | None]: conflict_result = self._conflicting_start_result(service_name, port) @@ -291,7 +394,6 @@ def start_additional_service(self, service_name: str, port: int) -> tuple[bool, launch_port=port, event_port=port, env_extra=launch.env, - pid_fallback=os.getpid(), ) def detect_backend_actual(self, pid: int | None, requested: int) -> int | None: @@ -466,3 +568,10 @@ def _emit_attach_phase(self, service_name: str, phase: str, started: float) -> N phase=phase, duration_ms=round((time.monotonic() - started) * 1000.0, 2), ) + + +def _service_has_recorded_pid(service: object) -> bool: + listener_pids = getattr(service, "listener_pids", None) + listeners = listener_pids if isinstance(listener_pids, (list, tuple, set, frozenset)) else () + candidates = [getattr(service, "pid", None), *listeners] + return any(isinstance(pid, int) and not isinstance(pid, bool) and pid > 0 for pid in candidates) diff --git a/python/envctl_engine/startup/session.py b/python/envctl_engine/startup/session.py index 9cd91a696..84d0367cf 100644 --- a/python/envctl_engine/startup/session.py +++ b/python/envctl_engine/startup/session.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Collection, Mapping import time from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, cast @@ -54,11 +55,13 @@ class StartupSession: restart_state: RunState | None = None requirements_by_project: dict[str, RequirementsResult] = field(default_factory=dict) services_by_project: dict[str, dict[str, ServiceRecord]] = field(default_factory=dict) + unterminated_services: dict[str, ServiceRecord] = field(default_factory=dict) started_context_names: list[str] = field(default_factory=list) dashboard_after_start: bool = False disabled_startup_mode: bool = False used_project_spinner_group: bool = False strict_truth_failed: bool = False + preserve_existing_state_on_failure: bool = False failure_message: str | None = None errors: list[str] = field(default_factory=list) warnings: list[str] = field(default_factory=list) @@ -82,6 +85,7 @@ def merged_services(self) -> dict[str, ServiceRecord]: services = dict(self.preserved_services) for project_services in self.services_by_project.values(): services.update(project_services) + services.update(self.unterminated_services) return services @property @@ -111,3 +115,57 @@ def plan_agent_launch_degraded(self) -> bool: if launch_result is None: return False return str(getattr(launch_result, "status", "")).strip().lower() == "partial" + + +def unconfirmed_service_names( + termination_result: object, + services: Mapping[str, object], +) -> set[str]: + """Return services whose exit was not positively confirmed. + + The termination contract returns a collection of names that remain alive; + an explicit empty collection therefore confirms complete cleanup. Legacy + adapters and failed mocks may return ``None`` or another scalar. Treating + those values as success loses process and port ownership, so they retain + every service for a later verified cleanup pass. + """ + + if not isinstance(termination_result, Collection) or isinstance( + termination_result, + (str, bytes, Mapping), + ): + return set(services) + return {name for raw_name in termination_result if (name := str(raw_name).strip()) and name in services} + + +def metadata_with_state_sources(metadata: dict[str, Any], state: RunState) -> dict[str, Any]: + """Return metadata that records every state replaced by a new startup run.""" + + updated = dict(metadata) + source_run_ids = {state.run_id} + for source in ( + updated.get("state_source_run_ids"), + state.metadata.get("state_source_run_ids"), + ): + if not isinstance(source, list): + continue + source_run_ids.update(str(run_id).strip() for run_id in source if str(run_id).strip()) + updated["state_source_run_ids"] = sorted(source_run_ids) + return updated + + +def track_startup_failure(session: StartupSession, failure: BaseException) -> None: + project = str(getattr(failure, "project", "") or "").strip() + requirements = getattr(failure, "requirements", None) + if project and isinstance(requirements, RequirementsResult): + session.requirements_by_project[project] = requirements + services = getattr(failure, "unterminated_services", None) + if not isinstance(services, dict): + return + for name, service in services.items(): + if isinstance(name, str) and isinstance(service, ServiceRecord): + session.unterminated_services[name] = service + + +def track_unterminated_services(session: StartupSession, failure: BaseException) -> None: + track_startup_failure(session, failure) diff --git a/python/envctl_engine/startup/startup_execution_support.py b/python/envctl_engine/startup/startup_execution_support.py index 631fb4a7e..51989c568 100644 --- a/python/envctl_engine/startup/startup_execution_support.py +++ b/python/envctl_engine/startup/startup_execution_support.py @@ -29,8 +29,8 @@ startup_breakdown_enabled, ) from envctl_engine.startup.service_execution import start_project_services as start_project_services_impl -from envctl_engine.startup.session import ProjectStartupResult -from envctl_engine.state.models import PortPlan, RequirementsResult +from envctl_engine.startup.session import ProjectStartupResult, unconfirmed_service_names +from envctl_engine.state.models import PortPlan, RequirementsResult, ServiceRecord @dataclass(slots=True) @@ -40,6 +40,23 @@ class _SharedProjectContext: ports: Mapping[str, PortPlan] +class ProjectStartupFailure(RuntimeError): + """A project failure that preserves already-started runtime resources.""" + + def __init__( + self, + message: str, + *, + project: str, + requirements: RequirementsResult, + unterminated_services: dict[str, ServiceRecord] | None = None, + ) -> None: + super().__init__(message) + self.project = project + self.requirements = requirements + self.unterminated_services = dict(unterminated_services or {}) + + def start_project_context( orchestrator: StartupOrchestratorLike, *, @@ -55,50 +72,74 @@ def start_project_context( report_progress_fn(route, f"Starting project {context.name}...", project=context.name) _reserve_project_ports(rt, context, route=route) requirements = _requirements_for_project_context(orchestrator, context=context, mode=mode, route=route) - if not rt._requirements_ready(requirements): - raise RuntimeError(_requirements_failure_message(context.name, requirements)) - _sync_supabase_auth_users_before_services( - orchestrator, - context=context, - mode=mode, - route=route, - requirements=requirements, - ) - requirements_progress_message = _requirements_ready_progress_message( - orchestrator, - context=context, - mode=mode, - route=route, - requirements=requirements, - ) - if requirements_progress_message is not None: - report_progress_fn(route, requirements_progress_message, project=context.name) - project_services = orchestrator.start_project_services( - context, - requirements=requirements, - run_id=run_id, - route=route, - ) + project_services: dict[str, ServiceRecord] = {} try: + if not rt._requirements_ready(requirements): + raise RuntimeError(_requirements_failure_message(context.name, requirements)) + _sync_supabase_auth_users_before_services( + orchestrator, + context=context, + mode=mode, + route=route, + requirements=requirements, + ) + requirements_progress_message = _requirements_ready_progress_message( + orchestrator, + context=context, + mode=mode, + route=route, + requirements=requirements, + ) + if requirements_progress_message is not None: + report_progress_fn(route, requirements_progress_message, project=context.name) + project_services = orchestrator.start_project_services( + context, + requirements=requirements, + run_id=run_id, + route=route, + ) rt._assert_project_services_post_start_truth(context=context, services=project_services) - except RuntimeError: - rt._terminate_started_services(project_services) - raise - report_progress_fn( - route, - f"Services ready for {context.name}: " - + " ".join( - f"{service_type}={_service_ready_label(project_services.get(f'{context.name} {service_type.title()}'))}" - for service_type in ("backend", "frontend") - if f"{context.name} {service_type.title()}" in project_services - ), - project=context.name, - ) - return ProjectStartupResult( - requirements=requirements, - services=project_services, - warnings=list(rt._consume_project_startup_warnings(context.name)), - ) + report_progress_fn( + route, + f"Services ready for {context.name}: " + + " ".join( + f"{service_type}={_service_ready_label(project_services.get(f'{context.name} {service_type.title()}'))}" + for service_type in ("backend", "frontend") + if f"{context.name} {service_type.title()}" in project_services + ), + project=context.name, + ) + return ProjectStartupResult( + requirements=requirements, + services=project_services, + warnings=list(rt._consume_project_startup_warnings(context.name)), + ) + except Exception as exc: + unterminated = dict(getattr(exc, "unterminated_services", {}) or {}) + cleanup_error: Exception | None = None + if project_services: + try: + failed_services = unconfirmed_service_names( + rt._terminate_started_services(project_services), + project_services, + ) + except Exception as cleanup_exc: # noqa: BLE001 + cleanup_error = cleanup_exc + failed_services = set(project_services) + unterminated.update( + (name, service) for name, service in project_services.items() if name in failed_services + ) + message = str(exc) + if unterminated and "cleanup could not confirm exit for:" not in message: + message = f"{message}; cleanup could not confirm exit for: {', '.join(sorted(unterminated))}" + if cleanup_error is not None: + message = f"{message}; cleanup error: {cleanup_error}" + raise ProjectStartupFailure( + message, + project=context.name, + requirements=requirements, + unterminated_services=unterminated, + ) from exc def _reserve_project_ports(rt: Any, context: ProjectContextLike, *, route: Route | None) -> None: @@ -335,10 +376,13 @@ def _annotate_shared_main_requirements( ) supabase = cloned.component("supabase") if bool(supabase.get("enabled", False)): - supabase["container_name"] = build_supabase_project_name( - project_root=project_root, - project_name="Main", - ) + "-supabase-db-1" + supabase["container_name"] = ( + build_supabase_project_name( + project_root=project_root, + project_name="Main", + ) + + "-supabase-db-1" + ) return _reconcile_shared_main_requirements(orchestrator, cloned) diff --git a/python/envctl_engine/startup/startup_selection_support.py b/python/envctl_engine/startup/startup_selection_support.py index 2235e19a6..d08ace792 100644 --- a/python/envctl_engine/startup/startup_selection_support.py +++ b/python/envctl_engine/startup/startup_selection_support.py @@ -97,13 +97,19 @@ def trees_start_selection_required(*, route: Route, runtime_mode: str) -> bool: def tree_preselected_projects_from_state( *, runtime: StartupRuntime, project_contexts: list[ProjectContextLike] ) -> list[str]: - state = runtime._try_load_existing_state(mode="trees", strict_mode_match=True) + repository = getattr(runtime, "state_repository", None) + load_all = getattr(repository, "load_all", None) + states = [state for state in load_all(mode="trees") if isinstance(state, RunState)] if callable(load_all) else [] + if not states: + state = runtime._try_load_existing_state(mode="trees", strict_mode_match=True) + states = [state] if state is not None else [] available = {str(context.name).strip().lower(): str(context.name).strip() for context in project_contexts} - if state is None: + if not states: return _tree_preselected_projects_from_plans(runtime=runtime, project_contexts=project_contexts) preselected: list[str] = [] seen: set[str] = set() - for name in sorted(state_project_names(runtime=runtime, state=state)): + active_names = {name for state in states for name in state_project_names(runtime=runtime, state=state)} + for name in sorted(active_names): normalized = str(name).strip().lower() resolved = available.get(normalized) if not resolved: @@ -138,9 +144,7 @@ def _tree_preselected_projects_from_plans( ] existing_counts = planning_existing_counts(projects=projects, planning_files=planning_files) plan_counts = OrderedDict( - (plan_file, count) - for plan_file in planning_files - if (count := int(existing_counts.get(plan_file, 0))) > 0 + (plan_file, count) for plan_file in planning_files if (count := int(existing_counts.get(plan_file, 0))) > 0 ) if not plan_counts: return [] @@ -354,7 +358,6 @@ def _project_name_from_service_name(name: str) -> str | None: return None - def _service_filter_types_for_project( *, route: Route, project_name: str, configured_service_types: set[str] ) -> set[str]: @@ -470,7 +473,10 @@ def _restart_service_types_for_project( ) return _apply_startup_service_launch_flags(configured_service_types, route=route) + def _apply_startup_service_launch_flags(service_types: set[str], *, route: Route) -> set[str]: + if route.flags.get("launch_backend") is False and route.flags.get("launch_frontend") is False: + return set() selected = set(service_types) if route.flags.get("launch_backend") is False: selected.discard("backend") diff --git a/python/envctl_engine/state/__init__.py b/python/envctl_engine/state/__init__.py index 37b34ba27..5a70281c1 100644 --- a/python/envctl_engine/state/__init__.py +++ b/python/envctl_engine/state/__init__.py @@ -30,6 +30,11 @@ def load_state(path: str, *, allowed_root: str | None = None) -> RunState: pid=svc.get("pid"), requested_port=svc.get("requested_port"), actual_port=svc.get("actual_port"), + port_lock_session=( + svc["port_lock_session"].strip() + if isinstance(svc.get("port_lock_session"), str) and svc["port_lock_session"].strip() + else None + ), log_path=svc.get("log_path"), status=svc.get("status", "unknown"), synthetic=bool(svc.get("synthetic", False)), @@ -169,6 +174,7 @@ def state_to_dict(state: RunState) -> dict[str, object]: "pid": svc.pid, "requested_port": svc.requested_port, "actual_port": svc.actual_port, + "port_lock_session": svc.port_lock_session, "log_path": svc.log_path, "status": svc.status, "synthetic": svc.synthetic, diff --git a/python/envctl_engine/state/fingerprints.py b/python/envctl_engine/state/fingerprints.py new file mode 100644 index 000000000..30087f574 --- /dev/null +++ b/python/envctl_engine/state/fingerprints.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +from envctl_engine.state import state_to_dict +from envctl_engine.state.models import RunState + + +def text_fingerprint(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def file_fingerprint(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def state_fingerprint(state: RunState) -> str: + return text_fingerprint(json.dumps(state_to_dict(state), sort_keys=True)) diff --git a/python/envctl_engine/state/lookup.py b/python/envctl_engine/state/lookup.py new file mode 100644 index 000000000..ce5580bb1 --- /dev/null +++ b/python/envctl_engine/state/lookup.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import inspect +from collections.abc import Callable, Sequence +from typing import Any + + +def call_state_loader( + loader: Callable[..., Any], + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, +) -> Any: + """Call current and legacy state-loader adapters without injecting unsupported keywords.""" + + accepts_any_keyword, accepted_keywords = _accepted_keywords(loader) + options: dict[str, object] = {} + for keyword, value in ( + ("mode", mode), + ("strict_mode_match", strict_mode_match), + ): + if accepts_any_keyword or keyword in accepted_keywords: + options[keyword] = value + if project_names is not None: + if not accepts_any_keyword and "project_names" not in accepted_keywords: + raise TypeError("Targeted state lookup requires a loader that accepts project_names") + options["project_names"] = project_names + return loader(**options) + + +def _accepted_keywords(loader: Callable[..., Any]) -> tuple[bool, frozenset[str]]: + try: + parameters = tuple(inspect.signature(loader).parameters.values()) + except (TypeError, ValueError): + return True, frozenset() + return ( + any(parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters), + frozenset( + parameter.name + for parameter in parameters + if parameter.kind in {inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY} + ), + ) + + +__all__ = ["call_state_loader"] diff --git a/python/envctl_engine/state/models.py b/python/envctl_engine/state/models.py index cfcab4405..68bb3c4fd 100644 --- a/python/envctl_engine/state/models.py +++ b/python/envctl_engine/state/models.py @@ -24,6 +24,7 @@ class ServiceRecord: pid: int | None = None requested_port: int | None = None actual_port: int | None = None + port_lock_session: str | None = None log_path: str | None = None status: str = "unknown" synthetic: bool = False diff --git a/python/envctl_engine/state/persistence.py b/python/envctl_engine/state/persistence.py new file mode 100644 index 000000000..6d8e29b64 --- /dev/null +++ b/python/envctl_engine/state/persistence.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import fcntl +import errno +import os +import re +import stat +import tempfile +import time +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + + +_ATOMIC_TEMP_TOKEN = re.compile(r"[a-z0-9_]{8}") + + +def atomic_write_text(path: Path, text: str) -> None: + """Replace a UTF-8 text file without exposing a partially written value.""" + + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + prefix=f".{path.name}.", + suffix=".tmp", + dir=path.parent, + delete=False, + ) as temporary_file: + temporary_path = Path(temporary_file.name) + temporary_file.write(text) + temporary_file.flush() + os.fsync(temporary_file.fileno()) + os.replace(temporary_path, path) + fsync_directory(path.parent) + except BaseException: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + raise + + +def scavenge_atomic_write_temps(path: Path) -> int: + """Remove crash leftovers for one exact atomic-write target. + + The caller must already hold the exclusive lock that serializes writes to + ``path``. Scanning is deliberately target-local: retired runtime history + can be unbounded, and unrelated ``*.tmp`` files must never be touched. + Directory and entry symlinks are not followed. + """ + + parent = path.parent + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + directory_fd = os.open(parent, flags) + removed = 0 + expected_prefix = f".{path.name}." + try: + if not stat.S_ISDIR(os.fstat(directory_fd).st_mode): + raise OSError(f"Atomic-write parent is not a real directory: {parent}") + with os.scandir(directory_fd) as entries: + for entry in entries: + name = entry.name + if not name.startswith(expected_prefix) or not name.endswith(".tmp"): + continue + token = name[len(expected_prefix) : -len(".tmp")] + if _ATOMIC_TEMP_TOKEN.fullmatch(token) is None: + continue + try: + metadata = entry.stat(follow_symlinks=False) + except FileNotFoundError: + continue + if not stat.S_ISREG(metadata.st_mode): + continue + try: + os.unlink(name, dir_fd=directory_fd) + except FileNotFoundError: + continue + removed += 1 + if removed: + fsync_directory(parent) + return removed + finally: + os.close(directory_fd) + + +def durable_mkdir(path: Path) -> None: + """Create a directory chain and durably publish every new directory entry.""" + + missing: list[Path] = [] + cursor = path + while not cursor.exists(): + missing.append(cursor) + if cursor.parent == cursor: + break + cursor = cursor.parent + if cursor.is_symlink() or not cursor.is_dir(): + raise OSError(f"Directory ancestor is not a real directory: {cursor}") + for directory in reversed(missing): + try: + directory.mkdir() + except FileExistsError: + if directory.is_symlink() or not directory.is_dir(): + raise + if directory.is_symlink() or not directory.is_dir(): + raise OSError(f"Created path is not a real directory: {directory}") + fsync_directory(directory) + fsync_directory(directory.parent) + + +def require_path_component(value: object, *, label: str) -> str: + """Return a single portable path component or reject ambiguous traversal input.""" + + raw = str(value) + if ( + not raw + or raw != raw.strip() + or raw in {".", ".."} + or "/" in raw + or "\\" in raw + or "\0" in raw + or Path(raw).is_absolute() + ): + raise ValueError(f"{label} must be a non-empty path component") + return raw + + +def fsync_directory(path: Path) -> None: + """Durably publish directory entry changes without following a replaced symlink.""" + + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + directory_fd = os.open(path, flags) + try: + if not stat.S_ISDIR(os.fstat(directory_fd).st_mode): + raise OSError(f"Path is not a real directory: {path}") + try: + os.fsync(directory_fd) + except OSError as exc: + if exc.errno not in {errno.EBADF, errno.EINVAL, errno.ENOTSUP, errno.EROFS}: + raise + finally: + os.close(directory_fd) + + +@contextmanager +def advisory_file_lock( + path: Path, + *, + exclusive: bool, + timeout_seconds: float | None = None, +) -> Iterator[None]: + """Hold a process-safe advisory lock backed by a stable lock-file inode.""" + + path.parent.mkdir(parents=True, exist_ok=True) + flags = os.O_CREAT | os.O_RDWR + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags, 0o600) + try: + lock_file = os.fdopen(descriptor, "a+b") + except BaseException: + os.close(descriptor) + raise + with lock_file: + if not stat.S_ISREG(os.fstat(lock_file.fileno()).st_mode): + raise OSError(f"Lock path is not a regular file: {path}") + operation = fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH + if timeout_seconds is None: + fcntl.flock(lock_file.fileno(), operation) + else: + deadline = time.monotonic() + max(0.0, timeout_seconds) + while True: + try: + fcntl.flock(lock_file.fileno(), operation | fcntl.LOCK_NB) + break + except BlockingIOError as exc: + if time.monotonic() >= deadline: + raise TimeoutError(f"Timed out acquiring lock: {path}") from exc + time.sleep(0.01) + try: + yield + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + +__all__ = [ + "advisory_file_lock", + "atomic_write_text", + "durable_mkdir", + "fsync_directory", + "require_path_component", + "scavenge_atomic_write_temps", +] diff --git a/python/envctl_engine/state/project_runtime.py b/python/envctl_engine/state/project_runtime.py index c3a69c4f5..aa68f87df 100644 --- a/python/envctl_engine/state/project_runtime.py +++ b/python/envctl_engine/state/project_runtime.py @@ -1,13 +1,34 @@ from __future__ import annotations +import json from dataclasses import dataclass from pathlib import Path from typing import Any, Mapping +from envctl_engine.dashboard_metadata import ( + DASHBOARD_CONFIGURED_SERVICE_TYPES_KEY, + DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY, + DASHBOARD_STOPPED_SERVICES_KEY, + normalize_dashboard_service_types, +) from envctl_engine.shared.services import service_project_name +from envctl_engine.state.fingerprints import text_fingerprint from envctl_engine.state.models import RequirementsResult, RunState +PROJECT_SCOPED_METADATA_MAP_KEYS = ( + "project_roots", + "project_pr_links", + "project_test_summaries", + "project_action_reports", + DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY, +) +_PROJECT_TEST_METADATA_COMPANION_KEYS = ( + "project_test_results_root", + "project_test_results_updated_at", +) + + @dataclass(slots=True) class ProjectRuntimeResolution: ok: bool @@ -140,6 +161,7 @@ def resolve_requested_project_state( state=filter_state_to_projects(state, selected, runtime=runtime), ) + def _service_project_name(service_name: str, service: object, *, runtime: Any | None = None) -> str: explicit_project = str(getattr(service, "project", "") or "").strip() if explicit_project: @@ -171,10 +193,8 @@ def filter_state_to_projects( if _service_project_name(name, service, runtime=runtime) in selected } requirements = _filtered_requirements(state.requirements, selected) - metadata = dict(state.metadata) - roots = metadata.get("project_roots") - if isinstance(roots, Mapping): - metadata["project_roots"] = {str(name): value for name, value in roots.items() if str(name) in selected} + metadata = filter_project_scoped_metadata(state.metadata, selected) + metadata["project_names"] = sorted(selected, key=lambda value: (value.casefold(), value)) return RunState( run_id=state.run_id, mode=state.mode, @@ -187,6 +207,104 @@ def filter_state_to_projects( ) +def filter_project_scoped_metadata( + metadata: Mapping[str, object], + projects: list[str] | tuple[str, ...] | set[str] | frozenset[str], + *, + case_sensitive: bool = True, +) -> dict[str, object]: + """Return metadata containing only records owned by the selected projects. + + Runtime state has several independently evolved project-keyed metadata + surfaces. Keeping their filtering in one place prevents a selected state + from retaining dashboard, PR, test, or action data for another run. + """ + + selected = {str(project).strip() for project in projects if str(project).strip()} + normalized_selected = selected if case_sensitive else {project.casefold() for project in selected} + + def selected_project(project: object) -> bool: + value = str(project).strip() + if not value: + return False + return value in normalized_selected if case_sensitive else value.casefold() in normalized_selected + + filtered = dict(metadata) + for key in PROJECT_SCOPED_METADATA_MAP_KEYS: + raw = metadata.get(key) + if not isinstance(raw, Mapping): + if key in metadata: + filtered.pop(key, None) + continue + retained = {str(project): value for project, value in raw.items() if selected_project(project)} + if retained: + filtered[key] = retained + else: + filtered.pop(key, None) + + raw_stopped = metadata.get(DASHBOARD_STOPPED_SERVICES_KEY) + if isinstance(raw_stopped, list): + stopped: list[dict[str, object]] = [] + for item in raw_stopped: + if not isinstance(item, Mapping) or not selected_project(item.get("project", "")): + continue + if not str(item.get("name", "")).strip() or not str(item.get("type", "")).strip(): + continue + stopped.append(dict(item)) + if stopped: + filtered[DASHBOARD_STOPPED_SERVICES_KEY] = stopped + else: + filtered.pop(DASHBOARD_STOPPED_SERVICES_KEY, None) + elif DASHBOARD_STOPPED_SERVICES_KEY in metadata: + filtered.pop(DASHBOARD_STOPPED_SERVICES_KEY, None) + + raw_summaries = metadata.get("project_test_summaries") + retained_summaries = filtered.get("project_test_summaries") + summaries_were_narrowed = ( + isinstance(raw_summaries, Mapping) + and isinstance(retained_summaries, Mapping) + and len(retained_summaries) != len(raw_summaries) + ) + if not isinstance(retained_summaries, Mapping) or not retained_summaries or summaries_were_narrowed: + for key in _PROJECT_TEST_METADATA_COMPANION_KEYS: + filtered.pop(key, None) + + if DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY in metadata or DASHBOARD_STOPPED_SERVICES_KEY in metadata: + configured_types: set[str] = set() + configured = filtered.get(DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY) + if isinstance(configured, Mapping): + for service_types in configured.values(): + configured_types.update(normalize_dashboard_service_types(service_types)) + stopped = filtered.get(DASHBOARD_STOPPED_SERVICES_KEY) + if isinstance(stopped, list): + configured_types.update( + normalize_dashboard_service_types( + [item.get("type", "") for item in stopped if isinstance(item, Mapping)] + ) + ) + if configured_types: + filtered[DASHBOARD_CONFIGURED_SERVICE_TYPES_KEY] = sorted(configured_types) + else: + filtered.pop(DASHBOARD_CONFIGURED_SERVICE_TYPES_KEY, None) + + raw_identity = metadata.get("startup_identity") + if isinstance(raw_identity, Mapping): + identity = dict(raw_identity) + raw_projects = identity.get("projects") + identity_projects = raw_projects if isinstance(raw_projects, list) else [] + identity["projects"] = [ + dict(project) + for project in identity_projects + if isinstance(project, Mapping) and selected_project(project.get("name", "")) + ] + identity.pop("fingerprint", None) + serialized = json.dumps(identity, sort_keys=True) + identity["fingerprint"] = text_fingerprint(serialized) + filtered["startup_identity"] = identity + + return filtered + + def _filtered_requirements( requirements: Mapping[str, RequirementsResult], selected: set[str], @@ -281,7 +399,6 @@ def cwd_runtime_warnings( ] - def project_resolution_event_payload( resolution: ProjectRuntimeResolution, state: RunState, diff --git a/python/envctl_engine/state/repository.py b/python/envctl_engine/state/repository.py index b750117ba..5e33259b9 100644 --- a/python/envctl_engine/state/repository.py +++ b/python/envctl_engine/state/repository.py @@ -1,17 +1,39 @@ from __future__ import annotations import json -import hashlib import shutil -from collections.abc import Mapping +import stat +import uuid +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Callable, cast +from envctl_engine.dashboard_metadata import DASHBOARD_STOPPED_SERVICES_KEY from envctl_engine.shared.artifact_names import safe_artifact_stem -from envctl_engine.state.models import RunState -from envctl_engine.state import dump_state, load_legacy_shell_state, load_state, load_state_from_pointer, state_to_dict +from envctl_engine.state import load_legacy_shell_state, load_state, state_to_dict +from envctl_engine.state.fingerprints import file_fingerprint, state_fingerprint, text_fingerprint +from envctl_engine.state.models import RunState, ServiceRecord +from envctl_engine.state.persistence import ( + advisory_file_lock, + atomic_write_text, + durable_mkdir, + fsync_directory, + require_path_component, + scavenge_atomic_write_temps, +) +from envctl_engine.state.project_runtime import filter_project_scoped_metadata +from envctl_engine.state.run_index import RunIndex, RunIndexCandidate, StateSelector +from envctl_engine.state.runtime_map import build_runtime_map +from envctl_engine.state.state_aggregation import ( + IndexedState as _IndexedState, + filter_state_to_owned_projects, + normalized_project_names, + select_indexed_owners, + state_from_indexed_owners, +) @dataclass(slots=True) @@ -24,6 +46,25 @@ class RuntimeStateRepository: COMPAT_READ_WRITE: str = "compat_read_write" COMPAT_READ_ONLY: str = "compat_read_only" SCOPED_ONLY: str = "scoped_only" + _RUNTIME_ARTIFACT_NAMES = frozenset( + { + "events.jsonl", + "runtime_readiness_report.json", + } + ) + _REVISION_ANCILLARY_ARTIFACT_NAMES = ( + "ports_manifest.json", + "error_report.json", + "events.jsonl", + "runtime_readiness_report.json", + ) + _RUN_ALIAS_ARTIFACT_NAMES = ( + "run_state.json", + "runtime_map.json", + *_REVISION_ANCILLARY_ARTIFACT_NAMES, + ) + _RUN_ALIAS_MANIFEST_NAME = ".revision_alias_manifest.json" + _REVISION_CLEANUP_DIR_NAME = ".revision-cleanup" def __init__( self, @@ -34,9 +75,18 @@ def __init__( runtime_scope_id: str, compat_mode: str, ) -> None: - self.runtime_root: Path = runtime_root - self.runtime_legacy_root: Path = runtime_legacy_root - self.runtime_dir: Path = runtime_dir + input_runtime_root = runtime_root.expanduser().absolute() + input_legacy_root = runtime_legacy_root.expanduser().absolute() + if input_runtime_root.is_symlink() or input_legacy_root.is_symlink(): + raise ValueError("runtime roots must not be symlinks") + self.runtime_dir: Path = runtime_dir.resolve() + self.runtime_root: Path = runtime_root.resolve() + self.runtime_legacy_root: Path = runtime_legacy_root.resolve() + try: + self.runtime_root.relative_to(self.runtime_dir) + self.runtime_legacy_root.relative_to(self.runtime_dir) + except ValueError as exc: + raise ValueError("runtime roots must be contained by runtime_dir") from exc self.runtime_scope_id: str = runtime_scope_id self.compat_mode: str = ( compat_mode @@ -48,6 +98,16 @@ def __init__( } else self.COMPAT_READ_WRITE ) + self.run_index = RunIndex( + runtime_root=self.runtime_root, + runtime_dir=self.runtime_dir, + runtime_scope_id=self.runtime_scope_id, + ) + self._lock_path = self.runtime_root / ".state_repository.lock" + self._registry_marker_path = self.runtime_root / "run_registry.json" + self._alias_manifest_path = self.runtime_root / "current_alias.json" + self._runtime_root_identity = self._existing_directory_identity(self.runtime_root) + self._legacy_root_identity = self._existing_directory_identity(self.runtime_legacy_root) def run_state_path(self) -> Path: return self.runtime_root / "run_state.json" @@ -61,25 +121,36 @@ def ports_manifest_path(self) -> Path: def error_report_path(self) -> Path: return self.runtime_root / "error_report.json" + def lifecycle_operation_lock_path(self) -> Path: + self._validate_runtime_roots() + return self.runtime_root / ".lifecycle_operation.lock" + + def ensure_runtime_roots(self) -> None: + self._validate_runtime_roots() + def run_dir_path(self, run_id: str | None) -> Path: + runs_root = self._contained_child(self.runtime_root, "runs", label="runs directory") if run_id is None: - return self.runtime_root / "runs" - return self.runtime_root / "runs" / run_id + return runs_root + normalized_run_id = require_path_component(run_id, label="run_id") + return self._contained_child(runs_root, normalized_run_id, label="run_id") def test_results_dir_path(self, run_id: str, stamp: str | None = None) -> Path: - root = self.run_dir_path(run_id) / "test-results" + root = self._contained_child(self.run_dir_path(run_id), "test-results", label="test results directory") if not stamp: return root - return root / stamp + normalized_stamp = require_path_component(stamp, label="test result stamp") + return self._contained_child(root, normalized_stamp, label="test result stamp") def tree_diffs_dir_path(self, run_id: str | None = None, name: str | None = None) -> Path: if run_id: - root = self.run_dir_path(run_id) / "tree-diffs" + root = self._contained_child(self.run_dir_path(run_id), "tree-diffs", label="tree diffs directory") else: - root = self.runtime_root / "tree-diffs" + root = self._contained_child(self.runtime_root, "tree-diffs", label="tree diffs directory") if not name: return root - return root / name + normalized_name = require_path_component(name, label="tree diff name") + return self._contained_child(root, normalized_name, label="tree diff name") def save_run( self, @@ -92,69 +163,56 @@ def save_run( runtime_map_builder: Callable[[RunState], dict[str, object]], write_runtime_readiness_report: Callable[[Path], None] | None = None, ) -> StateRepositoryPaths: - run_dir = self.run_dir_path(state.run_id) - run_dir.mkdir(parents=True, exist_ok=True) - - run_state_path = run_dir / "run_state.json" - run_state_text = self._write_run_state(state=state, run_state_path=run_state_path, emit=emit) - - runtime_map_text = self._write_runtime_map_text( - state=state, - runtime_map=runtime_map_builder(state), - runtime_map_path=run_dir / "runtime_map.json", - emit_path=run_dir / "runtime_map.json", - emit=emit, + project_names = sorted( + { + *self._context_names(contexts), + *self._runtime_project_names_from_state(state), + }, + key=str.casefold, ) - - ports_manifest = { - "run_id": state.run_id, - "mode": state.mode, - "projects": [self._context_ports_payload(context) for context in contexts], - } - ports_manifest_text = self._json_text(ports_manifest) - self._write_run_and_current_artifact( - run_path=run_dir / "ports_manifest.json", - current_path=self.ports_manifest_path(), - text=ports_manifest_text, - ) - - error_report = { - "run_id": state.run_id, - "errors": errors, - "generated_at": datetime.now(tz=UTC).isoformat(), + self._prepare_state_for_save(state, project_names=project_names, contexts=contexts) + runtime_map = runtime_map_builder(state) + artifacts = { + "run_state.json": self._json_text(state_to_dict(state)), + "runtime_map.json": self._json_text(runtime_map), + "ports_manifest.json": self._json_text( + { + "run_id": state.run_id, + "mode": state.mode, + "projects": [self._context_ports_payload(context) for context in contexts], + } + ), + "error_report.json": self._json_text( + { + "run_id": state.run_id, + "errors": errors, + "generated_at": datetime.now(tz=UTC).isoformat(), + } + ), + "events.jsonl": self._events_text(events), } - error_report_text = self._json_text(error_report) - self._write_run_and_current_artifact( - run_path=run_dir / "error_report.json", - current_path=self.error_report_path(), - text=error_report_text, - ) - - events_text = self._events_text(events) - self._write_run_and_current_artifact( - run_path=run_dir / "events.jsonl", - current_path=self.runtime_root / "events.jsonl", - text=events_text, - ) - - if write_runtime_readiness_report is not None: - write_runtime_readiness_report(run_dir) - - self._write_scoped_pointers(run_state_path=run_state_path, state=state, contexts=contexts) - - if self.compat_mode == self.COMPAT_READ_WRITE: - self._write_legacy_compat( - run_state_path=run_state_path, - run_state_text=run_state_text, - runtime_map_text=runtime_map_text, - ports_manifest_text=ports_manifest_text, - error_report_text=error_report_text, - events_text=events_text, + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + self._carry_forward_runtime_artifacts_unlocked(state, artifacts) + revision_state_path = self._commit_revision_unlocked( state=state, - contexts=contexts, + project_names=project_names, + artifacts=artifacts, + supersede_run_ids=self._source_run_ids(state), ) - - return StateRepositoryPaths(run_dir=run_dir, run_state=run_state_path) + run_dir = self.run_dir_path(state.run_id) + if write_runtime_readiness_report is not None: + try: + write_runtime_readiness_report(run_dir) + except Exception as exc: + self._safe_emit(emit, "state.readiness_report.error", run_id=state.run_id, error=str(exc)) + self._emit_saved_state( + emit, + state=state, + state_path=revision_state_path, + runtime_map_text=artifacts["runtime_map.json"], + ) + return StateRepositoryPaths(run_dir=run_dir, run_state=run_dir / "run_state.json") def save_resume_state( self, @@ -175,65 +233,187 @@ def save_selected_stop_state( state: RunState, emit: Callable[..., None], runtime_map_builder: Callable[[RunState], dict[str, object]], + authoritative_project_names: Sequence[str] | None = None, ) -> dict[str, object]: return self._save_runtime_update( state=state, emit=emit, runtime_map_builder=runtime_map_builder, + authoritative_project_names=authoritative_project_names, ) - def load_latest(self, *, mode: str | None = None, strict_mode_match: bool = False) -> RunState | None: - allowed_root = str(self.runtime_dir) - - def load_for_mode(expected_mode: str | None) -> RunState | None: - def candidate_matches(candidate: RunState) -> bool: - if expected_mode is not None and candidate.mode != expected_mode: - return False - return self._state_matches_scope(candidate) + def load_latest( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, + ) -> RunState | None: + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + self._reconcile_current_aliases_unlocked() + return self._load_latest_unlocked( + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ) - candidate = self._load_state_file_candidate(self.run_state_path(), allowed_root=allowed_root) - if candidate is not None and candidate_matches(candidate): - return candidate + def load_all(self, *, mode: str | None = None) -> list[RunState]: + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + self._reconcile_current_aliases_unlocked() + candidates = self.run_index.candidates(StateSelector(mode=mode, project_names=())) + indexed_states = self._load_valid_index_states(candidates, allowed_root=str(self.runtime_dir)) + if len(indexed_states) != len(candidates): + invalid_run_ids = sorted( + {candidate.run_id for candidate in candidates} + - {indexed.candidate.run_id for indexed in indexed_states} + ) + raise RuntimeError( + "Active runtime state failed integrity validation for run(s): " + ", ".join(invalid_run_ids) + ) + return [ + filter_state_to_owned_projects( + indexed_state.state, + frozenset(indexed_state.candidate.project_names), + service_project_name=self._service_project_name, + ) + for indexed_state in indexed_states + ] + + def has_active_runs(self) -> bool: + """Return registry activity without trusting or loading mutable run payloads.""" + + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + return bool(self.run_index.candidates(StateSelector(mode=None, project_names=()))) + + def deactivate_run(self, run_id: str) -> bool: + return self.deactivate_runs([run_id]) + + def deactivate_runs(self, run_ids: Sequence[str]) -> bool: + normalized_run_ids = {require_path_component(run_id, label="run_id") for run_id in run_ids} + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + candidates = self.run_index.candidates(StateSelector(mode=None, project_names=())) + removed_candidates = [candidate for candidate in candidates if candidate.run_id in normalized_run_ids] + if not removed_candidates: + return False + removed_run_ids = sorted({candidate.run_id for candidate in removed_candidates}) + self.run_index.remove_many(removed_run_ids) + for candidate in removed_candidates: + self._remove_pointers_to_state(self.runtime_root, candidate.state_path) + if self.compat_mode == self.COMPAT_READ_WRITE: + self._remove_pointers_to_state(self.runtime_legacy_root, candidate.state_path) + try: + self._promote_latest_active_state() + finally: + for candidate in removed_candidates: + self._prune_or_enqueue_revision_cleanup_unlocked(candidate) + return True - candidate = self._load_legacy_shell_candidate( - self.runtime_root / "run_state.state", allowed_root=allowed_root + def write_runtime_artifact( + self, + *, + run_id: str | None, + artifact_name: str, + text: str, + ) -> bool: + """Persist a mutable runtime artifact without bypassing lifecycle fencing. + + Run-bound writers are accepted only while that exact run remains in the + active registry. An unbound writer is a deliberate scope-level diagnostic + write (for example, ``envctl doctor``) and is never guessed onto a run. + """ + safe_artifact_name = require_path_component(artifact_name, label="runtime artifact name") + if safe_artifact_name not in self._RUNTIME_ARTIFACT_NAMES: + raise ValueError(f"unsupported mutable runtime artifact: {safe_artifact_name}") + normalized_run_id = require_path_component(run_id, label="run_id") if run_id is not None else None + + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + if normalized_run_id is None: + candidates = self.run_index.candidates(StateSelector(mode=None, project_names=())) + if candidates: + self._write_scope_diagnostic_artifact(safe_artifact_name, text) + else: + self._remove_scope_diagnostic_artifact(safe_artifact_name) + self._write_scope_artifact_aliases(safe_artifact_name, text) + return True + + candidates = self.run_index.candidates(StateSelector(mode=None, project_names=())) + indexed_states = self._load_valid_index_states( + candidates, + allowed_root=str(self.runtime_dir), ) - if candidate is not None and candidate_matches(candidate): - return candidate - - candidate = self._load_pointer_candidates( - self._ordered_scoped_pointers(mode=expected_mode), - allowed_root=allowed_root, - predicate=candidate_matches, + indexed_state = next( + (candidate for candidate in indexed_states if candidate.candidate.run_id == normalized_run_id), + None, ) - if candidate is not None: - return candidate + if indexed_state is None: + return False + + candidate = indexed_state.candidate + self._write_text(candidate.state_path.parent / safe_artifact_name, text) + self._write_text(self.run_dir_path(candidate.run_id) / safe_artifact_name, text) + self._run_alias_manifest_path(candidate.run_id).unlink(missing_ok=True) + self._promote_latest_active_state() + return True + + def _write_scope_diagnostic_artifact(self, artifact_name: str, text: str) -> None: + diagnostics_dir = self._contained_child( + self.runtime_root, + "diagnostics", + label="diagnostics directory", + ) + durable_mkdir(diagnostics_dir) + self._write_text(diagnostics_dir / artifact_name, text) - if self.compat_mode == self.SCOPED_ONLY: - return None + def _remove_scope_diagnostic_artifact(self, artifact_name: str) -> None: + diagnostics_dir = self.runtime_root / "diagnostics" + if diagnostics_dir.is_symlink() or not diagnostics_dir.is_dir(): + return + (diagnostics_dir / artifact_name).unlink(missing_ok=True) + try: + diagnostics_dir.rmdir() + except OSError: + pass - candidate = self._load_state_file_candidate( - self.runtime_legacy_root / "run_state.json", - allowed_root=allowed_root, - ) - if candidate is not None and candidate_matches(candidate): - return candidate + def _write_scope_artifact_aliases(self, artifact_name: str, text: str) -> None: + self._write_text(self.runtime_root / artifact_name, text) + if self.compat_mode == self.COMPAT_READ_WRITE: + self._write_text(self.runtime_legacy_root / artifact_name, text) - candidate = self._load_legacy_shell_candidate( - self.runtime_legacy_root / "run_state.state", - allowed_root=allowed_root, - ) - if candidate is not None and candidate_matches(candidate): - return candidate + def _load_latest_unlocked( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, + ) -> RunState | None: + allowed_root = str(self.runtime_dir) + selected_projects = tuple(project_names or ()) - candidate = self._load_pointer_candidates( - self._ordered_legacy_pointers(mode=expected_mode), + def load_for_mode(expected_mode: str | None) -> RunState | None: + candidates = self.run_index.candidates(StateSelector(mode=expected_mode, project_names=())) + if expected_mode is None and candidates: + valid_candidates = self._load_valid_index_states( + candidates, + allowed_root=allowed_root, + ) + if not valid_candidates: + return None + newest_mode = max( + valid_candidates, + key=lambda indexed: indexed.candidate.activation_sequence, + ).candidate.mode + candidates = [candidate for candidate in candidates if candidate.mode == newest_mode] + indexed_candidate = self._load_index_candidates( + candidates, allowed_root=allowed_root, - predicate=candidate_matches, + selected_projects=selected_projects, ) - if candidate is not None: - return candidate - return None + return indexed_candidate matched = load_for_mode(mode) if matched is not None: @@ -248,113 +428,1086 @@ def _save_runtime_update( state: RunState, emit: Callable[..., None], runtime_map_builder: Callable[[RunState], dict[str, object]], + authoritative_project_names: Sequence[str] | None = None, ) -> dict[str, object]: - run_state_path = self._runtime_update_run_state_path(state) - run_state_text = self._write_run_state(state=state, run_state_path=run_state_path, emit=emit) - + project_names = ( + self._runtime_project_names_from_state(state) + if authoritative_project_names is None + else sorted( + { + str(project_name).strip() + for project_name in authoritative_project_names + if str(project_name).strip() + }, + key=str.casefold, + ) + ) + self._prepare_state_for_save(state, project_names=project_names, authoritative_projects=True) runtime_map = runtime_map_builder(state) - runtime_map_text = self._write_runtime_map_text( + if not project_names: + run_ids = [state.run_id, *self._source_run_ids(state)] + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + self.run_index.remove_many(run_ids) + self._promote_latest_active_state() + self._safe_emit(emit, "state.deactivate", run_ids=sorted(set(run_ids))) + return runtime_map + artifacts = { + "run_state.json": self._json_text(state_to_dict(state)), + "runtime_map.json": self._json_text(runtime_map), + } + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + self._carry_forward_runtime_artifacts_unlocked(state, artifacts) + revision_state_path = self._commit_revision_unlocked( + state=state, + project_names=project_names, + artifacts=artifacts, + supersede_run_ids=self._source_run_ids(state), + ) + self._emit_saved_state( + emit, state=state, - runtime_map=runtime_map, - runtime_map_path=run_state_path.parent / "runtime_map.json", - emit_path=self.runtime_map_path(), - emit=emit, + state_path=revision_state_path, + runtime_map_text=artifacts["runtime_map.json"], ) + return runtime_map + def _carry_forward_runtime_artifacts_unlocked( + self, + state: RunState, + artifacts: dict[str, str], + ) -> None: + source_run_ids = {state.run_id, *self._source_run_ids(state)} + source_candidates = [ + candidate + for candidate in self.run_index.candidates(StateSelector(mode=state.mode, project_names=())) + if candidate.run_id in source_run_ids + ] + current_ports_manifest = artifacts.get("ports_manifest.json") + artifacts["ports_manifest.json"] = self._merged_ports_manifest_text( + state, + source_candidates, + current_manifest_text=current_ports_manifest, + ) + artifacts.setdefault( + "error_report.json", + self._merged_error_report_text(state, source_candidates), + ) + if not source_candidates: + return + source_candidate = max( + source_candidates, + key=lambda candidate: candidate.activation_sequence, + ) + source_directories = ( + source_candidate.state_path.parent, + self.run_dir_path(source_candidate.run_id), + ) + for artifact_name in self._RUNTIME_ARTIFACT_NAMES: + if artifact_name in artifacts: + continue + for source_directory in source_directories: + source = source_directory / artifact_name + if source.is_file() and not source.is_symlink(): + try: + artifacts[artifact_name] = source.read_text(encoding="utf-8") + except (OSError, UnicodeError): + continue + break + + def _merged_ports_manifest_text( + self, + state: RunState, + source_candidates: Sequence[RunIndexCandidate], + *, + current_manifest_text: str | None = None, + ) -> str: project_names = self._project_names_from_state(state) - self._write_mode_pointers( - root=self.runtime_root, - run_state_path=run_state_path, - mode=state.mode, - project_names=project_names, + selected = normalized_project_names(project_names) + projects_by_key: dict[str, dict[str, object]] = {} + for candidate in sorted( + source_candidates, + key=lambda item: item.activation_sequence, + ): + payload = self._read_revision_json_artifact(candidate, "ports_manifest.json") + raw_projects = payload.get("projects") if isinstance(payload, Mapping) else None + if not isinstance(raw_projects, list): + continue + for raw_project in raw_projects: + if not isinstance(raw_project, Mapping): + continue + project_name = str(raw_project.get("project", "")).strip() + project_key = project_name.casefold() + if project_key and project_key in selected: + projects_by_key[project_key] = dict(raw_project) + + if current_manifest_text is not None: + try: + current_payload = json.loads(current_manifest_text) + except (TypeError, ValueError, json.JSONDecodeError): + current_payload = {} + raw_current_projects = current_payload.get("projects") if isinstance(current_payload, Mapping) else None + if isinstance(raw_current_projects, list): + for raw_project in raw_current_projects: + if not isinstance(raw_project, Mapping): + continue + project_name = str(raw_project.get("project", "")).strip() + project_key = project_name.casefold() + if project_key and project_key in selected: + projects_by_key[project_key] = dict(raw_project) + + roots = state.metadata.get("project_roots") + project_roots = roots if isinstance(roots, Mapping) else {} + for project_name in project_names: + project_key = project_name.casefold() + project_payload = projects_by_key.get(project_key) + if project_payload is None: + project_payload = { + "project": project_name, + "root": next( + ( + str(root) + for name, root in project_roots.items() + if str(name).strip().casefold() == project_key + ), + "", + ), + "ports": {}, + } + projects_by_key[project_key] = project_payload + raw_ports = project_payload.get("ports") + ports = dict(raw_ports) if isinstance(raw_ports, Mapping) else {} + for service_name, service in state.services.items(): + if self._service_project_name(service_name, service).casefold() != project_key: + continue + service_key = str(service.type or service_name).strip().casefold() or "service" + ports.setdefault( + service_key, + { + "requested": service.requested_port, + "assigned": service.actual_port or service.requested_port, + "final": service.actual_port, + "source": "runtime_state", + "retries": None, + }, + ) + project_payload["ports"] = ports + return self._json_text( + { + "run_id": state.run_id, + "mode": state.mode, + "projects": [projects_by_key[key] for key in sorted(projects_by_key)], + } ) - if self.compat_mode == self.COMPAT_READ_WRITE: - self.runtime_legacy_root.mkdir(parents=True, exist_ok=True) - self._write_text(self.runtime_legacy_root / "run_state.json", run_state_text) - self._write_text(self.runtime_legacy_root / "runtime_map.json", runtime_map_text) - self._write_mode_pointers( - root=self.runtime_legacy_root, - run_state_path=run_state_path, + def _merged_error_report_text( + self, + state: RunState, + source_candidates: Sequence[RunIndexCandidate], + ) -> str: + errors: list[str] = [] + for candidate in sorted(source_candidates, key=lambda item: item.activation_sequence): + payload = self._read_revision_json_artifact(candidate, "error_report.json") + raw_errors = payload.get("errors") if isinstance(payload, Mapping) else None + if not isinstance(raw_errors, list): + continue + for raw_error in raw_errors: + error = str(raw_error).strip() + if error and error not in errors: + errors.append(error) + return self._json_text( + { + "run_id": state.run_id, + "errors": errors, + "source_run_ids": sorted({candidate.run_id for candidate in source_candidates}), + "generated_at": datetime.now(tz=UTC).isoformat(), + } + ) + + @staticmethod + def _read_revision_json_artifact( + candidate: RunIndexCandidate, + artifact_name: str, + ) -> Mapping[str, object]: + path = candidate.state_path.parent / artifact_name + if path.is_symlink() or not path.is_file(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return {} + return payload if isinstance(payload, Mapping) else {} + + def _commit_revision_unlocked( + self, + *, + state: RunState, + project_names: Sequence[str], + artifacts: Mapping[str, str], + supersede_run_ids: Sequence[str] = (), + ) -> Path: + superseded_run_ids = { + require_path_component(run_id, label="superseded run_id") + for run_id in supersede_run_ids + if str(run_id).strip() and str(run_id).strip() != state.run_id + } + superseded_candidates = [ + candidate + for candidate in self.run_index.candidates(StateSelector(mode=None, project_names=())) + if candidate.run_id in superseded_run_ids + ] + run_dir = self.run_dir_path(state.run_id) + revisions_dir = self._contained_child(run_dir, "revisions", label="revisions directory") + revision_id = f"{datetime.now(tz=UTC).strftime('%Y%m%dT%H%M%S%fZ')}-{uuid.uuid4().hex[:12]}" + revision_dir = self._contained_child(revisions_dir, revision_id, label="revision id") + revision_state_path = revision_dir / "run_state.json" + committed = False + try: + durable_mkdir(revision_dir) + for artifact_name, text in artifacts.items(): + safe_name = require_path_component(artifact_name, label="artifact name") + self._write_text(revision_dir / safe_name, text) + self.run_index.record( + state_path=revision_state_path, + run_id=state.run_id, mode=state.mode, project_names=project_names, + supersede_run_ids=supersede_run_ids, ) + committed = True + finally: + if not committed: + shutil.rmtree(revision_dir, ignore_errors=True) - return runtime_map + try: + self._publish_revision_aliases( + state=state, + artifacts=artifacts, + ) + self._promote_latest_active_state() + for candidate in superseded_candidates: + self._prune_or_enqueue_revision_cleanup_unlocked(candidate) + except OSError: + # The index is the commit point. Aliases are repaired on the next repository read. + for candidate in superseded_candidates: + self._enqueue_revision_cleanup_if_needed_unlocked(candidate) + except BaseException: + for candidate in superseded_candidates: + self._enqueue_revision_cleanup_if_needed_unlocked(candidate) + raise + return revision_state_path + + def _publish_revision_aliases( + self, + *, + state: RunState, + artifacts: Mapping[str, str], + ) -> None: + run_dir = self.run_dir_path(state.run_id) + for artifact_name in self._RUN_ALIAS_ARTIFACT_NAMES: + alias = run_dir / artifact_name + text = artifacts.get(artifact_name) + if text is None: + alias.unlink(missing_ok=True) + else: + self._write_text(alias, text) + candidates = self.run_index.candidates(StateSelector(mode=state.mode, project_names=())) + committed = next( + (candidate for candidate in candidates if candidate.run_id == state.run_id), + None, + ) + if committed is not None: + self._write_run_alias_manifest(committed) + + def _run_alias_manifest_path(self, run_id: str) -> Path: + return self.run_dir_path(run_id) / self._RUN_ALIAS_MANIFEST_NAME + + def _write_run_alias_manifest(self, candidate: RunIndexCandidate) -> None: + source_dir = candidate.state_path.parent + run_dir = self.run_dir_path(candidate.run_id) + artifacts = { + artifact_name: { + "source": self._path_lstat_signature(source_dir / artifact_name), + "alias": self._path_lstat_signature(run_dir / artifact_name), + } + for artifact_name in self._RUN_ALIAS_ARTIFACT_NAMES + } + self._write_text( + self._run_alias_manifest_path(candidate.run_id), + self._json_text( + { + "version": 1, + "source_dir": str(source_dir), + "artifacts": artifacts, + } + ), + ) - def load_by_pointer(self, pointer_path: str) -> RunState: - return load_state_from_pointer(pointer_path, allowed_root=str(self.runtime_dir)) + def _run_aliases_match_manifest(self, candidate: RunIndexCandidate) -> bool: + manifest_path = self._run_alias_manifest_path(candidate.run_id) + if manifest_path.is_symlink() or not manifest_path.is_file(): + return False + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return False + if not isinstance(manifest, Mapping): + return False + if manifest.get("version") != 1 or manifest.get("source_dir") != str(candidate.state_path.parent): + return False + expected_artifacts = manifest.get("artifacts") + if not isinstance(expected_artifacts, Mapping): + return False + source_dir = candidate.state_path.parent + run_dir = self.run_dir_path(candidate.run_id) + actual_artifacts = { + artifact_name: { + "source": self._path_lstat_signature(source_dir / artifact_name), + "alias": self._path_lstat_signature(run_dir / artifact_name), + } + for artifact_name in self._RUN_ALIAS_ARTIFACT_NAMES + } + return expected_artifacts == actual_artifacts - def _load_state_file_candidate(self, path: Path, *, allowed_root: str) -> RunState | None: - if not path.is_file(): + @staticmethod + def _path_lstat_signature(path: Path) -> dict[str, object]: + try: + metadata = path.lstat() + except FileNotFoundError: + return {"kind": "missing"} + if not stat.S_ISREG(metadata.st_mode): + return { + "kind": "unsafe", + "mode": metadata.st_mode, + "inode": metadata.st_ino, + "size": metadata.st_size, + "mtime_ns": metadata.st_mtime_ns, + "ctime_ns": metadata.st_ctime_ns, + } + return { + "kind": "file", + "inode": metadata.st_ino, + "size": metadata.st_size, + "mtime_ns": metadata.st_mtime_ns, + "ctime_ns": metadata.st_ctime_ns, + } + + def _prune_revisions_for_candidates_unlocked( + self, + indexed_states: Sequence[_IndexedState], + ) -> None: + seen_run_ids: set[str] = set() + for indexed in indexed_states: + candidate = indexed.candidate + if candidate.run_id in seen_run_ids: + continue + seen_run_ids.add(candidate.run_id) + try: + self._prune_run_revisions_unlocked(candidate) + except OSError: + # Active candidates are retried by the next reconciliation. + continue + self._retry_pending_revision_cleanups_unlocked() + + def _prune_run_revisions_unlocked(self, candidate: RunIndexCandidate) -> None: + run_dir = self.run_dir_path(candidate.run_id) + revisions_dir = self._contained_child( + run_dir, + "revisions", + label="revisions directory", + ) + current_revision = candidate.state_path.parent + if ( + revisions_dir.is_symlink() + or not revisions_dir.is_dir() + or current_revision.is_symlink() + or not current_revision.is_dir() + or current_revision.resolve().parent != revisions_dir.resolve() + ): + return + siblings: list[tuple[int, str, Path]] = [] + for path in revisions_dir.iterdir(): + if path == current_revision or path.is_symlink() or not path.is_dir(): + continue + try: + if path.resolve().parent != revisions_dir.resolve(): + continue + metadata = path.stat() + except OSError: + continue + siblings.append((metadata.st_mtime_ns, path.name, path)) + siblings.sort(reverse=True) + removed = False + try: + for _, _, stale_revision in siblings[1:]: + shutil.rmtree(stale_revision) + removed = True + finally: + if removed: + fsync_directory(revisions_dir) + + def _enqueue_revision_cleanup_unlocked(self, candidate: RunIndexCandidate) -> None: + marker_path = self._revision_cleanup_marker_path(candidate.run_id) + run_dir = self.run_dir_path(candidate.run_id) + revisions_dir = self._contained_child( + run_dir, + "revisions", + label="revisions directory", + ) + state_path = candidate.state_path.resolve() + if state_path.name != "run_state.json" or state_path.parent.parent != revisions_dir.resolve(): + raise ValueError("revision cleanup state path is outside the run revisions directory") + cleanup_dir = marker_path.parent + durable_mkdir(cleanup_dir) + self._write_text( + marker_path, + self._json_text( + { + "version": 1, + "runtime_scope_id": self.runtime_scope_id, + "run_id": candidate.run_id, + "mode": candidate.mode, + "state_path": str(state_path), + } + ), + ) + + def _prune_or_enqueue_revision_cleanup_unlocked( + self, + candidate: RunIndexCandidate, + ) -> None: + try: + self._prune_run_revisions_unlocked(candidate) + except OSError: + self._enqueue_revision_cleanup_if_needed_unlocked(candidate) + + def _enqueue_revision_cleanup_if_needed_unlocked( + self, + candidate: RunIndexCandidate, + ) -> None: + if self._run_revision_count(candidate.run_id) <= 2: + return + try: + self._enqueue_revision_cleanup_unlocked(candidate) + except (OSError, RuntimeError, ValueError): + return + + def _retry_pending_revision_cleanups_unlocked(self) -> None: + cleanup_dir = self.runtime_root / self._REVISION_CLEANUP_DIR_NAME + if cleanup_dir.is_symlink() or not cleanup_dir.is_dir(): + return + try: + marker_paths = list(cleanup_dir.iterdir()) + except OSError: + return + for marker_path in marker_paths: + if marker_path.is_symlink() or not marker_path.is_file(): + continue + candidate = self._revision_cleanup_candidate(marker_path) + if candidate is None: + self._remove_revision_cleanup_marker(marker_path) + continue + try: + self._prune_run_revisions_unlocked(candidate) + except OSError: + continue + if self._run_revision_count(candidate.run_id) <= 2: + self._remove_revision_cleanup_marker(marker_path) + try: + cleanup_dir.rmdir() + except OSError: + return + try: + fsync_directory(self.runtime_root) + except OSError: + pass + + def _revision_cleanup_candidate( + self, + marker_path: Path, + ) -> RunIndexCandidate | None: + try: + payload = json.loads(marker_path.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + return None + if payload.get("version") != 1 or payload.get("runtime_scope_id") != self.runtime_scope_id: + return None + raw_run_id = payload.get("run_id") + raw_mode = payload.get("mode") + raw_state_path = payload.get("state_path") + if not all(isinstance(value, str) and value for value in (raw_run_id, raw_mode, raw_state_path)): + return None + run_id = require_path_component(raw_run_id, label="run_id") + if marker_path != self._revision_cleanup_marker_path(run_id): + return None + run_dir = self.run_dir_path(run_id) + revisions_dir = self._contained_child( + run_dir, + "revisions", + label="revisions directory", + ) + state_path = Path(raw_state_path).expanduser().resolve() + if state_path.name != "run_state.json" or state_path.parent.parent != revisions_dir.resolve(): + return None + state = self._load_state_file_candidate( + state_path, + allowed_root=str(self.runtime_dir), + ) + if state is None or state.run_id != run_id: + return None + if state.metadata.get("repo_scope_id") != self.runtime_scope_id: + return None + if state.mode.casefold() != raw_mode.casefold(): + return None + return RunIndexCandidate( + state_path=state_path, + run_id=run_id, + mode=state.mode, + project_names=(), + sequence=0, + ) + except (OSError, RuntimeError, TypeError, ValueError, json.JSONDecodeError): return None + + def _revision_cleanup_marker_path(self, run_id: str) -> Path: + normalized_run_id = require_path_component(run_id, label="run_id") + cleanup_dir = self._contained_child( + self.runtime_root, + self._REVISION_CLEANUP_DIR_NAME, + label="revision cleanup directory", + ) + digest = text_fingerprint(normalized_run_id) + marker_name = f"{digest}.json" + return self._contained_child( + cleanup_dir, + marker_name, + label="revision cleanup marker", + ) + + def _remove_revision_cleanup_marker(self, marker_path: Path) -> None: try: - return load_state(str(path), allowed_root=allowed_root) + marker_path.unlink(missing_ok=True) + fsync_directory(marker_path.parent) + except OSError: + return + + def _run_revision_count(self, run_id: str) -> int: + revisions_dir = self.run_dir_path(run_id) / "revisions" + if revisions_dir.is_symlink() or not revisions_dir.is_dir(): + return 0 + try: + return sum(1 for path in revisions_dir.iterdir() if not path.is_symlink() and path.is_dir()) + except OSError: + return 3 + + def _emit_saved_state( + self, + emit: Callable[..., None], + *, + state: RunState, + state_path: Path, + runtime_map_text: str, + ) -> None: + self._safe_emit(emit, "state.save", run_id=state.run_id, path=str(state_path)) + self._safe_emit( + emit, + "state.fingerprint.after_save", + run_id=state.run_id, + state_fingerprint=state_fingerprint(state), + ) + self._safe_emit(emit, "runtime_map.write", path=str(state_path.parent / "runtime_map.json")) + self._safe_emit( + emit, + "runtime_map.fingerprint", + run_id=state.run_id, + runtime_map_fingerprint=text_fingerprint(runtime_map_text), + ) + + @staticmethod + def _safe_emit(emit: Callable[..., None], event: str, **payload: object) -> None: + try: + emit(event, **payload) except Exception: - return None + return - def _load_legacy_shell_candidate(self, path: Path, *, allowed_root: str) -> RunState | None: + def _load_state_file_candidate(self, path: Path, *, allowed_root: str) -> RunState | None: if not path.is_file(): return None try: - return load_legacy_shell_state(str(path), allowed_root=allowed_root) + return load_state(str(path), allowed_root=allowed_root) except Exception: return None - def _load_pointer_candidates( + def _load_index_candidates( self, - pointers: list[Path], + candidates: Sequence[RunIndexCandidate], *, allowed_root: str, - predicate: Callable[[RunState], bool], + selected_projects: Sequence[str], ) -> RunState | None: - for pointer in pointers: + indexed_states = self._load_valid_index_states(candidates, allowed_root=allowed_root) + if not indexed_states: + return None + chosen_states = select_indexed_owners( + indexed_states, + selected_projects=selected_projects, + service_project_name=self._service_project_name, + ) + return state_from_indexed_owners( + chosen_states, + project_names_from_state=self._project_names_from_state, + source_run_ids=self._source_run_ids, + ) + + def _ensure_active_index_unlocked(self) -> None: + if not self.run_index.needs_rebuild(): + self.run_index.repair_copies() + if not self._registry_marker_path.is_file(): + self._write_text( + self._registry_marker_path, + self._json_text({"version": 1, "runtime_scope_id": self.runtime_scope_id}), + ) + return + if self._registry_marker_path.is_file(): + raise RuntimeError( + "Active run registry is unavailable because both the primary and backup indexes are invalid." + ) + + recovered: list[tuple[int, Path, RunState, list[str]]] = [] + runs_root = self.run_dir_path(None) + if runs_root.is_dir(): + for run_dir in runs_root.iterdir(): + if run_dir.is_symlink() or not run_dir.is_dir(): + continue + state_path = run_dir / "run_state.json" + if state_path.is_symlink() or not state_path.is_file(): + continue + try: + run_id = require_path_component(run_dir.name, label="run_id") + if run_dir.resolve().parent != runs_root.resolve(): + continue + state = self._load_state_file_candidate(state_path, allowed_root=str(self.runtime_dir)) + if state is None or state.run_id != run_id: + continue + scope = state.metadata.get("repo_scope_id") + if isinstance(scope, str) and scope and scope != self.runtime_scope_id: + continue + project_names = self._project_names_from_state(state) + if not project_names: + project_names = self._project_names_from_ports_manifest( + state_path.parent / "ports_manifest.json" + ) + if not project_names: + project_names = self._project_names_from_pointers( + self.runtime_root, + state_path, + ) + modified_at = state_path.stat().st_mtime_ns + state.metadata["repo_scope_id"] = self.runtime_scope_id + state.metadata["project_names"] = sorted(set(project_names), key=str.casefold) + recovered.append((modified_at, state_path, state, project_names)) + except (OSError, RuntimeError, ValueError): + continue + recovered_run_ids = {state.run_id for _, _, state, _ in recovered} + for root in self._legacy_migration_roots(): + for candidate_path, legacy_loader in ( + (root / "run_state.json", self._load_state_file_candidate), + (root / "run_state.state", self._load_legacy_shell_candidate), + ): + if candidate_path.is_symlink() or not candidate_path.is_file(): + continue + try: + state = legacy_loader(candidate_path, allowed_root=str(self.runtime_dir)) + if state is None or state.run_id in recovered_run_ids: + continue + existing_scope = state.metadata.get("repo_scope_id") + if isinstance(existing_scope, str) and existing_scope and existing_scope != self.runtime_scope_id: + continue + project_names = self._project_names_from_state(state) + if not project_names: + project_names = self._project_names_from_ports_manifest(root / "ports_manifest.json") + if not project_names: + project_names = self._project_names_from_pointers(root, candidate_path) + state.metadata["repo_scope_id"] = self.runtime_scope_id + state.metadata["project_names"] = sorted(set(project_names), key=str.casefold) + recovered.append((candidate_path.stat().st_mtime_ns, candidate_path, state, project_names)) + recovered_run_ids.add(state.run_id) + except (OSError, RuntimeError, ValueError): + continue + + indexed_entries: list[RunIndexCandidate] = [] + registry_committed = False + try: + for sequence, (_, source_path, state, project_names) in enumerate( + sorted(recovered, key=lambda item: (item[0], str(item[1]))), + start=1, + ): + revision_state_path = self._stage_legacy_revision( + state=state, + source_path=source_path, + ) + indexed_entries.append( + RunIndexCandidate( + state_path=revision_state_path, + run_id=state.run_id, + mode=state.mode, + project_names=tuple(sorted(normalized_project_names(project_names))), + sequence=sequence, + ) + ) + self.run_index.replace_all(indexed_entries) + registry_committed = True + finally: + if not registry_committed: + for entry in indexed_entries: + shutil.rmtree(entry.state_path.parent, ignore_errors=True) + self._write_text( + self._registry_marker_path, + self._json_text({"version": 1, "runtime_scope_id": self.runtime_scope_id}), + ) + + def _legacy_migration_roots(self) -> list[Path]: + roots = [self.runtime_root] + if self.compat_mode != self.SCOPED_ONLY and self.runtime_legacy_root != self.runtime_root: + roots.append(self.runtime_legacy_root) + return roots + + def _stage_legacy_revision(self, *, state: RunState, source_path: Path) -> Path: + run_dir = self.run_dir_path(state.run_id) + revisions_dir = self._contained_child(run_dir, "revisions", label="revisions directory") + revision_id = f"legacy-{uuid.uuid4().hex}" + revision_dir = self._contained_child(revisions_dir, revision_id, label="revision id") + revision_state_path = revision_dir / "run_state.json" + staged = False + try: + durable_mkdir(revision_dir) + self._write_text(revision_state_path, self._json_text(state_to_dict(state))) + for artifact_name in ( + "runtime_map.json", + *self._REVISION_ANCILLARY_ARTIFACT_NAMES, + ): + source_artifact = source_path.parent / artifact_name + if source_artifact.is_symlink() or not source_artifact.is_file(): + continue + try: + self._write_text( + revision_dir / artifact_name, + source_artifact.read_text(encoding="utf-8"), + ) + except (OSError, UnicodeError): + continue + staged = True + finally: + if not staged: + shutil.rmtree(revision_dir, ignore_errors=True) + return revision_state_path + + @staticmethod + def _project_names_from_ports_manifest(path: Path) -> list[str]: + if not path.is_file(): + return [] + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return [] + raw_projects = payload.get("projects") if isinstance(payload, Mapping) else None + if not isinstance(raw_projects, list): + return [] + return sorted( + { + str(project.get("project", "")).strip() + for project in raw_projects + if isinstance(project, Mapping) and str(project.get("project", "")).strip() + }, + key=str.casefold, + ) + + @staticmethod + def _project_names_from_pointers(root: Path, state_path: Path) -> list[str]: + expected = state_path.resolve() + project_names: set[str] = set() + for pointer in root.glob(".last_state.trees.*"): + if pointer.is_symlink() or not pointer.is_file(): + continue try: - candidate = load_state_from_pointer(str(pointer), allowed_root=allowed_root) - except Exception: + raw_target = pointer.read_text(encoding="utf-8").splitlines()[0].strip() + target = Path(raw_target).expanduser().resolve() + except (IndexError, OSError, RuntimeError, ValueError): continue - if predicate(candidate): - return candidate - return None + if target == expected: + project_names.add(pointer.name.removeprefix(".last_state.trees.")) + return sorted(project_names, key=str.casefold) - def _write_run_state( - self, - *, - state: RunState, - run_state_path: Path, - emit: Callable[..., None], - ) -> str: - run_state_path.parent.mkdir(parents=True, exist_ok=True) - dump_state(state, str(run_state_path)) - run_state_text = run_state_path.read_text(encoding="utf-8") - self.run_state_path().parent.mkdir(parents=True, exist_ok=True) - self._write_text(self.run_state_path(), run_state_text) - emit("state.save", run_id=state.run_id, path=str(run_state_path)) - emit("state.fingerprint.after_save", run_id=state.run_id, state_fingerprint=self._state_fingerprint(state)) - return run_state_text - - def _write_runtime_map_text( + def _load_valid_index_states( self, + candidates: Sequence[RunIndexCandidate], *, - state: RunState, - runtime_map: dict[str, object], - runtime_map_path: Path, - emit_path: Path, - emit: Callable[..., None], - ) -> str: - runtime_map_text = self._json_text(runtime_map) - self._write_run_and_current_artifact( - run_path=runtime_map_path, - current_path=self.runtime_map_path(), - text=runtime_map_text, - ) - emit("runtime_map.write", path=str(emit_path)) - emit( - "runtime_map.fingerprint", - run_id=state.run_id, - runtime_map_fingerprint=self._text_fingerprint(runtime_map_text), + allowed_root: str, + ) -> list[_IndexedState]: + loaded: list[_IndexedState] = [] + for indexed in candidates: + try: + if file_fingerprint(indexed.state_path) != indexed.state_fingerprint: + continue + if indexed.runtime_map_fingerprint is not None: + runtime_map_path = indexed.state_path.parent / "runtime_map.json" + if ( + runtime_map_path.is_symlink() + or not runtime_map_path.is_file() + or file_fingerprint(runtime_map_path) != indexed.runtime_map_fingerprint + ): + continue + except OSError: + continue + state = self._load_state_file_candidate(indexed.state_path, allowed_root=allowed_root) + if state is None: + continue + if state.run_id != indexed.run_id or state.mode.casefold() != indexed.mode: + continue + if state.metadata.get("repo_scope_id") != self.runtime_scope_id: + continue + state_projects = normalized_project_names(self._project_names_from_state(state)) + indexed_projects = frozenset(indexed.project_names) + if indexed_projects and not indexed_projects.issubset(state_projects): + continue + loaded.append(_IndexedState(candidate=indexed, state=state)) + return loaded + + def _promote_latest_active_state(self) -> None: + candidates = self.run_index.candidates(StateSelector(mode=None, project_names=())) + if not candidates: + self._remove_current_aliases() + return + valid_candidates = self._load_valid_index_states( + candidates, + allowed_root=str(self.runtime_dir), ) - return runtime_map_text + if not valid_candidates: + self._remove_current_aliases() + return + latest_candidate = max( + valid_candidates, + key=lambda indexed: indexed.candidate.activation_sequence, + ).candidate + mode_candidates = [indexed for indexed in valid_candidates if indexed.candidate.mode == latest_candidate.mode] + chosen_states = select_indexed_owners( + mode_candidates, + selected_projects=(), + service_project_name=self._service_project_name, + ) + latest_state = state_from_indexed_owners( + chosen_states, + project_names_from_state=self._project_names_from_state, + source_run_ids=self._source_run_ids, + ) + if latest_state is None: + self._remove_current_aliases() + return + selected_candidates = [indexed.candidate for indexed in chosen_states] + latest_selected = max( + selected_candidates, + key=lambda candidate: ( + candidate.activation_sequence, + candidate.sequence, + str(candidate.state_path), + ), + ) + source_dir = latest_selected.state_path.parent + current_state_text = self._json_text(state_to_dict(latest_state)) + current_runtime_map_text = self._json_text(build_runtime_map(latest_state)) + aggregate_artifacts = { + "ports_manifest.json": self._merged_ports_manifest_text( + latest_state, + selected_candidates, + ), + "error_report.json": self._merged_error_report_text( + latest_state, + selected_candidates, + ), + } + self._write_text(self.run_state_path(), current_state_text) + self._write_text(self.runtime_map_path(), current_runtime_map_text) + for artifact_name in self._REVISION_ANCILLARY_ARTIFACT_NAMES: + current = self.runtime_root / artifact_name + aggregate_text = aggregate_artifacts.get(artifact_name) + if aggregate_text is not None: + self._write_text(current, aggregate_text) + else: + source = source_dir / artifact_name + if source.is_file() and not source.is_symlink(): + self._write_text(current, source.read_text(encoding="utf-8")) + else: + current.unlink(missing_ok=True) + self._rebuild_mode_pointers(self.runtime_root, valid_candidates) + if self.compat_mode == self.COMPAT_READ_WRITE: + self._write_text(self.runtime_legacy_root / "run_state.json", current_state_text) + self._write_text(self.runtime_legacy_root / "runtime_map.json", current_runtime_map_text) + for artifact_name in self._REVISION_ANCILLARY_ARTIFACT_NAMES: + current = self.runtime_legacy_root / artifact_name + aggregate_text = aggregate_artifacts.get(artifact_name) + if aggregate_text is not None: + self._write_text(current, aggregate_text) + else: + source = source_dir / artifact_name + if source.is_file() and not source.is_symlink(): + self._write_text(current, source.read_text(encoding="utf-8")) + else: + current.unlink(missing_ok=True) + self._rebuild_mode_pointers(self.runtime_legacy_root, valid_candidates) + self._write_text( + self._alias_manifest_path, + self._json_text( + { + "version": 2, + "sources": { + indexed.candidate.run_id: str(indexed.candidate.state_path) for indexed in valid_candidates + }, + "source_fingerprints": { + indexed.candidate.run_id: state_fingerprint(indexed.state) for indexed in valid_candidates + }, + "state_fingerprint": text_fingerprint(current_state_text), + "runtime_map_fingerprint": text_fingerprint(current_runtime_map_text), + "source_artifact_signatures": self._revision_artifact_signatures(valid_candidates), + "scoped_alias_signatures": self._artifact_signatures(self.runtime_root), + "legacy_alias_signatures": ( + self._artifact_signatures(self.runtime_legacy_root) + if self.compat_mode == self.COMPAT_READ_WRITE + else None + ), + } + ), + ) + self._prune_revisions_for_candidates_unlocked(valid_candidates) + + def _reconcile_current_aliases_unlocked(self) -> None: + candidates = self.run_index.candidates(StateSelector(mode=None, project_names=())) + if not candidates: + if self.run_state_path().exists() or (self.runtime_root / ".last_state").exists(): + self._remove_current_aliases() + return + valid_candidates = self._load_valid_index_states( + candidates, + allowed_root=str(self.runtime_dir), + ) + expected_sources = {indexed.candidate.run_id: str(indexed.candidate.state_path) for indexed in valid_candidates} + expected_source_fingerprints = { + indexed.candidate.run_id: state_fingerprint(indexed.state) for indexed in valid_candidates + } + expected_source_artifact_signatures = self._revision_artifact_signatures(valid_candidates) + try: + self._reconcile_run_aliases_unlocked(valid_candidates) + except OSError: + pass + try: + manifest = json.loads(self._alias_manifest_path.read_text(encoding="utf-8")) + state_text = self.run_state_path().read_text(encoding="utf-8") + runtime_map_text = self.runtime_map_path().read_text(encoding="utf-8") + if ( + isinstance(manifest, Mapping) + and manifest.get("version") == 2 + and manifest.get("sources") == expected_sources + and manifest.get("source_fingerprints") == expected_source_fingerprints + and manifest.get("source_artifact_signatures") == expected_source_artifact_signatures + and manifest.get("state_fingerprint") == text_fingerprint(state_text) + and manifest.get("runtime_map_fingerprint") == text_fingerprint(runtime_map_text) + and manifest.get("scoped_alias_signatures") == self._artifact_signatures(self.runtime_root) + and manifest.get("legacy_alias_signatures") + == ( + self._artifact_signatures(self.runtime_legacy_root) + if self.compat_mode == self.COMPAT_READ_WRITE + else None + ) + and self._mode_pointers_match(self.runtime_root, valid_candidates) + and ( + self.compat_mode != self.COMPAT_READ_WRITE + or self._mode_pointers_match(self.runtime_legacy_root, valid_candidates) + ) + ): + self._prune_revisions_for_candidates_unlocked(valid_candidates) + return + except (OSError, TypeError, ValueError, json.JSONDecodeError): + pass + try: + self._promote_latest_active_state() + except OSError: + return + + def _revision_artifact_signatures( + self, + indexed_states: Sequence[_IndexedState], + ) -> dict[str, dict[str, dict[str, object]]]: + return { + indexed.candidate.run_id: self._artifact_signatures(indexed.candidate.state_path.parent) + for indexed in indexed_states + } + + def _artifact_signatures(self, root: Path) -> dict[str, dict[str, object]]: + return { + artifact_name: self._path_lstat_signature(root / artifact_name) + for artifact_name in self._RUN_ALIAS_ARTIFACT_NAMES + } + + def _reconcile_run_aliases_unlocked( + self, + indexed_states: Sequence[_IndexedState], + ) -> None: + for indexed in indexed_states: + candidate = indexed.candidate + if self._run_aliases_match_manifest(candidate): + continue + source_dir = candidate.state_path.parent + run_dir = self.run_dir_path(candidate.run_id) + for artifact_name in self._RUN_ALIAS_ARTIFACT_NAMES: + source = source_dir / artifact_name + alias = run_dir / artifact_name + if not source.is_file() or source.is_symlink(): + alias.unlink(missing_ok=True) + continue + self._write_text(alias, source.read_text(encoding="utf-8")) + self._write_run_alias_manifest(candidate) + + def _remove_current_aliases(self) -> None: + roots = [self.runtime_root] + if self.compat_mode == self.COMPAT_READ_WRITE and self.runtime_legacy_root != self.runtime_root: + roots.append(self.runtime_legacy_root) + for root in roots: + for name in ( + "run_state.json", + "runtime_map.json", + "ports_manifest.json", + "error_report.json", + "events.jsonl", + "runtime_readiness_report.json", + "current_alias.json", + ): + (root / name).unlink(missing_ok=True) + self._remove_owned_mode_pointers(root) + + @staticmethod + def _remove_pointers_to_state(root: Path, state_path: Path) -> None: + expected = str(state_path.resolve()) + for pointer in root.glob(".last_state*"): + if not pointer.is_file(): + continue + try: + target = pointer.read_text(encoding="utf-8").splitlines()[0].strip() + except (IndexError, OSError, UnicodeError): + continue + try: + resolved_target = str(Path(target).expanduser().resolve()) + except (OSError, RuntimeError, ValueError): + continue + if resolved_target == expected: + pointer.unlink(missing_ok=True) + + def _load_legacy_shell_candidate(self, path: Path, *, allowed_root: str) -> RunState | None: + if not path.is_file(): + return None + try: + return load_legacy_shell_state(str(path), allowed_root=allowed_root) + except Exception: + return None @staticmethod def _json_text(payload: Mapping[str, object]) -> str: @@ -365,17 +1518,75 @@ def _events_text(events: list[dict[str, object]]) -> str: return "".join(json.dumps(event, sort_keys=True) + "\n" for event in events) @staticmethod - def _write_run_and_current_artifact(*, run_path: Path, current_path: Path, text: str) -> None: - run_path.parent.mkdir(parents=True, exist_ok=True) - current_path.parent.mkdir(parents=True, exist_ok=True) - RuntimeStateRepository._write_text(run_path, text) - RuntimeStateRepository._write_text(current_path, text) + def _write_text(path: Path, text: str) -> None: + scavenge_atomic_write_temps(path) + atomic_write_text(path, text) + + @contextmanager + def _repository_lock(self, *, exclusive: bool) -> Iterator[None]: + self._validate_runtime_roots() + with advisory_file_lock(self._lock_path, exclusive=exclusive): + self._validate_runtime_roots() + yield + + def _validate_runtime_roots(self) -> None: + self._runtime_root_identity = self._validate_directory_identity( + self.runtime_root, + expected=self._runtime_root_identity, + create=True, + label="runtime_root", + ) + if self.compat_mode == self.SCOPED_ONLY: + return + self._legacy_root_identity = self._validate_directory_identity( + self.runtime_legacy_root, + expected=self._legacy_root_identity, + create=self.compat_mode == self.COMPAT_READ_WRITE, + label="runtime_legacy_root", + ) + + @staticmethod + def _existing_directory_identity(path: Path) -> tuple[int, int] | None: + if not path.exists(): + return None + if path.is_symlink() or not path.is_dir(): + raise ValueError(f"runtime root is not a real directory: {path}") + metadata = path.stat() + return metadata.st_dev, metadata.st_ino @staticmethod - def _write_text(path: Path, text: str) -> None: - _ = path.write_text(text, encoding="utf-8") + def _validate_directory_identity( + path: Path, + *, + expected: tuple[int, int] | None, + create: bool, + label: str, + ) -> tuple[int, int] | None: + if path.is_symlink() or path.resolve(strict=False) != path: + raise RuntimeError(f"{label} changed identity or became a symlink: {path}") + if create: + durable_mkdir(path) + if not path.exists(): + return expected + if not path.is_dir(): + raise RuntimeError(f"{label} is not a directory: {path}") + metadata = path.stat() + current = metadata.st_dev, metadata.st_ino + if expected is not None and current != expected: + raise RuntimeError(f"{label} changed identity: {path}") + return current def purge(self, *, aggressive: bool = False) -> None: + with self._repository_lock(exclusive=True): + self._purge_unlocked(aggressive=aggressive) + + def _purge_unlocked(self, *, aggressive: bool) -> None: + self._retry_pending_revision_cleanups_unlocked() + self.run_index.purge() + self._write_text( + self._registry_marker_path, + self._json_text({"version": 1, "runtime_scope_id": self.runtime_scope_id}), + ) scoped_paths = ( self.run_state_path(), self.runtime_map_path(), @@ -383,15 +1594,18 @@ def purge(self, *, aggressive: bool = False) -> None: self.error_report_path(), self.runtime_root / "events.jsonl", self.runtime_root / "runtime_readiness_report.json", - self.runtime_root / ".last_state", - self.runtime_root / ".last_state.main", + self._alias_manifest_path, ) for path in scoped_paths: path.unlink(missing_ok=True) - for pointer in self.runtime_root.glob(".last_state.trees.*"): - pointer.unlink(missing_ok=True) + diagnostics_dir = self.runtime_root / "diagnostics" + if diagnostics_dir.is_symlink(): + diagnostics_dir.unlink(missing_ok=True) + elif diagnostics_dir.is_dir(): + shutil.rmtree(diagnostics_dir) + self._remove_owned_mode_pointers(self.runtime_root) - if self.compat_mode != self.SCOPED_ONLY: + if self.compat_mode == self.COMPAT_READ_WRITE: legacy_paths = ( self.runtime_legacy_root / "run_state.json", self.runtime_legacy_root / "runtime_map.json", @@ -399,44 +1613,35 @@ def purge(self, *, aggressive: bool = False) -> None: self.runtime_legacy_root / "error_report.json", self.runtime_legacy_root / "events.jsonl", self.runtime_legacy_root / "runtime_readiness_report.json", - self.runtime_legacy_root / ".last_state", - self.runtime_legacy_root / ".last_state.main", ) for path in legacy_paths: path.unlink(missing_ok=True) - for pointer in self.runtime_legacy_root.glob(".last_state.trees.*"): - pointer.unlink(missing_ok=True) + self._remove_owned_mode_pointers(self.runtime_legacy_root) if aggressive: - for runs_dir in (self.runtime_root / "runs", self.runtime_legacy_root / "runs"): + runs_dirs = [self.runtime_root / "runs"] + if self.compat_mode == self.COMPAT_READ_WRITE and self.runtime_legacy_root != self.runtime_root: + runs_dirs.append(self.runtime_legacy_root / "runs") + for runs_dir in runs_dirs: if runs_dir.is_dir(): - shutil.rmtree(runs_dir, ignore_errors=True) - - def _ordered_scoped_pointers(self, *, mode: str | None) -> list[Path]: - main_pointer = self.runtime_root / ".last_state.main" - tree_pointers = sorted(self.runtime_root.glob(".last_state.trees.*")) - generic_pointer = self.runtime_root / ".last_state" - if mode == "trees": - return [*tree_pointers, generic_pointer, main_pointer] - if mode == "main": - return [main_pointer, generic_pointer, *tree_pointers] - return [main_pointer, *tree_pointers, generic_pointer] - - def _ordered_legacy_pointers(self, *, mode: str | None) -> list[Path]: - main_pointer = self.runtime_legacy_root / ".last_state.main" - tree_pointers = sorted(self.runtime_legacy_root.glob(".last_state.trees.*")) - generic_pointer = self.runtime_legacy_root / ".last_state" - if mode == "trees": - return [*tree_pointers, generic_pointer, main_pointer] - if mode == "main": - return [main_pointer, generic_pointer, *tree_pointers] - return [main_pointer, *tree_pointers, generic_pointer] - - def _state_matches_scope(self, state: RunState) -> bool: - scope = state.metadata.get("repo_scope_id") - if isinstance(scope, str) and scope: - return scope == self.runtime_scope_id - return True + shutil.rmtree(runs_dir) + cleanup_dir = self.runtime_root / self._REVISION_CLEANUP_DIR_NAME + if cleanup_dir.is_symlink(): + cleanup_dir.unlink(missing_ok=True) + elif cleanup_dir.is_dir(): + shutil.rmtree(cleanup_dir) + self._scavenge_atomic_temps() + + def _scavenge_atomic_temps(self) -> None: + roots = {self.runtime_root} + if self.compat_mode == self.COMPAT_READ_WRITE: + roots.add(self.runtime_legacy_root) + for root in roots: + if not root.is_dir(): + continue + for temporary_path in root.rglob(".*.tmp"): + if temporary_path.is_file() and not temporary_path.is_symlink(): + temporary_path.unlink(missing_ok=True) def _context_ports_payload(self, context: object) -> dict[str, object]: ports = getattr(context, "ports", {}) @@ -457,39 +1662,6 @@ def _context_ports_payload(self, context: object) -> dict[str, object]: "ports": payload_ports, } - def _write_scoped_pointers(self, *, run_state_path: Path, state: RunState, contexts: list[object]) -> None: - self._write_mode_pointers( - root=self.runtime_root, - run_state_path=run_state_path, - mode=state.mode, - project_names=self._context_names(contexts), - ) - - def _write_legacy_compat( - self, - *, - run_state_path: Path, - run_state_text: str, - runtime_map_text: str, - ports_manifest_text: str, - error_report_text: str, - events_text: str, - state: RunState, - contexts: list[object], - ) -> None: - self.runtime_legacy_root.mkdir(parents=True, exist_ok=True) - self._write_text(self.runtime_legacy_root / "run_state.json", run_state_text) - self._write_text(self.runtime_legacy_root / "runtime_map.json", runtime_map_text) - self._write_text(self.runtime_legacy_root / "ports_manifest.json", ports_manifest_text) - self._write_text(self.runtime_legacy_root / "error_report.json", error_report_text) - self._write_text(self.runtime_legacy_root / "events.jsonl", events_text) - self._write_mode_pointers( - root=self.runtime_legacy_root, - run_state_path=run_state_path, - mode=state.mode, - project_names=self._context_names(contexts), - ) - @staticmethod def _context_names(contexts: list[object]) -> list[str]: names: set[str] = set() @@ -499,28 +1671,115 @@ def _context_names(contexts: list[object]) -> list[str]: names.add(name) return sorted(names, key=str.lower) - def _project_names_from_state(self, state: RunState) -> list[str]: + def _prepare_state_for_save( + self, + state: RunState, + *, + project_names: Sequence[str], + contexts: Sequence[object] = (), + authoritative_projects: bool = False, + ) -> None: + _ = self.run_dir_path(state.run_id) + if state.mode not in {"main", "trees"}: + raise ValueError("state mode must be 'main' or 'trees'") + existing_scope = state.metadata.get("repo_scope_id") + if isinstance(existing_scope, str) and existing_scope and existing_scope != self.runtime_scope_id: + raise ValueError("state repo_scope_id does not match this repository") + state.metadata["repo_scope_id"] = self.runtime_scope_id + + canonical_names = sorted( + {str(name).strip() for name in project_names if str(name).strip()}, + key=str.casefold, + ) + if canonical_names or authoritative_projects: + state.metadata["project_names"] = canonical_names + + existing_roots = state.metadata.get("project_roots") + project_roots = dict(existing_roots) if isinstance(existing_roots, Mapping) else {} + for context in contexts: + name = str(getattr(context, "name", "")).strip() + root = getattr(context, "root", None) + if name and root is not None and str(root).strip(): + project_roots[name] = str(Path(str(root)).expanduser().resolve(strict=False)) + if authoritative_projects: + normalized_names = normalized_project_names(canonical_names) + project_roots = { + str(name): root + for name, root in project_roots.items() + if str(name).strip().casefold() in normalized_names + } + if project_roots: + state.metadata["project_roots"] = project_roots + elif authoritative_projects: + state.metadata.pop("project_roots", None) + if canonical_names or authoritative_projects: + state.metadata = filter_project_scoped_metadata( + state.metadata, + canonical_names, + case_sensitive=False, + ) + state.metadata["project_names"] = canonical_names + + def _runtime_project_names_from_state(self, state: RunState) -> list[str]: names: set[str] = set() for project in state.requirements: normalized = str(project).strip() if normalized: names.add(normalized) - for service_name in state.services: - project_name = self._project_name_from_service_name(service_name) + for service_name, service in state.services.items(): + project_name = self._service_project_name(service_name, service) if project_name: names.add(project_name) + non_main_names = {name for name in names if name.casefold() != "main"} + if state.mode == "trees" and non_main_names: + names = non_main_names if state.mode == "main": names.add("Main") - return sorted(names, key=str.lower) + if names: + return sorted(names, key=str.casefold) + if bool(state.metadata.get("dashboard_runs_disabled")) or bool( + state.metadata.get(DASHBOARD_STOPPED_SERVICES_KEY) + ): + return self._project_names_from_state(state) + return [] @staticmethod - def _text_fingerprint(text: str) -> str: - return hashlib.sha256(text.encode("utf-8")).hexdigest() + def _source_run_ids(state: RunState) -> list[str]: + raw = state.metadata.get("state_source_run_ids") + values = raw if isinstance(raw, list) else [] + return [str(run_id).strip() for run_id in values if str(run_id).strip()] - @staticmethod - def _state_fingerprint(state: RunState) -> str: - serialized = json.dumps(state_to_dict(state), sort_keys=True) - return RuntimeStateRepository._text_fingerprint(serialized) + def _project_names_from_state(self, state: RunState) -> list[str]: + names: set[str] = set() + metadata_names = state.metadata.get("project_names") + if isinstance(metadata_names, Sequence) and not isinstance(metadata_names, (str, bytes)): + for project in metadata_names: + normalized = str(project).strip() + if normalized: + names.add(normalized) + metadata_roots = state.metadata.get("project_roots") + if isinstance(metadata_roots, Mapping): + for project in metadata_roots: + normalized = str(project).strip() + if normalized: + names.add(normalized) + if names: + return sorted(names, key=str.casefold) + for project in state.requirements: + normalized = str(project).strip() + if normalized: + names.add(normalized) + for service_name, service in state.services.items(): + project_name = str(getattr(service, "project", "") or "").strip() + if not project_name: + project_name = self._project_name_from_service_name(service_name) or "" + if project_name: + names.add(project_name) + if state.mode == "trees" and len(names) > 1: + names.discard("Main") + if state.mode == "main": + names.add("Main") + return sorted(names, key=str.lower) @staticmethod def _project_name_from_service_name(service_name: str) -> str | None: @@ -536,39 +1795,149 @@ def _project_name_from_service_name(service_name: str) -> str | None: return project_name or None return None - def _runtime_update_run_state_path(self, state: RunState) -> Path: - run_id = str(getattr(state, "run_id", "")).strip() - if not run_id: - return self.run_state_path() - return self.run_dir_path(run_id) / "run_state.json" - - def _write_mode_pointers( + def _service_project_name(self, service_name: str, service: ServiceRecord) -> str: + explicit_project = str(getattr(service, "project", "") or "").strip() + if explicit_project: + return explicit_project + return self._project_name_from_service_name(service_name) or "" + + def _contained_child(self, root: Path, component: str, *, label: str) -> Path: + runtime_root = self.runtime_root.resolve() + if root.is_symlink(): + raise ValueError(f"{label} root must not be a symlink") + resolved_root = root.resolve() + try: + resolved_root.relative_to(runtime_root) + except ValueError as exc: + raise ValueError(f"{label} root escapes runtime_root") from exc + candidate = root / component + if candidate.is_symlink(): + raise ValueError(f"{label} must not be a symlink") + if candidate.resolve().parent != resolved_root: + raise ValueError(f"{label} escapes its expected directory") + return candidate + + def _rebuild_mode_pointers( self, - *, root: Path, - run_state_path: Path, - mode: str, - project_names: list[str], + indexed_states: Sequence[_IndexedState], ) -> None: - root.mkdir(parents=True, exist_ok=True) - pointer_text = str(run_state_path) + "\n" - self._write_text(root / ".last_state", pointer_text) - existing_tree_pointers = sorted(root.glob(".last_state.trees.*")) - if mode == "main": - self._write_text(root / ".last_state.main", pointer_text) - return + desired = self._desired_mode_pointers(indexed_states) + stale_names = self._owned_pointer_names(root).difference(desired) + for name, target in desired.items(): + self._write_text(root / name, target + "\n") + for name in stale_names: + (root / name).unlink(missing_ok=True) + self._write_text( + self._pointer_manifest_path(root), + self._json_text( + { + "version": 1, + "runtime_scope_id": self.runtime_scope_id, + "pointers": desired, + } + ), + ) - expected_tree_pointer_names: set[str] = set() - for project_name in project_names: - pointer_name = self._tree_pointer_name(project_name) - if pointer_name is None: + def _desired_mode_pointers( + self, + indexed_states: Sequence[_IndexedState], + ) -> dict[str, str]: + if not indexed_states: + return {} + latest_active = max( + indexed_states, + key=lambda indexed: indexed.candidate.activation_sequence, + ) + desired = {".last_state": str(latest_active.candidate.state_path)} + owned_tree_projects: set[str] = set() + latest_main = max( + (indexed for indexed in indexed_states if indexed.candidate.mode == "main"), + key=lambda indexed: indexed.candidate.activation_sequence, + default=None, + ) + if latest_main is not None: + desired[".last_state.main"] = str(latest_main.candidate.state_path) + for indexed in indexed_states: + candidate = indexed.candidate + target = str(candidate.state_path) + if candidate.mode != "trees": continue - expected_tree_pointer_names.add(pointer_name) - self._write_text(root / pointer_name, pointer_text) + display_names = {name.casefold(): name for name in self._project_names_from_state(indexed.state)} + for project_key in candidate.project_names: + if project_key in owned_tree_projects: + continue + owned_tree_projects.add(project_key) + display_name = display_names.get(project_key, project_key) + pointer_name = self._tree_pointer_name(display_name) + if pointer_name is None: + continue + if pointer_name in desired and desired[pointer_name] != target: + digest = text_fingerprint(display_name)[:10] + pointer_name = f"{pointer_name}-{digest}" + desired[pointer_name] = target + return desired + + def _mode_pointers_match( + self, + root: Path, + indexed_states: Sequence[_IndexedState], + ) -> bool: + desired = self._desired_mode_pointers(indexed_states) + try: + payload = json.loads(self._pointer_manifest_path(root).read_text(encoding="utf-8")) + if not isinstance(payload, Mapping) or payload.get("pointers") != desired: + return False + if self._owned_pointer_names(root) != set(desired): + return False + return all((root / name).read_text(encoding="utf-8") == target + "\n" for name, target in desired.items()) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return False + + def _remove_owned_mode_pointers(self, root: Path) -> None: + for name in self._owned_pointer_names(root): + (root / name).unlink(missing_ok=True) + self._pointer_manifest_path(root).unlink(missing_ok=True) + + def _owned_pointer_names(self, root: Path) -> set[str]: + names: set[str] = set() + try: + payload = json.loads(self._pointer_manifest_path(root).read_text(encoding="utf-8")) + raw_pointers = payload.get("pointers") if isinstance(payload, Mapping) else None + if isinstance(raw_pointers, Mapping): + for raw_name in raw_pointers: + name = str(raw_name) + if name not in {".last_state", ".last_state.main"} and not name.startswith(".last_state.trees."): + continue + if root == self.runtime_root or self._pointer_targets_runtime_root(root / name): + names.add(name) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + pass + for pointer in root.glob(".last_state*"): + if pointer.name not in {".last_state", ".last_state.main"} and not pointer.name.startswith( + ".last_state.trees." + ): + continue + if root == self.runtime_root or self._pointer_targets_runtime_root(pointer): + names.add(pointer.name) + return names - for pointer in existing_tree_pointers: - if pointer.name not in expected_tree_pointer_names: - pointer.unlink(missing_ok=True) + def _pointer_targets_runtime_root(self, pointer: Path) -> bool: + if pointer.is_symlink() or not pointer.is_file(): + return False + try: + target = Path(pointer.read_text(encoding="utf-8").splitlines()[0].strip()).resolve() + target.relative_to(self.runtime_root) + except (IndexError, OSError, RuntimeError, ValueError): + return False + return True + + def _pointer_manifest_path(self, root: Path) -> Path: + if root == self.runtime_root: + return root / "pointer_manifest.json" + scope_stem = safe_artifact_stem(self.runtime_scope_id, fallback="scope") + scope_hash = text_fingerprint(self.runtime_scope_id)[:10] + return root / f".pointer_manifest.{scope_stem}-{scope_hash}.json" @staticmethod def _tree_pointer_name(project_name: str) -> str | None: @@ -576,4 +1945,7 @@ def _tree_pointer_name(project_name: str) -> str | None: if not normalized_project or normalized_project.lower() == "main": return None pointer_suffix = safe_artifact_stem(normalized_project, fallback="project") + if pointer_suffix != normalized_project: + digest = text_fingerprint(normalized_project)[:10] + pointer_suffix = f"{pointer_suffix}-{digest}" return f".last_state.trees.{pointer_suffix}" diff --git a/python/envctl_engine/state/run_index.py b/python/envctl_engine/state/run_index.py new file mode 100644 index 000000000..b5ae0e3f6 --- /dev/null +++ b/python/envctl_engine/state/run_index.py @@ -0,0 +1,582 @@ +from __future__ import annotations + +import json +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Any + +from envctl_engine.state.fingerprints import file_fingerprint +from envctl_engine.state.persistence import ( + advisory_file_lock, + atomic_write_text, + durable_mkdir, + require_path_component, + scavenge_atomic_write_temps, +) + + +_INDEX_VERSION = 4 + + +def _normalize_mode(mode: str | None) -> str | None: + normalized = str(mode or "").strip().casefold() + return normalized or None + + +def _normalize_project_names(project_names: Sequence[str]) -> tuple[str, ...]: + return tuple(sorted({name.strip().casefold() for name in project_names if name.strip()})) + + +@dataclass(frozen=True, slots=True) +class StateSelector: + mode: str | None + project_names: tuple[str, ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "mode", _normalize_mode(self.mode)) + object.__setattr__(self, "project_names", _normalize_project_names(self.project_names)) + + +@dataclass(frozen=True, slots=True) +class RunIndexCandidate: + state_path: Path + run_id: str + mode: str + project_names: tuple[str, ...] + sequence: int + state_fingerprint: str = "" + runtime_map_fingerprint: str | None = None + activation_sequence: int = 0 + + +@dataclass(frozen=True, slots=True) +class _RegistrySnapshot: + generation: int + entries: tuple[RunIndexCandidate, ...] + retired_run_ids: frozenset[str] + + +class RunIndex: + """Selection-aware registry of active project ownership by run artifact.""" + + def __init__(self, *, runtime_root: Path, runtime_dir: Path, runtime_scope_id: str) -> None: + self._input_runtime_root = runtime_root.absolute() + if self._input_runtime_root.is_symlink(): + raise ValueError("runtime_root must not be a symlink") + self.runtime_dir = runtime_dir.resolve() + self.runtime_root = runtime_root.resolve() + self.runtime_scope_id = runtime_scope_id.strip() + if not self.runtime_scope_id: + raise ValueError("runtime_scope_id must not be empty") + if not self._is_within_runtime_dir(self.runtime_root): + raise ValueError("runtime_root must be contained by runtime_dir") + self.index_path = self.runtime_root / "run_index.json" + self.backup_path = self.runtime_root / "run_index.backup.json" + self._lock_path = self.runtime_root / ".run_index.lock" + self._runtime_root_identity = self._existing_directory_identity() + + def record( + self, + *, + state_path: Path, + run_id: str, + mode: str, + project_names: Sequence[str], + supersede_run_ids: Sequence[str] = (), + ) -> None: + normalized_run_id = require_path_component(run_id, label="run_id") + resolved_state_path = self._validated_revision_state_path(state_path, run_id=normalized_run_id) + normalized_mode = _normalize_mode(mode) + if normalized_mode is None: + raise ValueError("mode must not be empty") + normalized_projects = _normalize_project_names(project_names) + state_fingerprint = file_fingerprint(resolved_state_path) + runtime_map_path = resolved_state_path.parent / "runtime_map.json" + runtime_map_fingerprint = ( + file_fingerprint(runtime_map_path) + if runtime_map_path.is_file() and not runtime_map_path.is_symlink() + else None + ) + superseded = { + require_path_component(candidate_run_id, label="superseded run_id") + for candidate_run_id in supersede_run_ids + } + superseded.discard(normalized_run_id) + + with self._lock(exclusive=True): + snapshot = self._read_registry() + entries = list(snapshot.entries) if snapshot is not None else [] + retired_run_ids = set(snapshot.retired_run_ids) if snapshot is not None else set() + if normalized_run_id in retired_run_ids: + raise RuntimeError(f"run_id is retired and cannot publish new state: {normalized_run_id}") + latest_sequence = max((entry.sequence for entry in entries), default=0) + latest_activation_sequence = max( + (entry.activation_sequence for entry in entries), + default=0, + ) + existing_entry = next((entry for entry in entries if entry.run_id == normalized_run_id), None) + if not normalized_projects and existing_entry is not None: + normalized_projects = existing_entry.project_names + entries = [ + entry for entry in entries if entry.run_id != normalized_run_id and entry.run_id not in superseded + ] + retired_run_ids.update(superseded) + entries = self._subtract_replaced_project_owners( + entries, + mode=normalized_mode, + project_names=normalized_projects, + retired_run_ids=retired_run_ids, + ) + sequence = existing_entry.sequence if existing_entry is not None else latest_sequence + 1 + activates_current_mode = existing_entry is None or self._entry_owns_any_project( + existing_entry, + entries, + normalized_projects, + ) + activation_sequence = ( + latest_activation_sequence + 1 if activates_current_mode else existing_entry.activation_sequence + ) + entries.append( + RunIndexCandidate( + state_path=resolved_state_path, + run_id=normalized_run_id, + mode=normalized_mode, + project_names=normalized_projects, + sequence=sequence, + state_fingerprint=state_fingerprint, + runtime_map_fingerprint=runtime_map_fingerprint, + activation_sequence=activation_sequence, + ) + ) + self._write_entries(entries, retired_run_ids=retired_run_ids) + + @staticmethod + def _subtract_replaced_project_owners( + entries: Sequence[RunIndexCandidate], + *, + mode: str, + project_names: Sequence[str], + retired_run_ids: set[str], + ) -> list[RunIndexCandidate]: + replaced_projects = frozenset(project_names) + if not replaced_projects: + return list(entries) + + retained: list[RunIndexCandidate] = [] + for entry in entries: + if entry.mode != mode or replaced_projects.isdisjoint(entry.project_names): + retained.append(entry) + continue + remaining_projects = tuple(project for project in entry.project_names if project not in replaced_projects) + if remaining_projects: + retained.append(replace(entry, project_names=remaining_projects)) + else: + retired_run_ids.add(entry.run_id) + return retained + + def remove(self, run_id: str) -> bool: + return self.remove_many([run_id]) + + def remove_many(self, run_ids: Sequence[str]) -> bool: + normalized_run_ids = {require_path_component(run_id, label="run_id") for run_id in run_ids} + with self._lock(exclusive=True): + snapshot = self._read_registry() + entries = list(snapshot.entries) if snapshot is not None else [] + retired_run_ids = set(snapshot.retired_run_ids) if snapshot is not None else set() + retained = [entry for entry in entries if entry.run_id not in normalized_run_ids] + removed_active_entry = len(retained) != len(entries) + added_tombstone = not normalized_run_ids.issubset(retired_run_ids) + if not removed_active_entry and not added_tombstone: + return False + retired_run_ids.update(normalized_run_ids) + self._write_entries(retained, retired_run_ids=retired_run_ids) + return removed_active_entry + + def candidates(self, selector: StateSelector) -> list[RunIndexCandidate]: + with self._lock(exclusive=False): + entries = self._read_entries() + if entries is None: + return [] + + selected_projects = frozenset(selector.project_names) + ranked: list[tuple[int, int, str, RunIndexCandidate]] = [] + for entry in entries: + if selector.mode is not None and entry.mode != selector.mode: + continue + relation_rank = self._selection_rank( + indexed=frozenset(entry.project_names), + selected=selected_projects, + ) + if relation_rank is None: + continue + ranked.append((-entry.sequence, relation_rank, str(entry.state_path), entry)) + ranked.sort(key=lambda item: (item[0], item[1], item[2])) + return [entry for _, _, _, entry in ranked] + + def candidate_paths(self, selector: StateSelector) -> list[Path]: + return [candidate.state_path for candidate in self.candidates(selector)] + + def purge(self) -> None: + with self._lock(exclusive=True): + snapshot = self._read_registry() + retired_run_ids = set(snapshot.retired_run_ids) if snapshot is not None else set() + if snapshot is not None: + retired_run_ids.update(entry.run_id for entry in snapshot.entries) + self._write_entries([], retired_run_ids=retired_run_ids) + + def needs_rebuild(self) -> bool: + with self._lock(exclusive=False): + return self._read_snapshot(self.index_path) is None and self._read_snapshot(self.backup_path) is None + + def repair_copies(self) -> None: + with self._lock(exclusive=True): + primary = self._read_snapshot(self.index_path) + backup = self._read_snapshot(self.backup_path) + if primary is not None and backup is not None and primary == backup: + return + snapshot = self._read_registry() + if snapshot is not None: + self._write_entries( + list(snapshot.entries), + retired_run_ids=set(snapshot.retired_run_ids), + ) + + def initialize_empty(self) -> None: + with self._lock(exclusive=True): + self._write_entries([], retired_run_ids=set()) + + def replace_all(self, entries: Sequence[RunIndexCandidate]) -> None: + with self._lock(exclusive=True): + self._write_entries( + [self._candidate_with_fingerprints(entry) for entry in entries], + retired_run_ids=set(), + ) + + @staticmethod + def _selection_rank(*, indexed: frozenset[str], selected: frozenset[str]) -> int | None: + if not selected: + return 0 + if not indexed: + return None + if indexed == selected: + return 0 + if indexed.issuperset(selected): + return 1 + if indexed.issubset(selected): + return 2 + if indexed.intersection(selected): + return 3 + return None + + @staticmethod + def _entry_owns_any_project( + existing_entry: RunIndexCandidate, + other_entries: Sequence[RunIndexCandidate], + updated_projects: Sequence[str], + ) -> bool: + if not updated_projects: + return True + for project in updated_projects: + if not any( + other.mode == existing_entry.mode + and other.sequence > existing_entry.sequence + and project in other.project_names + for other in other_entries + ): + return True + return False + + @contextmanager + def _lock(self, *, exclusive: bool) -> Iterator[None]: + self._validate_runtime_root() + with advisory_file_lock(self._lock_path, exclusive=exclusive): + self._validate_runtime_root() + yield + + def _existing_directory_identity(self) -> tuple[int, int] | None: + if not self.runtime_root.exists(): + return None + if self.runtime_root.is_symlink() or not self.runtime_root.is_dir(): + raise ValueError(f"runtime_root is not a real directory: {self.runtime_root}") + metadata = self.runtime_root.stat() + return metadata.st_dev, metadata.st_ino + + def _validate_runtime_root(self) -> None: + if self.runtime_root.is_symlink() or self.runtime_root.resolve(strict=False) != self.runtime_root: + raise RuntimeError(f"runtime_root changed identity or became a symlink: {self.runtime_root}") + durable_mkdir(self.runtime_root) + metadata = self.runtime_root.stat() + current = metadata.st_dev, metadata.st_ino + if self._runtime_root_identity is not None and current != self._runtime_root_identity: + raise RuntimeError(f"runtime_root changed identity: {self.runtime_root}") + self._runtime_root_identity = current + + def _read_entries(self) -> list[RunIndexCandidate] | None: + snapshot = self._read_registry() + return list(snapshot.entries) if snapshot is not None else None + + def _read_registry(self) -> _RegistrySnapshot | None: + primary = self._read_snapshot(self.index_path) + backup = self._read_snapshot(self.backup_path) + self._reject_equal_generation_divergence(primary, backup) + snapshots = [snapshot for snapshot in (primary, backup) if snapshot is not None] + if not snapshots: + return None + return max(snapshots, key=lambda snapshot: snapshot.generation) + + @staticmethod + def _reject_equal_generation_divergence( + primary: _RegistrySnapshot | None, + backup: _RegistrySnapshot | None, + ) -> None: + if primary is not None and backup is not None and primary.generation == backup.generation and primary != backup: + raise RuntimeError( + "Run registry primary and backup diverged at generation " + f"{primary.generation}; refusing to choose one copy" + ) + + def _read_snapshot(self, path: Path) -> _RegistrySnapshot | None: + if not path.is_file(): + return None + try: + payload = json.loads(path.read_text(encoding="utf-8")) + generation = payload.get("generation") if isinstance(payload, dict) else None + if not isinstance(generation, int) or isinstance(generation, bool) or generation < 1: + return None + entries = self._parse_payload(payload) + raw_retired = payload.get("retired_run_ids") + if not isinstance(raw_retired, list) or not all(isinstance(run_id, str) for run_id in raw_retired): + return None + retired = frozenset(require_path_component(run_id, label="retired run_id") for run_id in raw_retired) + if retired.intersection(entry.run_id for entry in entries): + return None + return _RegistrySnapshot(generation, tuple(entries), retired) + except (OSError, RuntimeError, TypeError, ValueError, json.JSONDecodeError): + return None + + def _parse_payload(self, payload: Any) -> list[RunIndexCandidate]: + if not isinstance(payload, dict): + raise ValueError("run index must be a JSON object") + version = payload.get("version") + if version not in {2, 3, _INDEX_VERSION}: + raise ValueError("unsupported run index version") + if payload.get("runtime_scope_id") != self.runtime_scope_id: + raise ValueError("run index scope does not match this repository") + raw_entries = payload.get("entries") + if not isinstance(raw_entries, list): + raise ValueError("run index entries must be a list") + + entries: list[RunIndexCandidate] = [] + seen_run_ids: set[str] = set() + for raw_entry in raw_entries: + if not isinstance(raw_entry, dict) or raw_entry.get("runtime_scope_id") != self.runtime_scope_id: + raise ValueError("invalid run index entry scope") + run_id = raw_entry.get("run_id") + mode = _normalize_mode(raw_entry.get("mode") if isinstance(raw_entry.get("mode"), str) else None) + raw_projects = raw_entry.get("project_names") + sequence = raw_entry.get("sequence") + raw_state_path = raw_entry.get("state_path") + raw_state_fingerprint = raw_entry.get("state_fingerprint") + raw_runtime_map_fingerprint = raw_entry.get("runtime_map_fingerprint") + raw_activation_sequence = raw_entry.get("activation_sequence") + if ( + not isinstance(run_id, str) + or not run_id.strip() + or mode is None + or not isinstance(raw_projects, list) + or not all(isinstance(project, str) for project in raw_projects) + or not isinstance(sequence, int) + or isinstance(sequence, bool) + or sequence < 1 + or not isinstance(raw_state_path, str) + or not raw_state_path.strip() + ): + raise ValueError("invalid run index entry") + normalized_run_id = require_path_component(run_id, label="indexed run_id") + if normalized_run_id in seen_run_ids: + raise ValueError("duplicate run id in run index") + state_path = self._validated_revision_state_path(Path(raw_state_path), run_id=normalized_run_id) + if version == 2: + state_fingerprint = file_fingerprint(state_path) + runtime_map_path = state_path.parent / "runtime_map.json" + runtime_map_fingerprint = ( + file_fingerprint(runtime_map_path) + if runtime_map_path.is_file() and not runtime_map_path.is_symlink() + else None + ) + else: + if not self._valid_fingerprint(raw_state_fingerprint) or ( + raw_runtime_map_fingerprint is not None and not self._valid_fingerprint(raw_runtime_map_fingerprint) + ): + raise ValueError("invalid run index artifact fingerprint") + state_fingerprint = str(raw_state_fingerprint) + runtime_map_fingerprint = ( + str(raw_runtime_map_fingerprint) if raw_runtime_map_fingerprint is not None else None + ) + if version in {2, 3}: + activation_sequence = sequence + elif ( + not isinstance(raw_activation_sequence, int) + or isinstance(raw_activation_sequence, bool) + or raw_activation_sequence < 1 + ): + raise ValueError("invalid run index activation sequence") + else: + activation_sequence = raw_activation_sequence + seen_run_ids.add(normalized_run_id) + entries.append( + RunIndexCandidate( + state_path=state_path, + run_id=normalized_run_id, + mode=mode, + project_names=_normalize_project_names(raw_projects), + sequence=sequence, + state_fingerprint=state_fingerprint, + runtime_map_fingerprint=runtime_map_fingerprint, + activation_sequence=activation_sequence, + ) + ) + return entries + + def _write_entries( + self, + entries: list[RunIndexCandidate], + *, + retired_run_ids: set[str], + ) -> None: + primary = self._read_snapshot(self.index_path) + backup = self._read_snapshot(self.backup_path) + self._reject_equal_generation_divergence(primary, backup) + existing_generations = [snapshot.generation for snapshot in (primary, backup) if snapshot is not None] + payload = { + "version": _INDEX_VERSION, + "generation": max(existing_generations, default=0) + 1, + "runtime_scope_id": self.runtime_scope_id, + "retired_run_ids": sorted(retired_run_ids), + "entries": [ + { + "state_path": str(entry.state_path), + "run_id": entry.run_id, + "mode": entry.mode, + "project_names": list(entry.project_names), + "sequence": entry.sequence, + "state_fingerprint": entry.state_fingerprint, + "runtime_map_fingerprint": entry.runtime_map_fingerprint, + "activation_sequence": entry.activation_sequence, + "runtime_scope_id": self.runtime_scope_id, + } + for entry in entries + ], + } + text = json.dumps(payload, indent=2, sort_keys=True) + "\n" + errors: list[OSError] = [] + successful_writes = 0 + for path in (self.backup_path, self.index_path): + try: + scavenge_atomic_write_temps(path) + atomic_write_text(path, text) + successful_writes += 1 + except OSError as exc: + errors.append(exc) + expected = _RegistrySnapshot( + generation=payload["generation"], + entries=tuple(entries), + retired_run_ids=frozenset(retired_run_ids), + ) + if any(self._read_snapshot(path) == expected for path in (self.index_path, self.backup_path)): + return + if errors: + raise errors[0] + if successful_writes: + raise RuntimeError("run registry write returned without publishing the expected generation") + + def _validated_revision_state_path(self, state_path: Path, *, run_id: str) -> Path: + if not state_path.is_absolute(): + raise ValueError("indexed state_path must be absolute") + resolved = state_path.resolve() + try: + relative = resolved.relative_to(self.runtime_root) + except ValueError as exc: + raise ValueError("indexed state_path escapes runtime_root") from exc + parts = relative.parts + if len(parts) != 5 or parts[0] != "runs" or parts[1] != run_id or parts[2] != "revisions": + raise ValueError("indexed state_path must identify a run revision") + _ = require_path_component(parts[3], label="revision id") + if parts[4] != "run_state.json": + raise ValueError("indexed state_path must identify run_state.json") + if ".." in state_path.parts or state_path.is_symlink(): + raise ValueError("indexed state_path must be canonical and must not traverse symlinks") + lexical_relative: Path | None = None + lexical_root: Path | None = None + for candidate_root in (self._input_runtime_root, self.runtime_root): + try: + lexical_relative = state_path.relative_to(candidate_root) + lexical_root = candidate_root + break + except ValueError: + continue + if lexical_relative is None or lexical_root is None: + raise ValueError("indexed state_path must be canonical and must not traverse symlinks") + lexical_parent = lexical_root + for component in lexical_relative.parts[:-1]: + lexical_parent /= component + if lexical_parent.is_symlink(): + raise ValueError("indexed state_path must be canonical and must not traverse symlinks") + for parent in ( + self.runtime_root / "runs", + self.runtime_root / "runs" / run_id, + self.runtime_root / "runs" / run_id / "revisions", + resolved.parent, + ): + if parent.is_symlink(): + raise ValueError("indexed state_path must not traverse symlinks") + if not self._is_within_runtime_root(resolved): + raise ValueError("indexed state_path must be canonical inside runtime_root") + return resolved + + def _candidate_with_fingerprints(self, entry: RunIndexCandidate) -> RunIndexCandidate: + state_path = self._validated_revision_state_path(entry.state_path, run_id=entry.run_id) + runtime_map_path = state_path.parent / "runtime_map.json" + return RunIndexCandidate( + state_path=state_path, + run_id=entry.run_id, + mode=entry.mode, + project_names=entry.project_names, + sequence=entry.sequence, + state_fingerprint=entry.state_fingerprint or file_fingerprint(state_path), + runtime_map_fingerprint=( + entry.runtime_map_fingerprint + if entry.runtime_map_fingerprint is not None + else ( + file_fingerprint(runtime_map_path) + if runtime_map_path.is_file() and not runtime_map_path.is_symlink() + else None + ) + ), + activation_sequence=entry.activation_sequence or entry.sequence, + ) + + @staticmethod + def _valid_fingerprint(value: object) -> bool: + return ( + isinstance(value, str) and len(value) == 64 and all(character in "0123456789abcdef" for character in value) + ) + + def _is_within_runtime_dir(self, path: Path) -> bool: + try: + path.relative_to(self.runtime_dir) + except ValueError: + return False + return True + + def _is_within_runtime_root(self, path: Path) -> bool: + try: + path.relative_to(self.runtime_root) + except ValueError: + return False + return True + + +__all__ = ["RunIndex", "RunIndexCandidate", "StateSelector"] diff --git a/python/envctl_engine/state/state_aggregation.py b/python/envctl_engine/state/state_aggregation.py new file mode 100644 index 000000000..bf63e9904 --- /dev/null +++ b/python/envctl_engine/state/state_aggregation.py @@ -0,0 +1,322 @@ +from __future__ import annotations + +import json +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass + +from envctl_engine.dashboard_metadata import DASHBOARD_STOPPED_SERVICES_KEY +from envctl_engine.state.fingerprints import text_fingerprint +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord +from envctl_engine.state.project_runtime import ( + PROJECT_SCOPED_METADATA_MAP_KEYS, + filter_project_scoped_metadata, +) +from envctl_engine.state.run_index import RunIndexCandidate + + +ServiceProjectResolver = Callable[[str, ServiceRecord], str] +ProjectNamesResolver = Callable[[RunState], Sequence[str]] +SourceRunIdsResolver = Callable[[RunState], Sequence[str]] + + +@dataclass(frozen=True, slots=True) +class IndexedState: + candidate: RunIndexCandidate + state: RunState + + +def normalized_project_names(project_names: Sequence[str]) -> frozenset[str]: + return frozenset(str(name).strip().casefold() for name in project_names if str(name).strip()) + + +def select_indexed_owners( + indexed_states: Sequence[IndexedState], + *, + selected_projects: Sequence[str], + service_project_name: ServiceProjectResolver, +) -> list[IndexedState]: + if not indexed_states: + return [] + owner_by_project: dict[str, str] = {} + for indexed_state in indexed_states: + for project_name in indexed_state.candidate.project_names: + owner_by_project.setdefault(project_name, indexed_state.candidate.run_id) + if not owner_by_project and not normalized_project_names(selected_projects): + return [indexed_states[0]] + + requested = normalized_project_names(selected_projects) or frozenset(owner_by_project) + chosen_run_ids = {owner_by_project[project_name] for project_name in requested if project_name in owner_by_project} + if not chosen_run_ids: + return [] + + chosen_states: list[IndexedState] = [] + for indexed_state in indexed_states: + run_id = indexed_state.candidate.run_id + if run_id not in chosen_run_ids: + continue + owned_projects = frozenset( + project_name for project_name, owner_run_id in owner_by_project.items() if owner_run_id == run_id + ) + filtered = filter_state_to_owned_projects( + indexed_state.state, + owned_projects, + service_project_name=service_project_name, + ) + chosen_states.append(IndexedState(candidate=indexed_state.candidate, state=filtered)) + return chosen_states + + +def state_from_indexed_owners( + chosen_states: Sequence[IndexedState], + *, + project_names_from_state: ProjectNamesResolver, + source_run_ids: SourceRunIdsResolver, +) -> RunState | None: + if not chosen_states: + return None + if len(chosen_states) == 1: + chosen = chosen_states[0] + if normalized_project_names(chosen.candidate.project_names) == normalized_project_names( + project_names_from_state(chosen.state) + ): + return chosen.state + return merge_indexed_states( + chosen_states, + project_names_from_state=project_names_from_state, + source_run_ids=source_run_ids, + ) + + +def filter_state_to_owned_projects( + state: RunState, + indexed_projects: frozenset[str], + *, + service_project_name: ServiceProjectResolver, +) -> RunState: + if not indexed_projects: + return state + + services = { + name: service + for name, service in state.services.items() + if service_project_name(name, service).casefold() in indexed_projects + } + requirements = { + name: requirement + for name, requirement in state.requirements.items() + if str(name).strip().casefold() in indexed_projects + or ( + state.mode == "trees" + and str(name).strip().casefold() == "main" + and bool(state.metadata.get("shared_dependencies")) + ) + } + metadata = filter_project_scoped_metadata( + state.metadata, + indexed_projects, + case_sensitive=False, + ) + metadata["project_names"] = _display_project_names( + state, + indexed_projects=indexed_projects, + service_project_name=service_project_name, + ) + filter_startup_identity_projects(metadata, indexed_projects) + return RunState( + run_id=state.run_id, + mode=state.mode, + schema_version=state.schema_version, + backend_mode=state.backend_mode, + services=services, + requirements=requirements, + pointers=dict(state.pointers), + metadata=metadata, + ) + + +def _display_project_names( + state: RunState, + *, + indexed_projects: frozenset[str], + service_project_name: ServiceProjectResolver, +) -> list[str]: + display_by_key: dict[str, str] = {} + + def add(raw_name: object) -> None: + name = str(raw_name or "").strip() + key = name.casefold() + if name and key in indexed_projects: + display_by_key.setdefault(key, name) + + metadata_names = state.metadata.get("project_names") + if isinstance(metadata_names, Sequence) and not isinstance(metadata_names, (str, bytes)): + for project in metadata_names: + add(project) + metadata_roots = state.metadata.get("project_roots") + if isinstance(metadata_roots, Mapping): + for project in metadata_roots: + add(project) + for project, requirements in state.requirements.items(): + add(getattr(requirements, "project", "") or project) + for service_name, service in state.services.items(): + add(service_project_name(service_name, service)) + return [display_by_key.get(key, key) for key in sorted(indexed_projects)] + + +def merge_indexed_states( + indexed_states: Sequence[IndexedState], + *, + project_names_from_state: ProjectNamesResolver, + source_run_ids: SourceRunIdsResolver, +) -> RunState: + ordered = sorted(indexed_states, key=lambda item: item.candidate.sequence) + latest = max( + indexed_states, + key=lambda item: ( + item.candidate.activation_sequence, + item.candidate.sequence, + str(item.candidate.state_path), + ), + ).state + services: dict[str, ServiceRecord] = {} + requirements: dict[str, RequirementsResult] = {} + pointers: dict[str, str] = {} + project_roots: dict[str, object] = {} + project_names: dict[str, str] = {} + for indexed_state in ordered: + state = indexed_state.state + services.update(state.services) + requirements.update(state.requirements) + pointers.update(state.pointers) + for name in project_names_from_state(state): + project_names.setdefault(name.casefold(), name) + roots = state.metadata.get("project_roots") + if isinstance(roots, Mapping): + project_roots.update({str(name): root for name, root in roots.items()}) + + selected_project_names = [project_names[key] for key in sorted(project_names)] + metadata = merge_project_scoped_metadata( + latest.metadata, + ordered, + project_names=selected_project_names, + ) + metadata["project_names"] = selected_project_names + all_source_run_ids = { + source_run_id + for indexed_state in ordered + for source_run_id in ( + indexed_state.candidate.run_id, + *source_run_ids(indexed_state.state), + ) + } + metadata["state_source_run_ids"] = sorted(all_source_run_ids) + if project_roots: + metadata["project_roots"] = project_roots + merge_startup_identities(metadata, ordered, project_roots=project_roots) + return RunState( + run_id=latest.run_id, + mode=latest.mode, + schema_version=latest.schema_version, + backend_mode=latest.backend_mode, + services=services, + requirements=requirements, + pointers=pointers, + metadata=metadata, + ) + + +def merge_project_scoped_metadata( + latest_metadata: Mapping[str, object], + indexed_states: Sequence[IndexedState], + *, + project_names: Sequence[str], +) -> dict[str, object]: + metadata = dict(latest_metadata) + for key in PROJECT_SCOPED_METADATA_MAP_KEYS: + merged: dict[str, object] = {} + for indexed_state in indexed_states: + raw = indexed_state.state.metadata.get(key) + if isinstance(raw, Mapping): + merged.update({str(project): value for project, value in raw.items()}) + if merged: + metadata[key] = merged + else: + metadata.pop(key, None) + + stopped_by_identity: dict[tuple[str, str, str], dict[str, object]] = {} + for indexed_state in indexed_states: + raw_stopped = indexed_state.state.metadata.get(DASHBOARD_STOPPED_SERVICES_KEY) + if not isinstance(raw_stopped, list): + continue + for item in raw_stopped: + if not isinstance(item, Mapping): + continue + project = str(item.get("project", "")).strip() + name = str(item.get("name", "")).strip() + service_type = str(item.get("type", "")).strip().lower() + if not project or not name or not service_type: + continue + stopped_by_identity[(project.casefold(), name.casefold(), service_type)] = dict(item) + if stopped_by_identity: + metadata[DASHBOARD_STOPPED_SERVICES_KEY] = [stopped_by_identity[key] for key in sorted(stopped_by_identity)] + else: + metadata.pop(DASHBOARD_STOPPED_SERVICES_KEY, None) + + if len(indexed_states) > 1: + metadata.pop("project_test_results_root", None) + metadata.pop("project_test_results_updated_at", None) + return filter_project_scoped_metadata( + metadata, + list(project_names), + case_sensitive=False, + ) + + +def filter_startup_identity_projects( + metadata: dict[str, object], + project_names: frozenset[str], +) -> None: + raw_identity = metadata.get("startup_identity") + if not isinstance(raw_identity, Mapping): + return + identity = dict(raw_identity) + raw_projects = identity.get("projects") + projects = raw_projects if isinstance(raw_projects, list) else [] + identity["projects"] = [ + project + for project in projects + if isinstance(project, Mapping) and str(project.get("name", "")).strip().casefold() in project_names + ] + identity.pop("fingerprint", None) + identity["fingerprint"] = text_fingerprint(json.dumps(identity, sort_keys=True)) + metadata["startup_identity"] = identity + + +def merge_startup_identities( + metadata: dict[str, object], + indexed_states: Sequence[IndexedState], + *, + project_roots: Mapping[str, object], +) -> None: + identities = [ + dict(identity) + for indexed_state in indexed_states + if isinstance((identity := indexed_state.state.metadata.get("startup_identity")), Mapping) + ] + if not identities: + metadata.pop("startup_identity", None) + return + comparable = [ + {key: value for key, value in identity.items() if key not in {"fingerprint", "projects"}} + for identity in identities + ] + if any(payload != comparable[0] for payload in comparable[1:]): + metadata.pop("startup_identity", None) + return + identity = dict(comparable[0]) + identity["projects"] = [ + {"name": name, "root": str(root)} + for name, root in sorted(project_roots.items(), key=lambda item: item[0].casefold()) + ] + identity["fingerprint"] = text_fingerprint(json.dumps(identity, sort_keys=True)) + metadata["startup_identity"] = identity diff --git a/tests/python/actions/ship_workflow_test_support.py b/tests/python/actions/ship_workflow_test_support.py index 16dca6348..4d7243175 100644 --- a/tests/python/actions/ship_workflow_test_support.py +++ b/tests/python/actions/ship_workflow_test_support.py @@ -11,6 +11,7 @@ from types import SimpleNamespace from typing import Any +from envctl_engine.actions.action_git_state_support import ExistingPullRequest from envctl_engine.actions.action_ship_support import run_ship_workflow @@ -44,6 +45,7 @@ def run( add_ship_pr_label: Callable[[object, Path, str], int] | None = None, probe_dirty_worktree: Callable[..., object] | None = None, existing_pr_url: Callable[[Path, str], str] | None = None, + existing_pull_request: Callable[[Path, str], ExistingPullRequest | None] | None = None, partition_envctl_protected_paths: Callable[[str], object] | None = None, ordered_unique_paths: Callable[..., list[str]] | None = None, github_pr_checks: Callable[..., dict[str, object]] | None = None, @@ -64,6 +66,7 @@ def run( add_ship_pr_label=add_ship_pr_label or self.add_ship_pr_label, probe_dirty_worktree=probe_dirty_worktree or self.probe_dirty_worktree, existing_pr_url=existing_pr_url or self.existing_pr_url, + existing_pull_request=existing_pull_request, partition_envctl_protected_paths=( partition_envctl_protected_paths or self.partition_envctl_protected_paths ), diff --git a/tests/python/actions/test_action_git_state_support.py b/tests/python/actions/test_action_git_state_support.py index 30acf4c4b..359253067 100644 --- a/tests/python/actions/test_action_git_state_support.py +++ b/tests/python/actions/test_action_git_state_support.py @@ -25,23 +25,85 @@ def fake_git_output(_git_root: Path, args: list[str]) -> str: self.assertTrue(report.untracked) self.assertTrue(report.dirty) - def test_detect_default_branch_prefers_origin_head_then_local_candidates(self) -> None: + def test_detect_default_branch_prefers_dev_then_main_then_master(self) -> None: calls: list[list[str]] = [] - def origin_head(_git_root: Path, args: list[str]) -> str: + def dev_exists(_git_root: Path, args: list[str]) -> str: calls.append(args) - return "origin/develop\n" + return "abc123\n" if args == ["rev-parse", "--verify", "refs/remotes/origin/dev"] else "" - self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=origin_head), "develop") - self.assertEqual(calls, [["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]]) + self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=dev_exists), "dev") + self.assertEqual(calls[0], ["rev-parse", "--verify", "refs/remotes/origin/dev"]) + self.assertTrue(any(args[:3] == ["ls-remote", "--symref", "origin"] for args in calls)) def local_main(_git_root: Path, args: list[str]) -> str: - if args == ["rev-parse", "--verify", "main"]: + if args == ["rev-parse", "--verify", "refs/heads/main"]: return "abc123\n" return "" self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=local_main), "main") + def remote_master(_git_root: Path, args: list[str]) -> str: + if args == ["rev-parse", "--verify", "refs/remotes/origin/master"]: + return "abc123\n" + return "" + + self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=remote_master), "master") + + def test_detect_default_branch_uses_remote_head_only_after_preferred_branches_are_absent(self) -> None: + def remote_head(_git_root: Path, args: list[str]) -> str: + if args == ["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]: + return "origin/develop\n" + return "" + + self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=remote_head), "develop") + + def test_detect_default_branch_discovers_dev_omitted_by_local_fetch_refspec(self) -> None: + def single_branch_clone(_git_root: Path, args: list[str]) -> str: + if args == ["rev-parse", "--verify", "refs/remotes/origin/main"]: + return "abc123\n" + if args[:3] == ["ls-remote", "--symref", "origin"]: + return "ref: refs/heads/main\tHEAD\nmainsha\tHEAD\ndevsha\trefs/heads/dev\nmainsha\trefs/heads/main\n" + return "" + + self.assertEqual( + git_state.detect_default_branch(Path("/repo"), git_output=single_branch_clone), + "dev", + ) + + def test_live_remote_inventory_overrides_stale_deleted_dev_tracking_ref(self) -> None: + def stale_dev(_git_root: Path, args: list[str]) -> str: + if args == ["rev-parse", "--verify", "refs/remotes/origin/dev"]: + return "stale-dev-sha\n" + if args == ["rev-parse", "--verify", "refs/remotes/origin/main"]: + return "main-sha\n" + if args[:3] == ["ls-remote", "--symref", "origin"]: + return ( + "ref: refs/heads/main\tHEAD\n" + "main-sha\tHEAD\n" + "main-sha\trefs/heads/main\n" + "master-sha\trefs/heads/master\n" + ) + return "" + + self.assertEqual( + git_state.detect_default_branch(Path("/repo"), git_output=stale_dev), + "main", + ) + + def test_live_remote_default_overrides_stale_dev_when_no_preferred_head_exists(self) -> None: + def no_preferred_remote(_git_root: Path, args: list[str]) -> str: + if args == ["rev-parse", "--verify", "refs/remotes/origin/dev"]: + return "stale-dev-sha\n" + if args[:3] == ["ls-remote", "--symref", "origin"]: + return "ref: refs/heads/feature/alpha\tHEAD\nfeature-sha\tHEAD\n" + return "" + + self.assertEqual( + git_state.detect_default_branch(Path("/repo"), git_output=no_preferred_remote), + "feature/alpha", + ) + def test_existing_pr_url_uses_gh_pr_list_for_open_branch(self) -> None: calls: list[tuple[list[str], str]] = [] @@ -57,7 +119,12 @@ def fake_run( self.assertTrue(capture_output) self.assertFalse(check) calls.append((args, cwd)) - return subprocess.CompletedProcess(args, 0, stdout="https://github.com/acme/repo/pull/9\n", stderr="") + return subprocess.CompletedProcess( + args, + 0, + stdout='[{"url":"https://github.com/acme/repo/pull/9","baseRefName":"dev"}]\n', + stderr="", + ) url = git_state.existing_pr_url(Path("/repo"), "feature", gh_path="/usr/bin/gh", run_process=fake_run) @@ -66,17 +133,95 @@ def fake_run( self.assertEqual(calls[0][0][:4], ["/usr/bin/gh", "pr", "list", "--head"]) self.assertIn("feature", calls[0][0]) + def test_existing_pull_request_preserves_base_metadata(self) -> None: + def fake_run(args: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + args, + 0, + stdout='[{"url":"https://github.com/acme/repo/pull/9","baseRefName":"release"}]', + stderr="", + ) + + existing = git_state.existing_pull_request( + Path("/repo"), + "feature", + gh_path="/usr/bin/gh", + run_process=fake_run, + ) + + self.assertEqual( + existing, + git_state.ExistingPullRequest( + url="https://github.com/acme/repo/pull/9", + base_branch="release", + ), + ) + + def test_update_pull_request_base_uses_pr_url_and_reports_missing_gh(self) -> None: + existing = git_state.ExistingPullRequest(url="https://github.com/acme/repo/pull/9", base_branch="main") + calls: list[list[str]] = [] + + def fake_run(args: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + calls.append(args) + return subprocess.CompletedProcess(args, 0, stdout="updated\n", stderr="") + + updated = git_state.update_pull_request_base( + Path("/repo"), + existing, + "dev", + gh_path="/usr/bin/gh", + run_process=fake_run, + ) + missing = git_state.update_pull_request_base( + Path("/repo"), + existing, + "dev", + gh_path=None, + run_process=lambda *_args, **_kwargs: self.fail("process should not run"), + ) + + self.assertEqual(updated.returncode, 0) + self.assertEqual( + calls, + [["/usr/bin/gh", "pr", "edit", "https://github.com/acme/repo/pull/9", "--base", "dev"]], + ) + self.assertEqual(missing.returncode, 127) + self.assertIn("gh is required", missing.stderr) + def test_existing_pr_url_skips_invalid_inputs_and_failed_gh(self) -> None: def unexpected_run(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: raise AssertionError("run_process should not be called") - self.assertEqual(git_state.existing_pr_url(Path("/repo"), "HEAD", gh_path="/usr/bin/gh", run_process=unexpected_run), "") - self.assertEqual(git_state.existing_pr_url(Path("/repo"), "feature", gh_path=None, run_process=unexpected_run), "") + self.assertEqual( + git_state.existing_pr_url(Path("/repo"), "HEAD", gh_path="/usr/bin/gh", run_process=unexpected_run), "" + ) + self.assertEqual( + git_state.existing_pr_url(Path("/repo"), "feature", gh_path=None, run_process=unexpected_run), "" + ) + + def test_remote_branch_probe_is_noninteractive_and_bounded(self) -> None: + captured: dict[str, object] = {} + + def timed_out(args: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + captured.update(kwargs) + raise subprocess.TimeoutExpired(args, timeout=float(kwargs["timeout"])) + + result = git_state.run_git( + Path("/repo"), + ["ls-remote", "--heads", "origin", "refs/heads/dev"], + run_process=timed_out, + ) + + self.assertEqual(result.returncode, 124) + self.assertEqual(captured["timeout"], 10) + self.assertEqual(captured["env"]["GIT_TERMINAL_PROMPT"], "0") # type: ignore[index] def failed_run(*args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: return subprocess.CompletedProcess(args, 1, stdout="", stderr="not found") - self.assertEqual(git_state.existing_pr_url(Path("/repo"), "feature", gh_path="/usr/bin/gh", run_process=failed_run), "") + self.assertEqual( + git_state.existing_pr_url(Path("/repo"), "feature", gh_path="/usr/bin/gh", run_process=failed_run), "" + ) if __name__ == "__main__": diff --git a/tests/python/actions/test_action_pr_workflow_support.py b/tests/python/actions/test_action_pr_workflow_support.py index 9ccaf6068..0020ae767 100644 --- a/tests/python/actions/test_action_pr_workflow_support.py +++ b/tests/python/actions/test_action_pr_workflow_support.py @@ -7,9 +7,65 @@ from types import SimpleNamespace from envctl_engine.actions import action_pr_workflow_support as support +from envctl_engine.actions.action_git_state_support import ExistingPullRequest class ActionPrWorkflowSupportTests(unittest.TestCase): + def test_pr_workflow_rejects_default_branch_as_the_head(self) -> None: + context = SimpleNamespace( + repo_root=Path("/repo"), + project_root=Path("/repo"), + project_name="Main", + ) + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "dev\n", + resolve_base_branch_fn=lambda _context, _git_root: "dev", + existing_pull_request_fn=lambda *_args: None, + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), + probe_dirty_worktree_fn=lambda *_args: self.fail("worktree should not be probed"), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: self.fail("title should not be built"), + pr_body_fn=lambda *_args: self.fail("body should not be built"), + write_pr_body_file_fn=lambda _body: self.fail("body file should not be written"), + print_process_output_fn=lambda _result: None, + gh_path="/usr/bin/gh", + ) + + self.assertEqual(code, 1) + + def test_pr_workflow_rediscovers_existing_pr_before_self_base_guard(self) -> None: + context = SimpleNamespace( + repo_root=Path("/repo"), + project_root=Path("/repo"), + project_name="Main", + ) + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "dev\n", + resolve_base_branch_fn=lambda _context, _git_root: "dev", + existing_pull_request_fn=lambda _git_root, _branch: ExistingPullRequest( + url="https://github.com/acme/repo/pull/12", + base_branch="dev", + ), + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), + probe_dirty_worktree_fn=lambda *_args: self.fail("worktree should not be probed"), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: self.fail("title should not be built"), + pr_body_fn=lambda *_args: self.fail("body should not be built"), + write_pr_body_file_fn=lambda _body: self.fail("body file should not be written"), + print_process_output_fn=lambda _result: None, + gh_path="/usr/bin/gh", + ) + + self.assertEqual(code, 0) + def test_pr_workflow_skips_existing_pull_request(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo_root = Path(tmpdir) @@ -22,7 +78,11 @@ def test_pr_workflow_skips_existing_pull_request(self) -> None: git_available=True, git_output_fn=lambda _git_root, _args: "feature/demo\n", resolve_base_branch_fn=lambda _context, _git_root: "main", - existing_pr_url_fn=lambda _git_root, _branch: "https://github.com/acme/repo/pull/12", + existing_pull_request_fn=lambda _git_root, _branch: ExistingPullRequest( + url="https://github.com/acme/repo/pull/12", + base_branch="main", + ), + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), probe_dirty_worktree_fn=lambda *_args: self.fail("dirty worktree should not be probed"), run_commit_action_fn=lambda _context: self.fail("commit should not run"), pr_title_fn=lambda *_args: self.fail("title should not be built"), @@ -46,7 +106,9 @@ def test_pr_workflow_creates_with_gh_and_removes_body_file(self) -> None: def run_process(args: list[str], **_kwargs): # noqa: ANN001 calls.append([str(token) for token in args]) self.assertTrue(body_file.is_file()) - return subprocess.CompletedProcess(args=args, returncode=0, stdout="https://example.test/pr/1\n", stderr="") + return subprocess.CompletedProcess( + args=args, returncode=0, stdout="https://example.test/pr/1\n", stderr="" + ) def write_body(_body: str) -> Path: body_file.write_text("Body", encoding="utf-8") @@ -58,7 +120,8 @@ def write_body(_body: str) -> Path: git_available=True, git_output_fn=lambda _git_root, _args: "feature/demo\n", resolve_base_branch_fn=lambda _context, _git_root: "main", - existing_pr_url_fn=lambda _git_root, _branch: "", + existing_pull_request_fn=lambda _git_root, _branch: None, + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), probe_dirty_worktree_fn=lambda *_args: SimpleNamespace(dirty=False), run_commit_action_fn=lambda _context: self.fail("commit should not run"), pr_title_fn=lambda _context, _git_root, _head_branch: "Demo PR", @@ -75,6 +138,157 @@ def write_body(_body: str) -> Path: self.assertIn("--base", calls[0]) self.assertIn("main", calls[0]) + def test_explicit_base_updates_existing_pr_and_verifies_result(self) -> None: + context = SimpleNamespace( + repo_root=Path("/repo"), + project_root=Path("/repo"), + project_name="Main", + env={"ENVCTL_PR_BASE": "release"}, + ) + existing_values = iter( + [ + ExistingPullRequest("https://github.com/acme/repo/pull/12", "main"), + ExistingPullRequest("https://github.com/acme/repo/pull/12", "release"), + ] + ) + update_calls: list[tuple[ExistingPullRequest, str]] = [] + + def update_base( + _git_root: Path, + existing: ExistingPullRequest, + base_branch: str, + ) -> subprocess.CompletedProcess[str]: + update_calls.append((existing, base_branch)) + return subprocess.CompletedProcess([], 0, stdout="updated\n", stderr="") + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "feature/demo\n", + resolve_base_branch_fn=lambda *_args: self.fail("implicit resolver should not run"), + existing_pull_request_fn=lambda *_args: next(existing_values), + update_pull_request_base_fn=update_base, + probe_dirty_worktree_fn=lambda *_args: self.fail("worktree should not be probed"), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: self.fail("title should not be built"), + pr_body_fn=lambda *_args: self.fail("body should not be built"), + write_pr_body_file_fn=lambda _body: self.fail("body file should not be written"), + print_process_output_fn=lambda _result: None, + gh_path="/usr/bin/gh", + ) + + self.assertEqual(code, 0) + self.assertEqual(len(update_calls), 1) + self.assertEqual(update_calls[0][1], "release") + + def test_implicit_default_preserves_existing_pr_base(self) -> None: + context = SimpleNamespace( + repo_root=Path("/repo"), + project_root=Path("/repo"), + project_name="Main", + env={}, + ) + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "feature/demo\n", + resolve_base_branch_fn=lambda *_args: self.fail("implicit resolver should not run for an existing PR"), + existing_pull_request_fn=lambda *_args: ExistingPullRequest( + "https://github.com/acme/repo/pull/12", + "main", + ), + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), + probe_dirty_worktree_fn=lambda *_args: self.fail("worktree should not be probed"), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: self.fail("title should not be built"), + pr_body_fn=lambda *_args: self.fail("body should not be built"), + write_pr_body_file_fn=lambda _body: self.fail("body file should not be written"), + print_process_output_fn=lambda _result: None, + gh_path="/usr/bin/gh", + ) + + self.assertEqual(code, 0) + + def test_failed_gh_create_recovers_concurrent_matching_pr(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + repo_root = Path(tmpdir) + context = SimpleNamespace(repo_root=repo_root, project_root=repo_root, project_name="Main", env={}) + existing_values = iter( + [ + None, + ExistingPullRequest("https://github.com/acme/repo/pull/13", "dev"), + ] + ) + body_file = repo_root / "body.md" + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "feature/race\n", + resolve_base_branch_fn=lambda *_args: "dev", + existing_pull_request_fn=lambda *_args: next(existing_values), + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), + probe_dirty_worktree_fn=lambda *_args: SimpleNamespace(dirty=False), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: "Race", + pr_body_fn=lambda *_args: "Body", + write_pr_body_file_fn=lambda _body: body_file, + print_process_output_fn=lambda _result: self.fail("recovered create error should stay quiet"), + gh_path="/usr/bin/gh", + run_process_fn=lambda args, **_kwargs: subprocess.CompletedProcess( + args, + 1, + stdout="", + stderr="a pull request already exists", + ), + ) + + self.assertEqual(code, 0) + + def test_failed_helper_create_recovers_concurrent_matching_pr(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + repo_root = Path(tmpdir) + helper = repo_root / "utils" / "create-pr.sh" + helper.parent.mkdir(parents=True) + helper.write_text("#!/bin/sh\n", encoding="utf-8") + helper.chmod(0o755) + context = SimpleNamespace(repo_root=repo_root, project_root=repo_root, project_name="Main", env={}) + existing_values = iter( + [ + None, + ExistingPullRequest("https://github.com/acme/repo/pull/14", "dev"), + ] + ) + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "feature/race\n", + resolve_base_branch_fn=lambda *_args: "dev", + existing_pull_request_fn=lambda *_args: next(existing_values), + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), + probe_dirty_worktree_fn=lambda *_args: SimpleNamespace(dirty=False), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: self.fail("title should not be built"), + pr_body_fn=lambda *_args: self.fail("body should not be built"), + write_pr_body_file_fn=lambda _body: self.fail("body file should not be written"), + print_process_output_fn=lambda _result: self.fail("recovered helper error should stay quiet"), + gh_path=None, + run_process_fn=lambda args, **_kwargs: subprocess.CompletedProcess( + args, + 1, + stdout="", + stderr="a pull request already exists", + ), + ) + + self.assertEqual(code, 0) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/actions/test_action_ship_support.py b/tests/python/actions/test_action_ship_support.py index df0ece678..36da1e92b 100644 --- a/tests/python/actions/test_action_ship_support.py +++ b/tests/python/actions/test_action_ship_support.py @@ -20,7 +20,7 @@ def test_public_run_ship_workflow_is_thin_compatibility_wrapper() -> None: source = inspect.getsource(run_ship_workflow) assert "ShipWorkflowRunner" in source - assert len(source.splitlines()) <= 35 + assert len(source.splitlines()) <= 40 def test_ship_workflow_runner_exposes_named_phases() -> None: @@ -58,6 +58,7 @@ def test_ship_workflow_dependencies_group_injected_collaborators() -> None: "partition_envctl_protected_paths", "ordered_unique_paths", "github_pr_checks", + "existing_pull_request", } runner_fields = set(getattr(ship_support.ShipWorkflowRunner, "__dataclass_fields__", {})) diff --git a/tests/python/actions/test_action_ship_workflow_lifecycle.py b/tests/python/actions/test_action_ship_workflow_lifecycle.py index ca8a86615..c53019a9a 100644 --- a/tests/python/actions/test_action_ship_workflow_lifecycle.py +++ b/tests/python/actions/test_action_ship_workflow_lifecycle.py @@ -3,6 +3,7 @@ from pathlib import Path from types import SimpleNamespace +from envctl_engine.actions.action_git_state_support import ExistingPullRequest from tests.python.actions.ship_workflow_test_support import ship_workflow_fixture @@ -55,6 +56,82 @@ def github_pr_checks( assert result.payload["checks_expected_head_sha"] == "abc123" +def test_run_ship_workflow_reconciles_explicit_base_before_reusing_existing_pr() -> None: + with ship_workflow_fixture() as fixture: + fixture.context.env["ENVCTL_PR_BASE"] = "dev" + pr_calls: list[object] = [] + resolved_bases: list[str] = [] + + def run_pr_action(context: object) -> int: + pr_calls.append(context) + return 0 + + def resolve_base_ref(_git_root: Path, base_branch: str) -> str: + resolved_bases.append(base_branch) + return f"origin/{base_branch}" + + result = fixture.run(run_pr_action=run_pr_action, resolve_base_ref=resolve_base_ref) + + assert result.code == 0 + assert pr_calls == [fixture.context] + assert resolved_bases == ["dev"] + assert result.payload["pr_created"] is False + assert result.payload["pr_url"] == "https://github.com/acme/repo/pull/7" + assert result.payload["step_statuses"] == ["clean_no_changes", "pr_exists", "checks_passed"] + + +def test_run_ship_workflow_predicts_conflicts_against_implicit_existing_pr_base() -> None: + with ship_workflow_fixture() as fixture: + resolved_bases: list[str] = [] + + def resolve_base_branch(_context: object, _git_root: Path) -> str: + raise AssertionError("default resolver must not replace an existing PR base") + + def resolve_base_ref(_git_root: Path, base_branch: str) -> str: + resolved_bases.append(base_branch) + return f"origin/{base_branch}" + + result = fixture.run( + existing_pull_request=lambda _git_root, _branch: ExistingPullRequest( + url="https://github.com/acme/repo/pull/7", + base_branch="main", + ), + resolve_base_branch=resolve_base_branch, + resolve_base_ref=resolve_base_ref, + ) + + assert result.code == 0 + assert resolved_bases == ["main"] + assert result.payload["merge_conflicts"]["base_branch"] == "main" + + +def test_run_ship_workflow_tracks_concurrent_pr_winner_after_pr_action_recovers() -> None: + with ship_workflow_fixture() as fixture: + existing_calls = 0 + pr_calls = 0 + + def existing_pr_url(_git_root: Path, _branch: str) -> str: + nonlocal existing_calls + existing_calls += 1 + return "" if existing_calls == 1 else "https://github.com/acme/repo/pull/9" + + def run_pr_action(_context: object) -> int: + nonlocal pr_calls + pr_calls += 1 + return 0 + + result = fixture.run( + existing_pr_url=existing_pr_url, + run_pr_action=run_pr_action, + ) + + assert result.code == 0 + assert pr_calls == 1 + assert existing_calls >= 2 + assert result.payload["pr_url"] == "https://github.com/acme/repo/pull/9" + assert result.payload["step_statuses"] == ["clean_no_changes", "pr_created", "checks_passed"] + + def test_run_ship_workflow_pushes_clean_local_head_when_existing_pr_branch_is_stale() -> None: with ship_workflow_fixture() as fixture: run_git_calls: list[list[str]] = [] @@ -357,6 +434,4 @@ def git_output(_git_root: Path, args: list[str]) -> str: assert result.payload["committed"] is False assert result.payload["pushed"] is False assert result.payload["protected_local_artifacts_skipped"] == [".envctl-state/code-intelligence.json"] - assert result.stderr.splitlines() == [ - "ship: PR already exists for Main: https://github.com/acme/repo/pull/7" - ] + assert result.stderr.splitlines() == ["ship: PR already exists for Main: https://github.com/acme/repo/pull/7"] diff --git a/tests/python/actions/test_action_worktree_runner.py b/tests/python/actions/test_action_worktree_runner.py index 39e25d99c..08bff0ad9 100644 --- a/tests/python/actions/test_action_worktree_runner.py +++ b/tests/python/actions/test_action_worktree_runner.py @@ -74,9 +74,9 @@ def test_worktree_runner_uses_named_owners_for_target_resolution_and_delete_exec target_resolution_source = ( REPO_ROOT / "python/envctl_engine/actions/action_worktree_target_resolution.py" ).read_text(encoding="utf-8") - self_destruct_source = ( - REPO_ROOT / "python/envctl_engine/actions/action_worktree_self_destruct.py" - ).read_text(encoding="utf-8") + self_destruct_source = (REPO_ROOT / "python/envctl_engine/actions/action_worktree_self_destruct.py").read_text( + encoding="utf-8" + ) self.assertIn("class ActionWorktreeDeleteRunner", source) self.assertIn("return ActionWorktreeDeleteRunner(", source) @@ -106,9 +106,9 @@ def test_resolve_current_worktree_target_discovers_matching_current_tree(self) - runtime=runtime, require_configured_main_root=True, current_cwd=lambda: Path("/should/not/be/used"), - discover_tree_projects_fn=lambda repo_root, trees_dir_name: [("feature-a-1", tree_root)] - if repo_root == repo and trees_dir_name == "trees" - else [], + discover_tree_projects_fn=lambda repo_root, trees_dir_name: ( + [("feature-a-1", tree_root)] if repo_root == repo and trees_dir_name == "trees" else [] + ), main_repo_root_for_linked_worktree_fn=lambda _worktree_root: None, git_main_repo_root_for_worktree_fn=lambda _worktree_root, trees_dir_name=None: repo, ) @@ -117,6 +117,69 @@ def test_resolve_current_worktree_target_discovers_matching_current_tree(self) - self.assertEqual(getattr(target, "name"), "feature-a-1") self.assertEqual(getattr(target, "root"), tree_root) + def test_resolve_current_worktree_target_accepts_linked_checkout_outside_trees_directory(self) -> None: + repo = Path("/tmp/repo").resolve() + linked_root = Path("/tmp/envctl-deep-runtime-refactor").resolve() + calls: list[tuple[str, ...]] = [] + + def run(command, *, cwd, timeout): # noqa: ANN001 + self.assertEqual(timeout, 10.0) + calls.append(tuple(command)) + if command[-2:] == ["rev-parse", "--show-toplevel"]: + return SimpleNamespace(returncode=0, stdout=f"{linked_root}\n") + if command[-2:] == ["branch", "--show-current"]: + self.assertEqual(Path(cwd), linked_root) + return SimpleNamespace(returncode=0, stdout="agent/deep-runtime-refactor\n") + return SimpleNamespace(returncode=1, stdout="") + + runtime = SimpleNamespace( + env={"ENVCTL_INVOCATION_CWD": str(linked_root / "python")}, + raw_runtime=SimpleNamespace( + config=SimpleNamespace(base_dir=repo, trees_dir_name="trees"), + process_runner=SimpleNamespace(run=run), + ), + ) + + target = resolve_current_worktree_target( + runtime=runtime, + require_configured_root_match=True, + current_cwd=lambda: Path("/should/not/be/used"), + discover_tree_projects_fn=lambda _repo_root, _trees_dir_name: [], + git_main_repo_root_for_worktree_fn=lambda **_kwargs: repo, + ) + + self.assertIsNotNone(target) + self.assertEqual(getattr(target, "name"), "agent/deep-runtime-refactor") + self.assertEqual(getattr(target, "root"), linked_root) + self.assertEqual( + calls, + [ + ("git", "rev-parse", "--show-toplevel"), + ("git", "branch", "--show-current"), + ], + ) + + def test_resolve_current_worktree_target_never_synthesizes_main_checkout_for_tree_mode(self) -> None: + repo = Path("/tmp/repo").resolve() + runtime = SimpleNamespace( + env={"ENVCTL_INVOCATION_CWD": str(repo)}, + raw_runtime=SimpleNamespace( + config=SimpleNamespace(base_dir=repo, trees_dir_name="trees"), + process_runner=SimpleNamespace( + run=lambda *_args, **_kwargs: SimpleNamespace(returncode=0, stdout=f"{repo}\n") + ), + ), + ) + + target = resolve_current_worktree_target( + runtime=runtime, + require_configured_root_match=True, + discover_tree_projects_fn=lambda _repo_root, _trees_dir_name: [], + git_main_repo_root_for_worktree_fn=lambda **_kwargs: repo, + ) + + self.assertIsNone(target) + def test_run_delete_worktree_action_runs_cleanup_and_delete(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo_root = Path(tmpdir) / "repo" @@ -245,15 +308,14 @@ def test_run_self_destruct_worktree_action_uses_explicit_current_worktree_safety calls: list[tuple[str, object]] = [] runtime = SimpleNamespace( - _blast_worktree_before_delete=lambda **kwargs: ( - calls.append(("cleanup", kwargs)) or ["cleanup warning"] - ), + _blast_worktree_before_delete=lambda **kwargs: calls.append(("cleanup", kwargs)) or ["cleanup warning"], _trees_root_for_worktree=lambda value: calls.append(("trees_root", value)) or tree_root.parent, ) orchestrator = SimpleNamespace( runtime=runtime, - resolve_targets=lambda route, *, trees_only: calls.append(("resolve", (route.command, trees_only))) - or ([target], None), + resolve_targets=lambda route, *, trees_only: ( + calls.append(("resolve", (route.command, trees_only))) or ([target], None) + ), _main_repo_root_for_worktree=lambda value: calls.append(("main_repo", value)) or tree_root.parent, _spawn_self_destruct_helper=lambda **kwargs: calls.append(("spawn", kwargs)) or True, ) diff --git a/tests/python/actions/test_actions_cli_pr.py b/tests/python/actions/test_actions_cli_pr.py index e92c9dd72..cf5522e0e 100644 --- a/tests/python/actions/test_actions_cli_pr.py +++ b/tests/python/actions/test_actions_cli_pr.py @@ -2,6 +2,7 @@ from __future__ import annotations from tests.python.actions.actions_cli_test_support import * # noqa: F403,F405 +from envctl_engine.actions.action_git_state_support import ExistingPullRequest class ActionsCliPrTests(unittest.TestCase): @@ -34,8 +35,11 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch( - "envctl_engine.actions.project_action_domain.existing_pr_url", - return_value="https://github.com/acme/supportopia/pull/42", + "envctl_engine.actions.project_action_domain.existing_pull_request", + return_value=ExistingPullRequest( + url="https://github.com/acme/supportopia/pull/42", + base_branch="main", + ), ), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), @@ -95,7 +99,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.probe_dirty_worktree") as dirty_probe, patch("envctl_engine.actions.project_action_domain.run_commit_action", return_value=0) as commit_action, patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -146,7 +150,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.probe_dirty_worktree") as dirty_probe, patch("envctl_engine.actions.project_action_domain.run_commit_action", return_value=1) as commit_action, patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -223,7 +227,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -302,7 +306,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -369,7 +373,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -431,7 +435,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 patch.dict(os.environ, {"ENVCTL_PR_BODY": "Typed PR body\n\nWith details."}, clear=False), patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -495,7 +499,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 patch("builtins.input", side_effect=AssertionError("input() should not be called for pr action")), patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -553,7 +557,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -602,7 +606,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -659,7 +663,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): diff --git a/tests/python/actions/test_actions_cli_ship.py b/tests/python/actions/test_actions_cli_ship.py index 569e6da2a..b947f5107 100644 --- a/tests/python/actions/test_actions_cli_ship.py +++ b/tests/python/actions/test_actions_cli_ship.py @@ -1,6 +1,7 @@ # ruff: noqa: F403,F405 from __future__ import annotations +from envctl_engine.actions.action_git_state_support import ExistingPullRequest from tests.python.actions.actions_cli_test_support import * # noqa: F403,F405 @@ -80,7 +81,14 @@ def fake_commit_action(_context: object) -> int: patch("envctl_engine.actions.project_action_domain.add_ship_pr_label", return_value=0) as label_pr, patch( "envctl_engine.actions.project_action_domain.existing_pr_url", - side_effect=["", "https://github.com/acme/repo/pull/7"], + return_value="https://github.com/acme/repo/pull/7", + ), + patch( + "envctl_engine.actions.project_action_domain.existing_pull_request", + side_effect=[ + None, + ExistingPullRequest("https://github.com/acme/repo/pull/7", "main"), + ], ), patch( "envctl_engine.actions.project_action_domain._github_pr_checks", @@ -169,6 +177,10 @@ def fake_run_git(_git_root: Path, args: list[str]) -> subprocess.CompletedProces "envctl_engine.actions.project_action_domain.existing_pr_url", return_value="https://github.com/acme/repo/pull/7", ), + patch( + "envctl_engine.actions.project_action_domain.existing_pull_request", + return_value=ExistingPullRequest("https://github.com/acme/repo/pull/7", "main"), + ), patch( "envctl_engine.actions.project_action_domain._github_pr_checks", return_value={ @@ -245,6 +257,10 @@ def fake_run(git_root: Path, args: list[str]): # noqa: ANN001 "envctl_engine.actions.project_action_domain.existing_pr_url", return_value="https://github.com/acme/repo/pull/7", ), + patch( + "envctl_engine.actions.project_action_domain.existing_pull_request", + return_value=ExistingPullRequest("https://github.com/acme/repo/pull/7", "main"), + ), redirect_stdout(StringIO()) as stdout, ): code = domain.run_ship_action(context) @@ -284,11 +300,7 @@ def fake_git_output(_git_root: Path, args: list[str]) -> str: if args == ["diff", "--name-only", "--diff-filter=U"]: return "src/service.py\n" if args == ["ls-files", "-u"]: - return ( - "100644 aaa 1\tsrc/service.py\n" - "100644 bbb 2\tsrc/service.py\n" - "100644 ccc 3\tsrc/service.py\n" - ) + return "100644 aaa 1\tsrc/service.py\n100644 bbb 2\tsrc/service.py\n100644 ccc 3\tsrc/service.py\n" return "" with ( diff --git a/tests/python/planning/plan_agent_launch_support_test_support.py b/tests/python/planning/plan_agent_launch_support_test_support.py index 260aa7385..63dce63f6 100644 --- a/tests/python/planning/plan_agent_launch_support_test_support.py +++ b/tests/python/planning/plan_agent_launch_support_test_support.py @@ -124,8 +124,14 @@ class _StateRepositoryHarness: def __init__(self, state: RunState | None) -> None: self.state = state - def load_latest(self, *, mode: str | None = None, strict_mode_match: bool = False) -> RunState | None: - _ = mode, strict_mode_match + def load_latest( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: list[str] | None = None, + ) -> RunState | None: + _ = mode, strict_mode_match, project_names return self.state diff --git a/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py b/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py index 127f9fcd4..ebc0619ca 100644 --- a/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py +++ b/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py @@ -24,8 +24,14 @@ class _StateRepositoryHarness: def __init__(self, state: RunState | None) -> None: self.state = state - def load_latest(self, *, mode: str | None = None, strict_mode_match: bool = False) -> RunState | None: - _ = mode, strict_mode_match + def load_latest( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: list[str] | None = None, + ) -> RunState | None: + _ = mode, strict_mode_match, project_names return self.state @@ -127,9 +133,10 @@ def test_matching_and_port_helpers_cover_stale_or_partial_state(self) -> None: self.assertTrue(_state_service_matches_worktree(matching_service, worktree)) self.assertFalse(_state_service_matches_worktree(stale_service, worktree)) self.assertEqual(_component_port({"requested": "0", "resources": {"primary": "49152"}}), 49152) - self.assertEqual(_service_address_lines(RunState(run_id="run-1", mode="trees", services={"api": matching_service})), [ - "Backend (backend): http://localhost:8000" - ]) + self.assertEqual( + _service_address_lines(RunState(run_id="run-1", mode="trees", services={"api": matching_service})), + ["Backend (backend): http://localhost:8000"], + ) def test_runtime_section_uses_legacy_state_loader_when_repository_is_unavailable(self) -> None: state = RunState( diff --git a/tests/python/planning/test_planning_worktree_setup_provenance.py b/tests/python/planning/test_planning_worktree_setup_provenance.py index 8fbd5cc96..c10e2e1f3 100644 --- a/tests/python/planning/test_planning_worktree_setup_provenance.py +++ b/tests/python/planning/test_planning_worktree_setup_provenance.py @@ -1,10 +1,62 @@ # ruff: noqa: F403,F405 from __future__ import annotations +from types import SimpleNamespace + +from envctl_engine.planning.worktree_provenance import detect_default_branch from tests.python.planning.planning_worktree_setup_test_support import * class PlanningWorktreeSetupProvenanceTests(PlanningWorktreeSetupTestCase): + def test_detached_default_branch_uses_live_remote_over_stale_tracking_ref(self) -> None: + calls: list[dict[str, object]] = [] + + def fake_run(cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 + args = [str(token) for token in cmd] + calls.append({"args": args, "cwd": cwd, "env": env, "timeout": timeout}) + git_args = args[3:] + if git_args == ["rev-parse", "--verify", "refs/remotes/origin/dev"]: + return subprocess.CompletedProcess(args, 0, stdout="stale-dev\n", stderr="") + if git_args == ["rev-parse", "--verify", "refs/remotes/origin/main"]: + return subprocess.CompletedProcess(args, 0, stdout="main\n", stderr="") + if git_args[:3] == ["ls-remote", "--symref", "origin"]: + return subprocess.CompletedProcess( + args, + 0, + stdout=("ref: refs/heads/main\tHEAD\nmain-sha\tHEAD\nmain-sha\trefs/heads/main\n"), + stderr="", + ) + return subprocess.CompletedProcess(args, 1, stdout="", stderr="missing") + + runtime = SimpleNamespace( + config=SimpleNamespace(base_dir=Path("/repo")), + process_runner=SimpleNamespace(run=fake_run), + _command_env=lambda *, port: {"BASE": "1", "PORT": str(port)}, + ) + + self.assertEqual(detect_default_branch(runtime), "main") + live_probe = next(call for call in calls if call["args"][3] == "ls-remote") + self.assertEqual(live_probe["timeout"], 10.0) + self.assertEqual(live_probe["env"]["GIT_TERMINAL_PROMPT"], "0") + self.assertEqual(live_probe["env"]["BASE"], "1") + + def test_detached_default_branch_falls_back_to_local_tracking_ref_offline(self) -> None: + def fake_run(cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 + _ = cwd, env, timeout + args = [str(token) for token in cmd] + git_args = args[3:] + if git_args == ["rev-parse", "--verify", "refs/remotes/origin/dev"]: + return subprocess.CompletedProcess(args, 0, stdout="local-dev\n", stderr="") + return subprocess.CompletedProcess(args, 1, stdout="", stderr="offline") + + runtime = SimpleNamespace( + config=SimpleNamespace(base_dir=Path("/repo")), + process_runner=SimpleNamespace(run=fake_run), + _command_env=lambda *, port: {"PORT": str(port)}, + ) + + self.assertEqual(detect_default_branch(runtime), "dev") + def test_setup_worktree_creation_writes_provenance_file(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -57,9 +109,7 @@ def fake_run(cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 'project_name: "repo"\n', ) self.assertEqual( - (repo / "trees" / "feature-a" / "1" / ".serena" / "project.local.yml").read_text( - encoding="utf-8" - ), + (repo / "trees" / "feature-a" / "1" / ".serena" / "project.local.yml").read_text(encoding="utf-8"), 'project_name: "repo-feature-a-1"\n', ) self.assertEqual( @@ -233,7 +283,9 @@ def fake_run(cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 if git_cwd == str(unrelated.resolve()) and command[3:] == ["rev-parse", "--show-toplevel"]: return subprocess.CompletedProcess(args=command, returncode=0, stdout=f"{unrelated}\n", stderr="") if git_cwd == str(repo.resolve()) and command[3:] == ["rev-parse", "--git-common-dir"]: - return subprocess.CompletedProcess(args=command, returncode=0, stdout=f"{repo / '.git'}\n", stderr="") + return subprocess.CompletedProcess( + args=command, returncode=0, stdout=f"{repo / '.git'}\n", stderr="" + ) if git_cwd == str(unrelated.resolve()) and command[3:] == ["rev-parse", "--git-common-dir"]: return subprocess.CompletedProcess( args=command, returncode=0, stdout=f"{unrelated / '.git'}\n", stderr="" diff --git a/tests/python/runtime/engine_runtime_real_startup_test_support.py b/tests/python/runtime/engine_runtime_real_startup_test_support.py index 7f46ce704..7da958593 100644 --- a/tests/python/runtime/engine_runtime_real_startup_test_support.py +++ b/tests/python/runtime/engine_runtime_real_startup_test_support.py @@ -102,6 +102,7 @@ def __init__(self) -> None: self._next_pid = 4000 self.auto_track_listener_ports = False self._pid_listener_ports: dict[int, int] = {} + self._terminated_pids: set[int] = set() self.wait_for_port_result = False self.wait_for_port_calls: list[tuple[int, float]] = [] self.wait_for_pid_port_result = False @@ -199,6 +200,7 @@ def start_background(self, cmd, *, cwd=None, env=None, stdout_path=None, stderr_ _ = stderr_path self._next_pid += 1 pid = self._next_pid + self._terminated_pids.discard(pid) if parsed_port is not None and parsed_port in self.fail_start_ports: if stdout_path: @@ -269,6 +271,8 @@ def find_pid_listener_port( return self.find_pid_listener_port_result def is_pid_running(self, _pid: int) -> bool: + if _pid in self._terminated_pids: + return False if self.auto_track_listener_ports and _pid in self._pid_listener_ports: return True return self.pid_running_result @@ -281,6 +285,7 @@ def pid_owns_port(self, _pid: int, _port: int) -> bool: def terminate(self, _pid: int, *, term_timeout: float = 2.0, kill_timeout: float = 1.0) -> bool: _ = term_timeout, kill_timeout + self._terminated_pids.add(_pid) if self.auto_track_listener_ports: self._pid_listener_ports.pop(_pid, None) return True diff --git a/tests/python/runtime/lifecycle_parity_test_support.py b/tests/python/runtime/lifecycle_parity_test_support.py index 6b1c15376..8126f913d 100644 --- a/tests/python/runtime/lifecycle_parity_test_support.py +++ b/tests/python/runtime/lifecycle_parity_test_support.py @@ -33,7 +33,6 @@ dump_state = state_module.dump_state - class _NoopPlanner: def __init__(self) -> None: self.released = False @@ -46,6 +45,7 @@ def release(self, port: int, owner: str | None = None) -> None: _ = owner self.released_ports.append(port) + class _TrackingPlanner: def __init__(self) -> None: self.released_session = False @@ -57,6 +57,7 @@ def release_session(self) -> None: def release_all(self) -> None: self.released_all = True + class _TrackingRunner: def __init__(self) -> None: self.terminated: list[int] = [] @@ -72,7 +73,7 @@ def terminate(self, pid: int, *, term_timeout: float = 2.0, kill_timeout: float return True def is_pid_running(self, pid: int) -> bool: - return pid > 0 + return pid > 0 and pid not in self.terminated def pid_owns_port(self, pid: int, port: int) -> bool: _ = pid, port @@ -94,11 +95,13 @@ def run(self, cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 return type("Result", (), {"returncode": 0, "stdout": stdout, "stderr": ""})() return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + class _OwnershipDenyRunner(_TrackingRunner): def pid_owns_port(self, pid: int, port: int) -> bool: _ = pid, port return False + class _ResumeRestoreRunner: def __init__(self) -> None: self.run_calls: list[tuple[str, ...]] = [] @@ -175,6 +178,7 @@ def terminate(self, pid: int, *, term_timeout: float = 2.0, kill_timeout: float self._running.discard(pid) return True + __all__ = [ "Path", "PYTHON_ROOT", diff --git a/tests/python/runtime/test_engine_runtime_artifacts.py b/tests/python/runtime/test_engine_runtime_artifacts.py index cf3973fee..05c3f32e2 100644 --- a/tests/python/runtime/test_engine_runtime_artifacts.py +++ b/tests/python/runtime/test_engine_runtime_artifacts.py @@ -107,6 +107,34 @@ def test_persist_events_snapshot_updates_bound_run_directory(self) -> None: self.assertEqual((legacy_root / "events.jsonl").read_text(encoding="utf-8"), expected) self.assertEqual((run_dir / "events.jsonl").read_text(encoding="utf-8"), expected) + def test_persist_events_snapshot_uses_repository_fence_for_bound_run(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_root = Path(tmpdir) / "runtime" + legacy_root = Path(tmpdir) / "legacy" + calls: list[dict[str, object]] = [] + runtime = SimpleNamespace( + events=[{"event": "test"}], + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + env={"ENVCTL_DEBUG_UI_RUN_ID": "run-1"}, + state_repository=SimpleNamespace(write_runtime_artifact=lambda **kwargs: calls.append(kwargs) or False), + ) + + persist_events_snapshot(runtime) + + self.assertEqual( + calls, + [ + { + "run_id": "run-1", + "artifact_name": "events.jsonl", + "text": '{"event": "test"}\n', + } + ], + ) + self.assertFalse((runtime_root / "events.jsonl").exists()) + self.assertFalse((legacy_root / "events.jsonl").exists()) + def test_write_artifacts_writes_runtime_readiness_report_synchronously_without_explicit_async_opt_in(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: runtime_root = Path(tmpdir) / "runtime" @@ -235,6 +263,51 @@ def test_write_runtime_readiness_report_writes_runtime_and_run_reports(self) -> self.assertEqual(run_report["gap_report"]["sha256"], "gap123") self.assertEqual(run_report["feature_matrix"]["sha256"], "matrix123") + def test_readiness_report_does_not_bypass_repository_when_stale_run_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_root = Path(tmpdir) / "runtime" + legacy_root = Path(tmpdir) / "legacy" + run_dir = runtime_root / "runs" / "run-retired" + calls: list[dict[str, object]] = [] + result = RuntimeReadinessResult( + passed=True, + report_path=Path(tmpdir) / "gap.json", + report_generated_at="now", + report_sha256="gap", + matrix_path=Path(tmpdir) / "matrix.json", + matrix_generated_at="now", + matrix_sha256="matrix", + parity_manifest_path=Path(tmpdir) / "manifest.json", + parity_manifest_generated_at="now", + parity_manifest_sha256="manifest", + blocking_gap_count=0, + high_gap_count=0, + medium_gap_count=0, + low_gap_count=0, + total_gap_count=0, + errors=[], + warnings=[], + ) + runtime = SimpleNamespace( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + config=SimpleNamespace(base_dir=Path(tmpdir)), + state_repository=SimpleNamespace(write_runtime_artifact=lambda **kwargs: calls.append(kwargs) or False), + ) + + persisted = write_runtime_readiness_report( + runtime, + run_dir=run_dir, + readiness_result=result, + ) + + self.assertFalse(persisted) + self.assertEqual(calls[0]["run_id"], "run-retired") + self.assertEqual(calls[0]["artifact_name"], "runtime_readiness_report.json") + self.assertFalse((runtime_root / "runtime_readiness_report.json").exists()) + self.assertFalse((legacy_root / "runtime_readiness_report.json").exists()) + self.assertFalse((run_dir / "runtime_readiness_report.json").exists()) + def test_write_runtime_readiness_report_reuses_cached_payload_when_report_hash_matches(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo_root = Path(tmpdir) / "repo" @@ -249,10 +322,16 @@ def test_write_runtime_readiness_report_reuses_cached_payload_when_report_hash_m run_dir.mkdir() report_text = json.dumps({"generated_at": "now", "summary": {"total_gap_count": 0}}, sort_keys=True) report_hash = hashlib.sha256(report_text.encode("utf-8")).hexdigest() - matrix_text = json.dumps({"generated_at": "now", "features": [], "summary": {"feature_count": 0}}, sort_keys=True) - matrix_hash = hashlib.sha256((json.dumps(json.loads(matrix_text), indent=2, sort_keys=True) + "\n").encode("utf-8")).hexdigest() + matrix_text = json.dumps( + {"generated_at": "now", "features": [], "summary": {"feature_count": 0}}, sort_keys=True + ) + matrix_hash = hashlib.sha256( + (json.dumps(json.loads(matrix_text), indent=2, sort_keys=True) + "\n").encode("utf-8") + ).hexdigest() gap_report_path.write_text(report_text, encoding="utf-8") - matrix_path.write_text(json.dumps(json.loads(matrix_text), indent=2, sort_keys=True) + "\n", encoding="utf-8") + matrix_path.write_text( + json.dumps(json.loads(matrix_text), indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) manifest_path.write_text( json.dumps({"generated_at": "now", "commands": {}, "modes": {}}, sort_keys=True), encoding="utf-8" ) diff --git a/tests/python/runtime/test_engine_runtime_command_parity_state_config.py b/tests/python/runtime/test_engine_runtime_command_parity_state_config.py index a63b11e1b..374fcbed2 100644 --- a/tests/python/runtime/test_engine_runtime_command_parity_state_config.py +++ b/tests/python/runtime/test_engine_runtime_command_parity_state_config.py @@ -2,10 +2,10 @@ from __future__ import annotations from tests.python.runtime.engine_runtime_command_parity_test_support import * +from envctl_engine.state.run_index import StateSelector class EngineRuntimeCommandParityStateConfigTests(EngineRuntimeCommandParityTestCase): - def test_list_targets_discovers_projects_in_main_mode(self) -> None: tmpdir = tempfile.TemporaryDirectory() self.addCleanup(tmpdir.cleanup) @@ -82,7 +82,11 @@ def test_list_trees_json_reports_selected_state_presence_and_service_truth(self) state=RunState( run_id="run-prev", mode="trees", - services={"feature-1 Backend": ServiceRecord(name="feature-1 Backend", type="backend", cwd=".", status="unknown")}, + services={ + "feature-1 Backend": ServiceRecord( + name="feature-1 Backend", type="backend", cwd=".", status="unknown" + ) + }, requirements={}, metadata={"repo_scope_id": config.runtime_scope_id, "project_roots": {"feature-1": str(feature_dir)}}, ), @@ -104,6 +108,66 @@ def test_list_trees_json_reports_selected_state_presence_and_service_truth(self) self.assertFalse(feature["services_running"]) self.assertFalse(feature["running"]) + def test_tree_list_json_reports_discovery_and_integrity_warning_for_tampered_active_run(self) -> None: + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + repo = Path(tmpdir.name) / "repo" + runtime_root = Path(tmpdir.name) / "runtime" + (repo / ".git").mkdir(parents=True, exist_ok=True) + feature_dirs = { + name: repo / "trees" / "feature" / suffix for name, suffix in (("feature-1", "1"), ("feature-2", "2")) + } + for feature_dir in feature_dirs.values(): + feature_dir.mkdir(parents=True, exist_ok=True) + (feature_dir / ".envctl").write_text("ENVCTL_DEFAULT_MODE=trees\n") + config = load_config({"RUN_REPO_ROOT": str(repo), "RUN_SH_RUNTIME_DIR": str(runtime_root)}) + runtime = PythonEngineRuntime(config, env={}) + + for run_id, project_name in (("run-valid", "feature-1"), ("run-tampered", "feature-2")): + runtime.state_repository.save_resume_state( + state=RunState( + run_id=run_id, + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=str(feature_dirs[project_name]), + project=project_name, + ) + }, + requirements={}, + metadata={ + "repo_scope_id": config.runtime_scope_id, + "project_roots": {project_name: str(feature_dirs[project_name])}, + }, + ), + emit=lambda *args, **kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + tampered = next( + candidate + for candidate in runtime.state_repository.run_index.candidates( + StateSelector(mode="trees", project_names=()) + ) + if candidate.run_id == "run-tampered" + ) + tampered.state_path.write_text('{"broken":', encoding="utf-8") + + for args in (("--list-trees", "--json"), ("--list-targets", "--trees", "--json")): + with self.subTest(args=args): + buffer = StringIO() + with redirect_stdout(buffer): + code = runtime.dispatch(parse_route(list(args), env={})) + + self.assertEqual(code, 0) + payload = json.loads(buffer.getvalue()) + self.assertEqual({project["name"] for project in payload["projects"]}, set(feature_dirs)) + self.assertEqual(payload["state_error"]["code"], "runtime_state_integrity_error") + self.assertIn("run-tampered", payload["state_error"]["message"]) + self.assertEqual(payload["warnings"], [payload["state_error"]]) + self.assertFalse(any(project["state_present"] for project in payload["projects"])) + def test_list_trees_json_reports_imported_branch_worktrees_individually(self) -> None: tmpdir = tempfile.TemporaryDirectory() self.addCleanup(tmpdir.cleanup) @@ -289,7 +353,10 @@ def test_show_config_json_reports_additional_service_validation_errors_and_metad code = runtime.dispatch(parse_route(["--show-config", "--json"], env={})) self.assertEqual(code, 0) payload = json.loads(buffer.getvalue()) - self.assertIn("additional service dependency cycle: voice-runtime -> worker -> voice-runtime", payload["additional_service_errors"]) + self.assertIn( + "additional service dependency cycle: voice-runtime -> worker -> voice-runtime", + payload["additional_service_errors"], + ) self.assertTrue(payload["service_dependency_env_section_present"]["voice-runtime"]) self.assertEqual(payload["service_dependency_env_templates"]["voice-runtime"][0]["name"], "VOICE_PUBLIC") diff --git a/tests/python/runtime/test_engine_runtime_dispatch.py b/tests/python/runtime/test_engine_runtime_dispatch.py index ec0b61560..27d5a4367 100644 --- a/tests/python/runtime/test_engine_runtime_dispatch.py +++ b/tests/python/runtime/test_engine_runtime_dispatch.py @@ -1,29 +1,40 @@ from __future__ import annotations import importlib -from pathlib import Path +import tempfile +import threading import unittest -from io import StringIO from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path from types import SimpleNamespace -import tempfile from unittest.mock import patch +from envctl_engine.state.repository import RuntimeStateRepository + REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" dispatch_module = importlib.import_module("envctl_engine.runtime.engine_runtime_dispatch") +command_policy_module = importlib.import_module("envctl_engine.runtime.command_policy") +lifecycle_lease_module = importlib.import_module("envctl_engine.runtime.lifecycle_operation_lease") utility_module = importlib.import_module("envctl_engine.runtime.utility_command_support") dispatch = dispatch_module.dispatch dispatch_command = dispatch_module.dispatch_command class EngineRuntimeDispatchTests(unittest.TestCase): + def _runtime_root(self) -> Path: + temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(temporary_directory.cleanup) + return Path(temporary_directory.name) + def test_dispatch_entry_configures_probe_debug_recorder_and_route_events(self) -> None: events: list[tuple[str, dict[str, object]]] = [] calls: list[str] = [] route = SimpleNamespace(command="start", mode="main") runtime = SimpleNamespace( + runtime_root=self._runtime_root(), process_probe=None, _build_process_probe_backend=lambda: calls.append("probe_backend") or "backend", _effective_start_mode=lambda value: calls.append(f"effective:{value.command}") or "trees", @@ -87,6 +98,213 @@ def test_dispatch_entry_uses_route_mode_for_non_startup_commands(self) -> None: self.assertEqual(code, 4) self.assertTrue(all(payload["effective_mode"] == "trees" for _event, payload in events)) + def test_exclusive_operation_commands_follow_mutating_command_policy_sets(self) -> None: + expected = { + *command_policy_module.LIFECYCLE_CLEANUP_COMMANDS, + *command_policy_module.STARTUP_COMMANDS, + "blast-worktree", + "delete-worktree", + "ensure-worktree", + "resume", + "self-destruct-worktree", + } + + self.assertEqual(dispatch_module._EXCLUSIVE_OPERATION_COMMANDS, frozenset(expected)) + self.assertIn("import", expected) + self.assertIn("self-destruct-worktree", expected) + self.assertNotIn("test-focused", expected) + self.assertNotIn("ship", expected) + + def test_nested_lifecycle_dispatch_is_reentrant_for_owning_runtime_thread(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + commands: list[str] = [] + runtime = SimpleNamespace( + runtime_root=self._runtime_root(), + process_probe=None, + _build_process_probe_backend=lambda: "backend", + _effective_start_mode=lambda route: route.mode, + _configure_debug_recorder=lambda _route: None, + _emit=lambda event, **payload: events.append((event, payload)), + ) + outer_route = SimpleNamespace(command="start", mode="trees") + nested_route = SimpleNamespace(command="stop", mode="trees") + + def command_dispatch(_runtime: object, route: object) -> int: + command = str(getattr(route, "command", "")) + commands.append(command) + if command == "start": + return dispatch(runtime, nested_route) + return 7 + + with ( + patch("envctl_engine.runtime.engine_runtime_dispatch.ProcessProbe", return_value="probe"), + patch( + "envctl_engine.runtime.engine_runtime_dispatch.dispatch_command", + side_effect=command_dispatch, + ), + ): + code = dispatch(runtime, outer_route) + + self.assertEqual(code, 7) + self.assertEqual(commands, ["start", "stop"]) + self.assertFalse(any(event == "lifecycle.operation.busy" for event, _payload in events)) + self.assertIsNone(getattr(runtime, "_lifecycle_operation_owner", None)) + self.assertEqual(getattr(runtime, "_lifecycle_operation_depth", 0), 0) + + def test_concurrent_lifecycle_dispatch_on_another_thread_fails_busy_without_overlap(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + commands: list[str] = [] + entered = threading.Event() + release = threading.Event() + results: list[int] = [] + errors: list[BaseException] = [] + runtime = SimpleNamespace( + runtime_root=self._runtime_root(), + process_probe=None, + _build_process_probe_backend=lambda: "backend", + _effective_start_mode=lambda route: route.mode, + _configure_debug_recorder=lambda _route: None, + _emit=lambda event, **payload: events.append((event, payload)), + ) + outer_route = SimpleNamespace(command="start", mode="trees") + concurrent_route = SimpleNamespace(command="stop", mode="trees") + + def command_dispatch(_runtime: object, route: object) -> int: + command = str(getattr(route, "command", "")) + commands.append(command) + if command == "start": + entered.set() + if not release.wait(timeout=2): + raise AssertionError("timed out waiting to release lifecycle operation") + return 0 + + def run_outer() -> None: + try: + results.append(dispatch(runtime, outer_route)) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + with ( + patch("envctl_engine.runtime.engine_runtime_dispatch.ProcessProbe", return_value="probe"), + patch( + "envctl_engine.runtime.engine_runtime_dispatch.dispatch_command", + side_effect=command_dispatch, + ), + ): + thread = threading.Thread(target=run_outer, name="envctl-dispatch-lock-test") + thread.start() + try: + self.assertTrue(entered.wait(timeout=2)) + output = StringIO() + with redirect_stdout(output): + concurrent_code = dispatch(runtime, concurrent_route) + finally: + release.set() + thread.join(timeout=2) + + self.assertFalse(thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(results, [0]) + self.assertEqual(concurrent_code, 1) + self.assertEqual(commands, ["start"]) + self.assertIn("Another envctl lifecycle operation is already active", output.getvalue()) + self.assertTrue(any(event == "lifecycle.operation.busy" for event, _payload in events)) + + def test_committed_start_releases_lease_before_interactive_wait_so_stop_can_run(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + commands: list[str] = [] + interactive_wait_entered = threading.Event() + release_interactive_wait = threading.Event() + results: list[int] = [] + runtime = SimpleNamespace( + runtime_root=self._runtime_root(), + process_probe=None, + _build_process_probe_backend=lambda: "backend", + _effective_start_mode=lambda route: route.mode, + _configure_debug_recorder=lambda _route: None, + _emit=lambda event, **payload: events.append((event, payload)), + ) + start_route = SimpleNamespace(command="start", mode="trees") + stop_route = SimpleNamespace(command="stop", mode="trees") + + def command_dispatch(_runtime: object, route: object) -> int: + command = str(getattr(route, "command", "")) + commands.append(command) + if command == "start": + released = lifecycle_lease_module.release_lifecycle_operation(runtime) + self.assertTrue(released) + interactive_wait_entered.set() + self.assertTrue(release_interactive_wait.wait(timeout=2)) + return 0 + + def run_start() -> None: + results.append(dispatch(runtime, start_route)) + + with ( + patch("envctl_engine.runtime.engine_runtime_dispatch.ProcessProbe", return_value="probe"), + patch( + "envctl_engine.runtime.engine_runtime_dispatch.dispatch_command", + side_effect=command_dispatch, + ), + ): + thread = threading.Thread(target=run_start, name="envctl-interactive-start-test") + thread.start() + try: + self.assertTrue(interactive_wait_entered.wait(timeout=2)) + stop_code = dispatch(runtime, stop_route) + finally: + release_interactive_wait.set() + thread.join(timeout=2) + + self.assertFalse(thread.is_alive()) + self.assertEqual(results, [0]) + self.assertEqual(stop_code, 0) + self.assertEqual(commands, ["start", "stop"]) + self.assertFalse(any(event == "lifecycle.operation.busy" for event, _payload in events)) + + def test_operation_lock_rejects_retargeted_repository_root_before_external_write(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + runtime_root.mkdir(parents=True) + legacy_root.mkdir() + repository = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + external = Path(tmpdir) / "external" + external.mkdir() + sentinel = external / "sentinel.txt" + sentinel.write_text("untouched", encoding="utf-8") + runtime_root.rmdir() + runtime_root.symlink_to(external, target_is_directory=True) + runtime = SimpleNamespace( + runtime_root=runtime_root, + state_repository=repository, + process_probe=None, + _build_process_probe_backend=lambda: "backend", + _effective_start_mode=lambda route: route.mode, + _configure_debug_recorder=lambda _route: None, + _emit=lambda *_args, **_kwargs: None, + ) + + with ( + patch("envctl_engine.runtime.engine_runtime_dispatch.ProcessProbe", return_value="probe"), + patch( + "envctl_engine.runtime.engine_runtime_dispatch.dispatch_command", + side_effect=AssertionError("command must not run"), + ), + self.assertRaisesRegex(RuntimeError, "symlink"), + ): + dispatch(runtime, SimpleNamespace(command="start", mode="trees")) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "untouched") + self.assertEqual(list(external.iterdir()), [sentinel]) + def test_list_commands_dispatch_prints_supported_commands(self) -> None: runtime = SimpleNamespace() route = SimpleNamespace(command="list-commands", mode="main") diff --git a/tests/python/runtime/test_engine_runtime_event_support.py b/tests/python/runtime/test_engine_runtime_event_support.py index c0812ebe1..04db99b74 100644 --- a/tests/python/runtime/test_engine_runtime_event_support.py +++ b/tests/python/runtime/test_engine_runtime_event_support.py @@ -53,6 +53,32 @@ def test_emit_records_event_and_notifies_listener(self) -> None: self.assertEqual(received[0][0], "ui.input.submit") self.assertIn("command_hash", received[0][1]) + def test_emit_disables_failed_debug_recorder_without_interrupting_lifecycle(self) -> None: + received: list[str] = [] + + class FailedRecorder: + def record(self, *_args: object, **_kwargs: object) -> None: + raise OSError("debug disk unavailable") + + runtime = SimpleNamespace( + _active_command_id="cmd-1", + _debug_hash_salt="salt", + _emit_lock=threading.Lock(), + events=[], + _debug_recorder=FailedRecorder(), + _debug_recorder_failure=None, + _emit_listeners=[lambda event, _payload: received.append(event)], + env={}, + config=SimpleNamespace(raw={}), + ) + + emit(runtime, "service.start", project="Main") + + self.assertEqual(runtime.events[0]["event"], "service.start") + self.assertEqual(received, ["service.start"]) + self.assertIsNone(runtime._debug_recorder) + self.assertEqual(runtime._debug_recorder_failure, "debug disk unavailable") + def test_debug_should_auto_pack_policies(self) -> None: runtime = SimpleNamespace(env={"ENVCTL_DEBUG_AUTO_PACK": "anomaly"}, config=SimpleNamespace(raw={})) diff --git a/tests/python/runtime/test_engine_runtime_lifecycle_support.py b/tests/python/runtime/test_engine_runtime_lifecycle_support.py index 78647f164..6ca190ecc 100644 --- a/tests/python/runtime/test_engine_runtime_lifecycle_support.py +++ b/tests/python/runtime/test_engine_runtime_lifecycle_support.py @@ -36,12 +36,17 @@ def test_release_requirement_ports_releases_enabled_final_ports(self) -> None: project="Main", db={"enabled": True, "final": 5432}, redis={"enabled": False, "final": 6379}, + supabase={ + "enabled": True, + "final": 15432, + "resources": {"db": 15432, "api": 15433}, + }, n8n={"enabled": True, "final": 5678}, ) release_requirement_ports(runtime, requirements) - self.assertEqual(released, [5432, 5678]) + self.assertEqual(released, [5432, 5678, 15432, 15433]) def test_release_requirement_ports_skips_external_dependency_ports(self) -> None: released: list[int] = [] @@ -108,10 +113,16 @@ def test_terminate_services_from_state_releases_ports_for_terminated_services(se def test_terminate_service_record_prefers_process_group_termination(self) -> None: calls: list[tuple[str, int]] = [] + live_pids = {3210} runtime = SimpleNamespace( _emit=lambda *args, **kwargs: None, process_runner=SimpleNamespace( - terminate_process_group=lambda pid, **kwargs: calls.append(("group", pid)) or True, + is_pid_running=lambda pid: pid in live_pids, + terminate_process_group=lambda pid, **kwargs: ( + calls.append(("group", pid)), + live_pids.discard(pid), + True, + )[-1], terminate=lambda pid, **kwargs: calls.append(("single", pid)) or True, ), ) @@ -129,6 +140,7 @@ def test_terminate_service_record_prefers_process_group_termination(self) -> Non def test_terminate_service_record_accepts_verified_listener_child_ownership(self) -> None: calls: list[tuple[str, int]] = [] skips: list[dict[str, object]] = [] + live_pids = {111, 222} def pid_owns_port(pid: int, port: int) -> bool: return pid == 222 and port == 9000 @@ -136,8 +148,13 @@ def pid_owns_port(pid: int, port: int) -> bool: runtime = SimpleNamespace( _emit=lambda event, **payload: skips.append(payload) if event == "cleanup.skip" else None, process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, pid_owns_port=pid_owns_port, - terminate_process_group=lambda pid, **kwargs: calls.append(("group", pid)) or True, + terminate_process_group=lambda pid, **kwargs: ( + calls.append(("group", pid)), + live_pids.discard(pid), + True, + )[-1], terminate=lambda pid, **kwargs: calls.append(("single", pid)) or True, ), ) @@ -150,7 +167,7 @@ def pid_owns_port(pid: int, port: int) -> bool: ) self.assertTrue(terminated) - self.assertEqual(calls, [("group", 111)]) + self.assertEqual(calls, [("group", 111), ("group", 222)]) self.assertEqual(skips, []) def test_blast_worktree_before_delete_updates_state_and_emits_finish(self) -> None: @@ -366,7 +383,10 @@ def _resolve_proc_path(path_obj: Path, strict: bool = False) -> Path: # noqa: A return path_obj with ( - patch("envctl_engine.runtime.engine_runtime_lifecycle_support.Path.iterdir", return_value=[proc_root / "111", proc_root / "222"]), + patch( + "envctl_engine.runtime.engine_runtime_lifecycle_support.Path.iterdir", + return_value=[proc_root / "111", proc_root / "222"], + ), patch("pathlib.Path.resolve", autospec=True, side_effect=_resolve_proc_path), patch( "envctl_engine.runtime.engine_runtime_lifecycle_support._remove_tree_containers", diff --git a/tests/python/runtime/test_engine_runtime_resume_startup.py b/tests/python/runtime/test_engine_runtime_resume_startup.py index b7a62022f..30bc9014b 100644 --- a/tests/python/runtime/test_engine_runtime_resume_startup.py +++ b/tests/python/runtime/test_engine_runtime_resume_startup.py @@ -214,7 +214,9 @@ def fake_result(context: ProjectContext) -> ProjectStartupResult: first_engine = PythonEngineRuntime(self._config(repo, runtime), env={}) first_engine._reconcile_state_truth = lambda _state: [] # type: ignore[assignment] - with patch.object(first_engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"])): + with patch.object( + first_engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"]) + ): first_code = first_engine.dispatch(parse_route(["--plan", "feature-a", "--batch"], env={})) self.assertEqual(first_code, 0) first_state = first_engine.state_repository.load_latest(mode="trees", strict_mode_match=True) @@ -223,7 +225,9 @@ def fake_result(context: ProjectContext) -> ProjectStartupResult: second_engine = PythonEngineRuntime(self._config(repo, runtime, extra={"BACKEND_DIR": "api"}), env={}) second_engine._reconcile_state_truth = lambda _state: [] # type: ignore[assignment] - with patch.object(second_engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"])): + with patch.object( + second_engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"]) + ): second_code = second_engine.dispatch(parse_route(["--plan", "feature-a", "--batch"], env={})) self.assertEqual(second_code, 0) @@ -242,7 +246,9 @@ def test_project_root_change_forces_fresh_run(self) -> None: (tree / "frontend").mkdir(parents=True, exist_ok=True) engine = PythonEngineRuntime(self._config(repo, runtime), env={}) context = engine._discover_projects(mode="trees")[0] - wrong_root_context = ProjectContext(name=context.name, root=repo / "trees" / "feature-a" / "99", ports=context.ports) + wrong_root_context = ProjectContext( + name=context.name, root=repo / "trees" / "feature-a" / "99", ports=context.ports + ) metadata = startup_support.build_startup_identity_metadata( engine, runtime_mode="trees", @@ -287,7 +293,9 @@ def fake_result(selected_context: ProjectContext) -> ProjectStartupResult: ) engine._reconcile_state_truth = lambda _state: [] # type: ignore[assignment] - with patch.object(engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"])): + with patch.object( + engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"]) + ): code = engine.dispatch(parse_route(["--plan", "feature-a", "--batch"], env={})) self.assertEqual(code, 0) @@ -324,6 +332,9 @@ def test_resume_rejects_state_without_resumable_services(self) -> None: with ( patch.object(engine, "_try_load_existing_state", return_value=state), patch.object(engine, "_run_interactive_dashboard_loop", return_value=0) as loop_mock, + patch( + "envctl_engine.startup.resume_orchestrator.release_lifecycle_operation" + ) as release_lifecycle_operation_mock, patch("sys.stdin.isatty", return_value=True), patch("sys.stdout.isatty", return_value=True), patch.dict(os.environ, {"TERM": "xterm-256color"}, clear=False), @@ -333,6 +344,7 @@ def test_resume_rejects_state_without_resumable_services(self) -> None: self.assertEqual(code, 1) self.assertEqual(loop_mock.call_count, 0) + release_lifecycle_operation_mock.assert_not_called() self.assertIn("No active services to resume.", output.getvalue()) def test_dashboard_interactive_flag_enters_loop(self) -> None: @@ -441,4 +453,3 @@ def test_resume_batch_flag_skips_interactive_loop(self) -> None: self.assertEqual(code, 0) self.assertEqual(loop_mock.call_count, 0) - diff --git a/tests/python/runtime/test_engine_runtime_startup_support.py b/tests/python/runtime/test_engine_runtime_startup_support.py index ca8a73908..988d3618a 100644 --- a/tests/python/runtime/test_engine_runtime_startup_support.py +++ b/tests/python/runtime/test_engine_runtime_startup_support.py @@ -61,17 +61,15 @@ def _reuse_runtime( "backend": backend_enabled, "frontend": frontend_enabled, }.get(str(service_name).strip().lower(), False), - requirement_enabled_for_mode=lambda _mode, requirement_name: str(requirement_name).strip().lower() - in enabled_dependencies, + requirement_enabled_for_mode=lambda _mode, requirement_name: ( + str(requirement_name).strip().lower() in enabled_dependencies + ), ) return SimpleNamespace( env=dict(external_env or {}), config=config, _try_load_existing_state=lambda **_kwargs: state, - _project_name_from_service=lambda name: str(name) - .replace(" Backend", "") - .replace(" Frontend", "") - .strip(), + _project_name_from_service=lambda name: str(name).replace(" Backend", "").replace(" Frontend", "").strip(), ) def test_effective_start_mode_switches_main_to_trees_for_setup_worktree(self) -> None: @@ -144,6 +142,61 @@ def test_run_reuse_accepts_exact_project_roots_and_startup_identity_match(self) self.assertEqual(decision.reason, "exact_match") self.assertIs(decision.candidate_state, state) + def test_run_reuse_forwards_selected_project_names_to_state_lookup(self) -> None: + context = ProjectContext(name="feature-a", root=Path("/repo/trees/feature-a"), ports={}) + state = RunState( + run_id="run-a", + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/repo/trees/feature-a/backend", + ) + }, + metadata={"project_roots": {"feature-a": str(context.root)}}, + ) + runtime = self._reuse_runtime(state=state) + calls: list[dict[str, object]] = [] + runtime._try_load_existing_state = lambda **kwargs: calls.append(kwargs) or state + + startup_support.evaluate_run_reuse( + runtime, + runtime_mode="trees", + route=Route(command="start", mode="trees"), + contexts=[context], + ) + + self.assertEqual( + calls, + [{"mode": "trees", "strict_mode_match": True, "project_names": ["feature-a"]}], + ) + + def test_explicit_no_resume_keeps_matching_candidate_for_safe_replacement(self) -> None: + context = ProjectContext(name="feature-a", root=Path("/repo/trees/feature-a"), ports={}) + state = RunState( + run_id="run-a", + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/repo/trees/feature-a/backend", + ) + }, + metadata={"project_roots": {"feature-a": str(context.root)}}, + ) + decision = startup_support.evaluate_run_reuse( + self._reuse_runtime(state=state), + runtime_mode="trees", + route=Route(command="start", mode="trees", flags={"no_resume": True}), + contexts=[context], + ) + + self.assertEqual(decision.decision_kind, "fresh_run") + self.assertEqual(decision.reason, "explicit_no_resume") + self.assertIs(decision.candidate_state, state) + def test_run_reuse_rejects_project_root_mismatch_even_when_names_match(self) -> None: context = ProjectContext(name="feature-a-1", root=Path("/repo/trees/feature-a/1"), ports={}) route = Route(command="plan", mode="trees", raw_args=[], passthrough_args=[], projects=["feature-a"], flags={}) @@ -571,6 +624,143 @@ def test_discover_projects_and_reserve_project_ports(self) -> None: reserve_project_ports(runtime, context) self.assertEqual(events[-1][0], "port.reserved") + def test_no_infra_reserves_no_application_or_dependency_ports(self) -> None: + reserve_calls: list[tuple[int, str]] = [] + events: list[tuple[str, dict[str, object]]] = [] + plans = { + name: PortPlan(project="Main", requested=port, assigned=port, final=port, source="fixed") + for name, port in { + "backend": 8000, + "frontend": 9000, + "db": 5432, + "redis": 6379, + "n8n": 5678, + "supabase_api": 54321, + "worker": 9100, + }.items() + } + runtime = SimpleNamespace( + port_planner=SimpleNamespace( + reserve_next=lambda assigned, owner: reserve_calls.append((assigned, owner)) or assigned, + max_port=65000, + availability_mode="lock", + update_final_port=lambda plan, reserved, source: None, + ), + _emit=lambda event, **payload: events.append((event, payload)), + _lock_inventory=lambda: [], + _requirement_enabled=lambda *_args, **_kwargs: self.fail("disabled dependencies need no lookup"), + ) + route = Route( + command="start", + mode="main", + flags={"launch_backend": False, "launch_frontend": False, "launch_dependencies": False}, + ) + + reserve_project_ports( + runtime, + ProjectContext(name="Main", root=Path("/repo"), ports=plans), + route=route, + ) + + self.assertEqual(reserve_calls, []) + self.assertEqual( + [payload["service"] for event, payload in events if event == "port.reservation.skipped"], + list(plans), + ) + + def test_disabled_requirements_do_not_reserve_dependency_ports(self) -> None: + reserve_calls: list[tuple[int, str]] = [] + plans = { + "backend": PortPlan("Main", 8000, 8000, 8000, "fixed"), + "db": PortPlan("Main", 5432, 5432, 5432, "fixed"), + "redis": PortPlan("Main", 6379, 6379, 6379, "fixed"), + "n8n": PortPlan("Main", 5678, 5678, 5678, "fixed"), + "supabase_api": PortPlan("Main", 54321, 54321, 54321, "fixed"), + } + runtime = SimpleNamespace( + port_planner=SimpleNamespace( + reserve_next=lambda assigned, owner: reserve_calls.append((assigned, owner)) or assigned, + max_port=65000, + availability_mode="lock", + update_final_port=lambda plan, reserved, source: None, + ), + _emit=lambda *_args, **_kwargs: None, + _lock_inventory=lambda: [], + _requirement_enabled=lambda *_args, **_kwargs: False, + ) + + reserve_project_ports( + runtime, + ProjectContext(name="Main", root=Path("/repo"), ports=plans), + route=Route(command="start", mode="main", flags={"runtime_scope": "entire-system"}), + ) + + self.assertEqual(reserve_calls, [(8000, "Main:backend")]) + + def test_port_reservations_match_the_selected_runtime_surface(self) -> None: + voice = SimpleNamespace( + name="voice", + depends_on=(), + enabled_for_project_root=lambda _mode, _root: True, + ) + cases = ( + ("frontend only", {"runtime_scope": "frontend"}, {"frontend", "db"}), + ("backend only", {"runtime_scope": "backend"}, {"backend", "db"}), + ("dependencies only", {"runtime_scope": "dependencies"}, {"db"}), + ( + "no infra", + {"launch_backend": False, "launch_frontend": False, "launch_dependencies": False}, + set(), + ), + ("selected additional service", {"services": ["Main Voice"]}, {"voice", "db"}), + ) + + for label, flags, expected_services in cases: + with self.subTest(label): + reserve_calls: list[tuple[int, str]] = [] + plans = { + name: PortPlan("Main", port, port, port, "fixed") + for name, port in { + "backend": 8000, + "frontend": 9000, + "voice": 9100, + "db": 5432, + "redis": 6379, + "n8n": 5678, + "supabase_api": 54321, + }.items() + } + runtime = SimpleNamespace( + config=SimpleNamespace( + additional_services=(voice,), + all_app_service_names_for_mode=lambda _mode, project_root=None: ( + "backend", + "frontend", + "voice", + ), + ), + port_planner=SimpleNamespace( + reserve_next=lambda assigned, owner: reserve_calls.append((assigned, owner)) or assigned, + max_port=65000, + availability_mode="lock", + update_final_port=lambda plan, reserved, source: None, + ), + _emit=lambda *_args, **_kwargs: None, + _lock_inventory=lambda: [], + _requirement_enabled=lambda requirement, **_kwargs: requirement == "postgres", + ) + + reserve_project_ports( + runtime, + ProjectContext(name="Main", root=Path("/repo"), ports=plans), + route=Route(command="start", mode="main", flags=flags), + ) + + self.assertEqual( + {owner.removeprefix("Main:") for _port, owner in reserve_calls}, + expected_services, + ) + def test_reserve_project_ports_releases_matching_restart_lock_before_reuse(self) -> None: events: list[tuple[str, dict[str, object]]] = [] released: list[tuple[int, str | None]] = [] @@ -627,7 +817,9 @@ def update_final_port(plan: PortPlan, reserved: int, source: str) -> None: ) context = ProjectContext(name="Main", root=Path("/repo"), ports={"backend": port_plan}) - reserve_project_ports(runtime, context, route=Route(command="start", mode="main", flags={"interactive_command": True})) + reserve_project_ports( + runtime, context, route=Route(command="start", mode="main", flags={"interactive_command": True}) + ) rebound_payloads = [payload for event, payload in events if event == "port.rebound"] self.assertEqual(len(rebound_payloads), 1) diff --git a/tests/python/runtime/test_engine_runtime_state_lookup.py b/tests/python/runtime/test_engine_runtime_state_lookup.py index 4e3eba439..3f9b5c2a1 100644 --- a/tests/python/runtime/test_engine_runtime_state_lookup.py +++ b/tests/python/runtime/test_engine_runtime_state_lookup.py @@ -18,19 +18,29 @@ class EngineRuntimeStateLookupTests(unittest.TestCase): def test_try_load_existing_state_emits_fingerprint_after_reload(self) -> None: state = RunState(run_id="run-1", mode="main") events: list[tuple[str, dict[str, object]]] = [] + load_calls: list[dict[str, object]] = [] runtime = SimpleNamespace( - state_repository=SimpleNamespace(load_latest=lambda **kwargs: state), + state_repository=SimpleNamespace(load_latest=lambda **kwargs: load_calls.append(kwargs) or state), _emit=lambda event, **payload: events.append((event, payload)), _state_fingerprint=lambda loaded: f"fp:{loaded.run_id}", ) - loaded = try_load_existing_state(runtime, mode="main", strict_mode_match=True) + loaded = try_load_existing_state( + runtime, + mode="main", + strict_mode_match=True, + project_names=["Main"], + ) self.assertIs(loaded, state) self.assertEqual( events, [("state.fingerprint.after_reload", {"run_id": "run-1", "state_fingerprint": "fp:run-1"})], ) + self.assertEqual( + load_calls, + [{"mode": "main", "strict_mode_match": True, "project_names": ["Main"]}], + ) def test_state_matches_scope_accepts_missing_scope_and_matches_exact_scope(self) -> None: runtime = SimpleNamespace(config=SimpleNamespace(runtime_scope_id="repo-123")) diff --git a/tests/python/runtime/test_help_topics.py b/tests/python/runtime/test_help_topics.py index b84c8ebad..97dc17a99 100644 --- a/tests/python/runtime/test_help_topics.py +++ b/tests/python/runtime/test_help_topics.py @@ -29,6 +29,15 @@ def test_pr_preview_controller_help_topic_is_available(self) -> None: self.assertIn("envctl pr-preview-controller - run the GitHub PR-label preview controller", text) self.assertIn("--command ", text) + def test_stop_help_distinguishes_detach_from_managed_docker_removal(self) -> None: + stop_text = render_command_help(COMMAND_HELP_TOPICS["stop"]) + stop_all_text = render_command_help(COMMAND_HELP_TOPICS["stop-all"]) + + self.assertIn("managed Docker stacks and storage remain reusable", stop_text) + self.assertIn("stop-all --remove-volumes", stop_text) + self.assertIn("releases dependency records and port locks", stop_all_text) + self.assertIn("remove supported managed containers and volumes", stop_all_text) + def test_help_text_for_route_returns_none_for_general_help(self) -> None: self.assertIsNone(help_text_for_route(parse_route(["--help"], env={}))) diff --git a/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py b/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py index d28f7f304..cff6c5e24 100644 --- a/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py +++ b/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py @@ -1,17 +1,26 @@ from __future__ import annotations +import json +import tempfile +import unittest from contextlib import contextmanager, redirect_stdout from io import StringIO from pathlib import Path from types import SimpleNamespace -import unittest from unittest.mock import patch REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.lifecycle_cleanup_orchestrator import LifecycleCleanupOrchestrator +from envctl_engine.runtime.lifecycle_service_termination import ( + terminate_service_record, + terminate_services_from_state, +) +from envctl_engine.shared.ports import PortPlanner from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord +from envctl_engine.state.repository import RuntimeStateRepository +from envctl_engine.state.run_index import StateSelector from envctl_engine.ui.spinner_service import SpinnerPolicy from envctl_engine.ui.target_selector import TargetSelection @@ -19,16 +28,43 @@ class _StateRepoStub: def __init__(self) -> None: self.saved_states: list[RunState] = [] + self.saved_authoritative_project_names: list[list[str] | None] = [] self.purge_calls: list[bool] = [] + self.active_states: list[RunState] = [] + self.deactivated_run_ids: list[str] = [] - def save_selected_stop_state(self, *, state, emit, runtime_map_builder): # noqa: ANN001 + def save_selected_stop_state( # noqa: ANN001 + self, + *, + state, + emit, + runtime_map_builder, + authoritative_project_names=None, + ): _ = emit, runtime_map_builder self.saved_states.append(state) + self.saved_authoritative_project_names.append(authoritative_project_names) + self.active_states = [ + state if active_state.run_id == state.run_id else active_state for active_state in self.active_states + ] return {} def purge(self, *, aggressive: bool = False) -> None: self.purge_calls.append(aggressive) + def load_all(self, *, mode: str | None = None) -> list[RunState]: + return [state for state in self.active_states if mode is None or state.mode == mode] + + def deactivate_runs(self, run_ids: list[str]) -> bool: + selected = set(run_ids) + removed = [state for state in self.active_states if state.run_id in selected] + self.active_states = [state for state in self.active_states if state.run_id not in selected] + self.deactivated_run_ids.extend(state.run_id for state in removed) + return bool(removed) + + def has_active_runs(self) -> bool: + return bool(self.active_states) + class _RuntimeStub: def __init__(self) -> None: @@ -165,6 +201,549 @@ def test_stop_all_tolerates_runtime_without_port_session_hook(self) -> None: self.assertEqual(runtime.state_repository.purge_calls, [False]) self.assertTrue(any(item.get("event") == "cleanup.stop_all" for item in runtime.events)) + def test_stop_all_releases_every_scoped_port_lock_after_last_run_is_removed(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime = _RuntimeStub() + scoped_planner = PortPlanner( + lock_dir=str(Path(tmpdir) / "scope-a"), + availability_checker=lambda _port: True, + ) + foreign_planner = PortPlanner( + lock_dir=str(Path(tmpdir) / "scope-b"), + availability_checker=lambda _port: True, + ) + scoped_planner.reserve_next(8100, owner="old-startup") + foreign_planner.reserve_next(8200, owner="other-repository") + runtime.port_planner = scoped_planner + + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(code, 0) + self.assertEqual(list(scoped_planner.lock_dir.glob("*.lock")), []) + self.assertEqual( + [path.name for path in foreign_planner.lock_dir.glob("*.lock")], + ["8200.lock"], + ) + self.assertEqual(runtime.state_repository.purge_calls, [False]) + + def test_stop_all_purge_failure_preserves_scoped_port_locks(self) -> None: + runtime = _RuntimeStub() + releases: list[str] = [] + runtime.port_planner = SimpleNamespace(release_all=lambda: releases.append("all")) + + with ( + patch.object(runtime.state_repository, "purge", side_effect=OSError("registry unavailable")), + self.assertRaisesRegex(OSError, "registry unavailable"), + ): + LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(releases, []) + + def test_stop_all_terminates_every_independent_active_run_once(self) -> None: + runtime = _RuntimeStub() + runtime.state_repository.active_states = [ + RunState(run_id="run-a", mode="trees"), + RunState(run_id="run-b", mode="trees"), + RunState(run_id="run-a", mode="trees"), + ] + terminated: list[str] = [] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: terminated.append(state.run_id) + ) + + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="trees")) + + self.assertEqual(code, 0) + self.assertEqual(terminated, ["run-a", "run-b"]) + self.assertEqual(runtime.state_repository.purge_calls, [False]) + + def test_stop_all_trees_preserves_main_registry_alias_and_port_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime_dir = root / "runtime" + repository = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + for run_id, mode, project in ( + ("run-main", "main", "Main"), + ("run-alpha", "trees", "FeatureA"), + ("run-beta", "trees", "FeatureB"), + ): + repository.save_resume_state( + state=RunState( + run_id=run_id, + mode=mode, + services={ + f"{project} Backend": ServiceRecord( + name=f"{project} Backend", + type="backend", + cwd=str(root / project / "backend"), + project=project, + pid=100 + len(project), + ) + }, + metadata={ + "project_names": [project], + "project_roots": {project: str(root / project)}, + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, + ) + + planner = PortPlanner( + lock_dir=str(repository.runtime_root / "locks"), + session_id="main-owner", + availability_checker=lambda _port: True, + ) + main_port = planner.reserve_next(8111, owner="Main:backend") + runtime = _RuntimeStub() + runtime.state_repository = repository # type: ignore[assignment] + runtime.port_planner = planner + terminated: list[str] = [] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: terminated.append(state.run_id) or set() + ) + + with patch.object(repository, "purge", wraps=repository.purge) as purge: + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="trees")) + + self.assertEqual(code, 0) + self.assertEqual(terminated, ["run-beta", "run-alpha"]) + purge.assert_not_called() + remaining = repository.load_all() + self.assertEqual([state.run_id for state in remaining], ["run-main"]) + self.assertEqual(repository.load_latest(mode="main", strict_mode_match=True).run_id, "run-main") + current = json.loads(repository.run_state_path().read_text(encoding="utf-8")) + self.assertEqual((current["run_id"], current["mode"]), ("run-main", "main")) + self.assertTrue((planner.lock_dir / f"{main_port}.lock").exists()) + + def test_mode_scoped_stop_all_ignores_tampered_other_mode_and_preserves_it(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repository = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + for run_id, mode, project, pid in ( + ("run-valid", "main", "Main", 101), + ("run-tampered", "trees", "FeatureA", 202), + ): + service_name = f"{project} Backend" + repository.save_resume_state( + state=RunState( + run_id=run_id, + mode=mode, + services={ + service_name: ServiceRecord( + name=service_name, + type="backend", + cwd=f"/tmp/{project}/backend", + project=project, + pid=pid, + ) + }, + metadata={"project_names": [project]}, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, + ) + + candidates = repository.run_index.candidates(StateSelector(mode=None, project_names=())) + tampered = next(candidate for candidate in candidates if candidate.run_id == "run-tampered") + tampered.state_path.write_text('{"broken":', encoding="utf-8") + + runtime = _RuntimeStub() + runtime.state_repository = repository # type: ignore[assignment] + termination_calls: list[str] = [] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: termination_calls.append(state.run_id) or set() + ) + + with patch.object(repository, "purge", wraps=repository.purge) as purge: + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(code, 0) + self.assertEqual(termination_calls, ["run-valid"]) + purge.assert_not_called() + remaining = repository.run_index.candidates(StateSelector(mode=None, project_names=())) + self.assertEqual({candidate.run_id for candidate in remaining}, {"run-tampered"}) + self.assertTrue(all(candidate.state_path.exists() for candidate in remaining)) + + def test_targeted_stop_deactivates_no_infra_state_with_only_disabled_requirements(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-no-infra", + mode="trees", + services={}, + requirements={"FeatureA": RequirementsResult(project="FeatureA")}, + metadata={ + "project_names": ["FeatureA"], + "project_roots": {"FeatureA": "/tmp/FeatureA"}, + }, + ) + runtime.state_repository.active_states = [state] + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"batch": True, "runtime_scope": "entire-system"}, + ) + ) + + self.assertEqual(code, 0) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-no-infra"]) + self.assertEqual(runtime.state_repository.active_states, []) + + def test_targeted_stop_removes_only_selected_project_from_shared_no_infra_state(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-no-infra-ab", + mode="trees", + services={}, + requirements={project: RequirementsResult(project=project) for project in ("FeatureA", "FeatureB")}, + metadata={ + "project_names": ["FeatureA", "FeatureB"], + "project_roots": { + "FeatureA": "/tmp/FeatureA", + "FeatureB": "/tmp/FeatureB", + }, + }, + ) + runtime.state_repository.active_states = [state] + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"batch": True, "runtime_scope": "entire-system"}, + ) + ) + + self.assertEqual(code, 0) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(len(runtime.state_repository.saved_states), 1) + saved = runtime.state_repository.saved_states[0] + self.assertEqual(set(saved.requirements), {"FeatureB"}) + self.assertEqual(saved.metadata["project_names"], ["FeatureB"]) + self.assertEqual(saved.metadata["project_roots"], {"FeatureB": "/tmp/FeatureB"}) + self.assertEqual(runtime.state_repository.saved_authoritative_project_names, [["FeatureB"]]) + self.assertEqual(runtime.state_repository.active_states, [saved]) + + def test_targeted_service_stop_preserves_unselected_owner_and_shared_main_requirements(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-services-ab", + mode="trees", + services={ + f"{project} Backend": ServiceRecord( + name=f"{project} Backend", + type="backend", + cwd=f"/tmp/{project}", + project=project, + pid=pid, + ) + for project, pid in (("FeatureA", 101), ("FeatureB", 202)) + }, + requirements={ + "Main": RequirementsResult( + project="Main", + components={ + "postgres": { + "enabled": True, + "success": True, + "final": 5432, + } + }, + ) + }, + metadata={ + "project_names": ["FeatureA", "FeatureB"], + "project_roots": { + "FeatureA": "/tmp/FeatureA", + "FeatureB": "/tmp/FeatureB", + }, + "shared_dependencies": True, + }, + ) + runtime.state_repository.active_states = [state] + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"batch": True, "runtime_scope": "backend"}, + ) + ) + + self.assertEqual(code, 0) + saved = runtime.state_repository.saved_states[0] + self.assertEqual(set(saved.services), {"FeatureB Backend"}) + self.assertEqual(set(saved.requirements), {"Main"}) + self.assertTrue(saved.metadata["shared_dependencies"]) + self.assertEqual(saved.metadata["project_names"], ["FeatureB"]) + self.assertEqual(saved.metadata["project_roots"], {"FeatureB": "/tmp/FeatureB"}) + self.assertEqual(runtime.state_repository.saved_authoritative_project_names, [["FeatureB"]]) + + def test_targeted_stop_terminates_and_deactivates_every_shadowed_run_for_project(self) -> None: + runtime = _RuntimeStub() + release_snapshots: list[list[str]] = [] + runtime.port_planner = SimpleNamespace( + release_all=lambda: release_snapshots.append( + [state.run_id for state in runtime.state_repository.active_states] + ) + ) + runtime.state_repository.active_states = [ + RunState( + run_id=run_id, + mode="trees", + services={ + f"FeatureA Backend {run_id}": ServiceRecord( + name=f"FeatureA Backend {run_id}", + type="backend", + cwd=f"/tmp/{run_id}/backend", + project="FeatureA", + pid=pid, + ) + }, + metadata={"project_names": ["FeatureA"]}, + ) + for run_id, pid in (("run-new", 202), ("run-old", 101)) + ] + terminated: list[str] = [] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: terminated.append(state.run_id) + ) + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"batch": True}, + ) + ) + + self.assertEqual(code, 0) + self.assertEqual(terminated, ["run-new", "run-old"]) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-new", "run-old"]) + self.assertEqual(runtime.state_repository.active_states, []) + self.assertEqual(runtime.state_repository.purge_calls, []) + self.assertEqual(release_snapshots, [[]]) + + def test_targeted_stop_failure_preserves_service_and_run_tracking(self) -> None: + runtime = _RuntimeStub() + runtime.port_planner = SimpleNamespace( + release_all=lambda: self.fail("incomplete termination must retain scoped locks") + ) + state = RunState( + run_id="run-a", + mode="trees", + services={ + "FeatureA Backend": ServiceRecord( + name="FeatureA Backend", + type="backend", + cwd="/tmp/FeatureA/backend", + project="FeatureA", + pid=101, + ) + }, + metadata={"project_names": ["FeatureA"]}, + ) + runtime.state_repository.active_states = [state] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda _state, *, selected_services, **_kwargs: set(selected_services or ()) + ) + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"batch": True}, + ) + ) + + self.assertEqual(code, 1) + self.assertEqual(set(state.services), {"FeatureA Backend"}) + self.assertEqual(runtime.state_repository.saved_states, [state]) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(runtime.state_repository.purge_calls, []) + + def test_implicit_mode_targeted_stop_cleans_every_tree_shadow_with_duplicate_pid(self) -> None: + runtime = _RuntimeStub() + main_state = RunState( + run_id="run-main", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + pid=303, + ) + }, + metadata={"project_names": ["Main"]}, + ) + shadow_states = [ + RunState( + run_id=run_id, + mode="trees", + services={ + f"FeatureA Backend {run_id}": ServiceRecord( + name=f"FeatureA Backend {run_id}", + type="backend", + cwd=f"/tmp/{run_id}/backend", + project="FeatureA", + pid=404, + actual_port=8404, + ) + }, + metadata={"project_names": ["FeatureA"]}, + ) + for run_id in ("run-tree-new", "run-tree-old") + ] + runtime.state_repository.active_states = [main_state, *shadow_states] + runtime._try_load_existing_state = ( # type: ignore[method-assign] + lambda *, mode=None, strict_mode_match=False, project_names=None: ( + shadow_states[0] if mode == "main" and not strict_mode_match and project_names == ["FeatureA"] else None + ) + ) + live_pids = {404} + termination_calls: list[int] = [] + released_ports: list[int] = [] + runtime.port_planner = SimpleNamespace(release=lambda port: released_ports.append(port)) + runtime.process_runner = SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + pid_owns_port=lambda pid, port: pid in live_pids and pid == 404 and port == 8404, + terminate_process_group=lambda pid, **_kwargs: termination_calls.append(pid) or not live_pids.discard(pid), + ) + runtime._terminate_service_record = ( # type: ignore[attr-defined] + lambda service, *, aggressive, verify_ownership: terminate_service_record( + runtime, + service, + aggressive=aggressive, + verify_ownership=verify_ownership, + ) + ) + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, *, selected_services, aggressive, verify_ownership: terminate_services_from_state( + runtime, + state, + selected_services=selected_services, + aggressive=aggressive, + verify_ownership=verify_ownership, + ) + ) + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="main", + projects=["FeatureA"], + flags={"batch": True}, + ) + ) + + self.assertEqual(code, 0) + self.assertEqual(termination_calls, [404]) + self.assertEqual(released_ports, [8404, 8404]) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-tree-new", "run-tree-old"]) + self.assertEqual(runtime.state_repository.active_states, [main_state]) + + def test_stop_all_failure_preserves_failed_run_and_never_purges_registry(self) -> None: + runtime = _RuntimeStub() + failed_state = RunState( + run_id="run-failed", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/backend", + project="Main", + pid=101, + ), + "FeatureC Backend": ServiceRecord( + name="FeatureC Backend", + type="backend", + cwd="/tmp/FeatureC/backend", + project="FeatureC", + pid=102, + ), + }, + requirements={ + "shared-main": RequirementsResult(project="Main", db={"enabled": True, "final": 5432}), + "FeatureC": RequirementsResult(project="FeatureC", redis={"enabled": True, "final": 6379}), + }, + ) + successful_state = RunState( + run_id="run-success", + mode="trees", + services={ + "FeatureB Backend": ServiceRecord( + name="FeatureB Backend", + type="backend", + cwd="/tmp/FeatureB/backend", + project="FeatureB", + pid=202, + ) + }, + requirements={ + "FeatureB": RequirementsResult(project="FeatureB", db={"enabled": True, "final": 5433}), + }, + ) + runtime.state_repository.active_states = [failed_state, successful_state] + released_requirements: list[str] = [] + runtime._release_requirement_ports = ( # type: ignore[attr-defined] + lambda requirements: released_requirements.append(requirements.project) + ) + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: {"Main Backend"} if state.run_id == "run-failed" else set() + ) + + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(code, 1) + self.assertEqual(runtime.state_repository.saved_states, [failed_state]) + self.assertEqual(set(failed_state.services), {"Main Backend"}) + self.assertEqual(set(failed_state.requirements), {"shared-main"}) + self.assertEqual(set(released_requirements), {"FeatureC"}) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(runtime.state_repository.active_states, [failed_state, successful_state]) + self.assertEqual(runtime.state_repository.purge_calls, []) + self.assertTrue(any(event.get("event") == "cleanup.incomplete" for event in runtime.events)) + + runtime._terminate_services_from_state = lambda _state, **_kwargs: set() # type: ignore[method-assign] + retry_code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(retry_code, 0) + self.assertEqual(released_requirements.count("Main"), 1) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-failed"]) + self.assertEqual(runtime.state_repository.active_states, [successful_state]) + self.assertEqual(runtime.state_repository.purge_calls, []) + + trees_code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="trees")) + + self.assertEqual(trees_code, 0) + self.assertEqual(released_requirements.count("FeatureB"), 1) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-failed", "run-success"]) + self.assertEqual(runtime.state_repository.active_states, []) + self.assertEqual(runtime.state_repository.purge_calls, [False]) + def test_stop_selector_miss_does_not_start_spinner(self) -> None: runtime = _RuntimeStub() runtime._try_load_existing_state = lambda *args, **kwargs: RunState( # type: ignore[method-assign] diff --git a/tests/python/runtime/test_lifecycle_dependency_stop.py b/tests/python/runtime/test_lifecycle_dependency_stop.py index 59d176d1e..7d1dca6df 100644 --- a/tests/python/runtime/test_lifecycle_dependency_stop.py +++ b/tests/python/runtime/test_lifecycle_dependency_stop.py @@ -1,6 +1,9 @@ from __future__ import annotations import unittest +from pathlib import Path +import tempfile +from types import SimpleNamespace from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.lifecycle_dependency_stop import ( @@ -9,6 +12,8 @@ requirements_have_enabled_components, select_dependency_components_for_stop, ) +from envctl_engine.runtime.lifecycle_requirement_ports import requirement_component_port_owners +from envctl_engine.shared.ports import PortPlanner from envctl_engine.state.models import RequirementsResult, RunState @@ -60,9 +65,11 @@ def test_release_selected_dependency_components_releases_internal_ports_and_prun release_selected_dependency_components( state, {"Main": {"postgres", "redis"}, "Aux": {"postgres"}}, - release_component_ports_fn=lambda component: release_requirement_component_ports( - component, - release_port_fn=released.append, + release_component_ports_fn=lambda _requirements, _dependency_id, component: ( + release_requirement_component_ports( + component, + port_planner=SimpleNamespace(release=released.append), + ) ), ) @@ -86,9 +93,11 @@ def test_release_selected_dependency_components_preserves_projects_with_enabled_ release_selected_dependency_components( state, {"Main": {"redis"}}, - release_component_ports_fn=lambda component: release_requirement_component_ports( - component, - release_port_fn=released.append, + release_component_ports_fn=lambda _requirements, _dependency_id, component: ( + release_requirement_component_ports( + component, + port_planner=SimpleNamespace(release=released.append), + ) ), ) @@ -103,11 +112,53 @@ def test_release_requirement_component_ports_dedupes_positive_final_and_resource release_requirement_component_ports( {"final": 5432, "resources": {"shadow": 15432, "duplicate": 5432, "ignored": 0}}, - release_port_fn=released.append, + port_planner=SimpleNamespace(release=released.append), ) self.assertEqual(released, [5432, 15432]) + def test_selected_dependency_stop_releases_exact_prior_session_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(5432, owner="Main:db") + requirements = RequirementsResult( + project="Main", + db={ + "enabled": True, + "final": 5432, + "port_lock_session": "prior-session", + }, + redis={"enabled": True, "final": 6379}, + ) + state = RunState( + run_id="run-prior", + mode="main", + requirements={"Main": requirements}, + ) + + release_selected_dependency_components( + state, + {"Main": {"postgres"}}, + release_component_ports_fn=lambda owner, dependency_id, component: release_requirement_component_ports( + component, + port_planner=current, + owner_candidates=requirement_component_port_owners(owner, dependency_id), + ), + ) + + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + self.assertFalse(requirements.db.get("enabled", False)) + self.assertTrue(requirements.redis.get("enabled", False)) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/runtime/test_lifecycle_requirement_ports.py b/tests/python/runtime/test_lifecycle_requirement_ports.py index afa5fb5cc..34a67a0a7 100644 --- a/tests/python/runtime/test_lifecycle_requirement_ports.py +++ b/tests/python/runtime/test_lifecycle_requirement_ports.py @@ -1,6 +1,8 @@ from __future__ import annotations import unittest +import tempfile +from pathlib import Path from types import SimpleNamespace from envctl_engine.runtime.lifecycle_requirement_ports import ( @@ -10,6 +12,7 @@ requirement_port_values, ) from envctl_engine.state.models import RequirementsResult, RunState +from envctl_engine.shared.ports import PortPlanner class LifecycleRequirementPortsTests(unittest.TestCase): @@ -20,16 +23,21 @@ def test_requirement_key_for_project_matches_case_insensitive_key(self) -> None: self.assertIsNone(requirement_key_for_project(state, " ")) self.assertIsNone(requirement_key_for_project(state, "missing")) - def test_requirement_port_values_returns_enabled_internal_final_ports_only(self) -> None: + def test_requirement_port_values_returns_all_enabled_internal_resource_ports(self) -> None: requirements = RequirementsResult( project="Main", db={"enabled": True, "final": 5432}, redis={"enabled": False, "final": 6379}, - supabase={"enabled": True, "external": True, "final": 54321}, - n8n={"enabled": True, "final": 5678, "resources": {"ignored": 15678}}, + supabase={ + "enabled": True, + "external": True, + "final": 54321, + "resources": {"external_api": 55432}, + }, + n8n={"enabled": True, "final": 5678, "resources": {"secondary": 15678}}, ) - self.assertEqual(requirement_port_values(requirements), {5432, 5678}) + self.assertEqual(requirement_port_values(requirements), {5432, 5678, 15678}) def test_component_port_values_collects_final_and_resource_ports(self) -> None: self.assertEqual( @@ -43,12 +51,115 @@ def test_release_requirement_ports_uses_runtime_port_planner_in_sorted_order(sel requirements = RequirementsResult( project="Main", db={"enabled": True, "final": 5432}, + supabase={ + "enabled": True, + "final": 15432, + "resources": {"db": 15432, "api": 15433}, + }, n8n={"enabled": True, "final": 5678}, ) release_requirement_ports(runtime, requirements) - self.assertEqual(released, [5432, 5678]) + self.assertEqual(released, [5432, 5678, 15432, 15433]) + + def test_release_requirement_ports_can_release_verified_prior_session_owners(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(5432, owner="Main:db") + prior.reserve_next(5433, owner="Main:requirements") + requirements = RequirementsResult( + project="Main", + db={ + "enabled": True, + "final": 5432, + "resources": {"primary": 5432, "retry": 5433}, + "port_lock_session": "prior-session", + }, + ) + + release_requirement_ports(SimpleNamespace(port_planner=current), requirements) + + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + + def test_stale_requirement_state_cannot_release_newer_live_same_owner_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + newer = PortPlanner( + lock_dir=tmpdir, + session_id="newer-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + newer.reserve_next(5432, owner="Main:db") + stale_requirements = RequirementsResult( + project="Main", + db={ + "enabled": True, + "final": 5432, + "port_lock_session": "stale-session", + }, + ) + + release_requirement_ports( + SimpleNamespace(port_planner=cleanup), + stale_requirements, + ) + + self.assertTrue((Path(tmpdir) / "5432.lock").exists()) + + def test_legacy_requirement_state_preserves_live_foreign_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + live = PortPlanner( + lock_dir=tmpdir, + session_id="live-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + live.reserve_next(5432, owner="Main:db") + + release_requirement_ports( + SimpleNamespace(port_planner=cleanup), + RequirementsResult(project="Main", db={"enabled": True, "final": 5432}), + ) + + self.assertTrue((Path(tmpdir) / "5432.lock").exists()) + + def test_legacy_requirement_state_reaps_only_proven_dead_foreign_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "5432.lock" + lock_path.write_text( + '{"created_at": 0, "owner": "Main:db", "pid": 99999999, "session": "dead-session"}', + encoding="utf-8", + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + + release_requirement_ports( + SimpleNamespace(port_planner=cleanup), + RequirementsResult(project="Main", db={"enabled": True, "final": 5432}), + ) + + self.assertFalse(lock_path.exists()) if __name__ == "__main__": diff --git a/tests/python/runtime/test_lifecycle_service_stop_selection.py b/tests/python/runtime/test_lifecycle_service_stop_selection.py index 2f4bb8306..2e0154388 100644 --- a/tests/python/runtime/test_lifecycle_service_stop_selection.py +++ b/tests/python/runtime/test_lifecycle_service_stop_selection.py @@ -34,6 +34,44 @@ def test_runtime_scope_selects_matching_services_without_interactive_prompt(self self.assertEqual(selected, {"Aux Backend", "Main Backend"}) + def test_runtime_scope_intersects_explicit_project_selector(self) -> None: + state = RunState( + run_id="run-1", + mode="trees", + services={ + "FeatureA Backend": ServiceRecord( + name="FeatureA Backend", + type="backend", + cwd=".", + pid=1, + project="FeatureA", + ), + "FeatureB Backend": ServiceRecord( + name="FeatureB Backend", + type="backend", + cwd=".", + pid=2, + project="FeatureB", + ), + }, + ) + + selected = select_services_for_stop( + state, + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"runtime_scope": "backend"}, + ), + project_name_from_service_fn=lambda name: name.split(" ", 1)[0], + selectors_from_passthrough_fn=lambda _args: set(), + interactive_stop_selection_fn=lambda _route, _state: self.fail("interactive prompt should not run"), + services_from_selection_fn=lambda _selection, _state: self.fail("selection projection should not run"), + ) + + self.assertEqual(selected, {"FeatureA Backend"}) + def test_service_and_project_selectors_match_slug_display_name_and_passthrough_project(self) -> None: state = RunState( run_id="run-1", diff --git a/tests/python/runtime/test_lifecycle_service_termination.py b/tests/python/runtime/test_lifecycle_service_termination.py index eb43fa87d..6438d2d2d 100644 --- a/tests/python/runtime/test_lifecycle_service_termination.py +++ b/tests/python/runtime/test_lifecycle_service_termination.py @@ -1,30 +1,36 @@ from __future__ import annotations import os +import signal +import subprocess +import sys +import tempfile import unittest +from pathlib import Path from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import Mock, patch from envctl_engine.runtime.lifecycle_service_termination import ( + _wait_for_pid_exit, service_port, terminate_service_record, terminate_services_from_state, terminate_started_services, ) from envctl_engine.state.models import RunState, ServiceRecord +from envctl_engine.shared.ports import PortPlanner class LifecycleServiceTerminationTests(unittest.TestCase): def test_terminate_started_services_uses_non_aggressive_unverified_shutdown(self) -> None: calls: list[tuple[int, bool, bool]] = [] runtime = SimpleNamespace( - _terminate_service_record=lambda service, *, aggressive, verify_ownership: calls.append( - (service.pid, aggressive, verify_ownership) + _terminate_service_record=lambda service, *, aggressive, verify_ownership: ( + calls.append((service.pid, aggressive, verify_ownership)) or True ) - or True ) - terminate_started_services( + failed = terminate_started_services( runtime, { "Main Backend": ServiceRecord(name="Main Backend", type="backend", cwd=".", pid=1), @@ -33,16 +39,56 @@ def test_terminate_started_services_uses_non_aggressive_unverified_shutdown(self ) self.assertEqual(calls, [(1, False, False), (2, False, False)]) + self.assertEqual(failed, set()) + + def test_terminate_started_services_reports_every_unconfirmed_exit(self) -> None: + runtime = SimpleNamespace(_terminate_service_record=lambda service, **_kwargs: service.name != "Main Backend") + + failed = terminate_started_services( + runtime, + { + "Main Backend": ServiceRecord(name="Main Backend", type="backend", cwd=".", pid=1), + "Main Frontend": ServiceRecord(name="Main Frontend", type="frontend", cwd=".", pid=2), + }, + ) + + self.assertEqual(failed, {"Main Backend"}) + + def test_terminate_started_services_continues_after_terminator_exception(self) -> None: + calls: list[str] = [] + events: list[tuple[str, dict[str, object]]] = [] + + def terminate(service: ServiceRecord, **_kwargs: object) -> bool: + calls.append(service.name) + if service.name == "Main Backend": + raise RuntimeError("probe failed") + return True + + runtime = SimpleNamespace( + _terminate_service_record=terminate, + _emit=lambda event, **payload: events.append((event, payload)), + ) + + failed = terminate_started_services( + runtime, + { + "Main Backend": ServiceRecord(name="Main Backend", type="backend", cwd="."), + "Main Frontend": ServiceRecord(name="Main Frontend", type="frontend", cwd="."), + }, + ) + + self.assertEqual(calls, ["Main Backend", "Main Frontend"]) + self.assertEqual(failed, {"Main Backend"}) + self.assertEqual(events[0][0], "cleanup.error") def test_terminate_services_from_state_filters_selected_services_and_releases_terminated_ports(self) -> None: released: list[int] = [] terminated: list[tuple[int, bool, bool]] = [] runtime = SimpleNamespace( port_planner=SimpleNamespace(release=lambda port: released.append(port)), - _terminate_service_record=lambda service, *, aggressive, verify_ownership: terminated.append( - (service.pid, aggressive, verify_ownership) - ) - or True, + _terminate_service_record=lambda service, *, aggressive, verify_ownership: ( + terminated.append((service.pid, aggressive, verify_ownership)) or True + ), ) state = RunState( run_id="run-1", @@ -55,7 +101,7 @@ def test_terminate_services_from_state_filters_selected_services_and_releases_te }, ) - terminate_services_from_state( + failed = terminate_services_from_state( runtime, state, selected_services={"Main Frontend"}, @@ -65,12 +111,229 @@ def test_terminate_services_from_state_filters_selected_services_and_releases_te self.assertEqual(terminated, [(2, True, True)]) self.assertEqual(released, [3000]) + self.assertEqual(failed, set()) + + def test_terminated_service_releases_verified_prior_session_port_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(8000, owner="Main:backend") + runtime = SimpleNamespace( + port_planner=current, + _terminate_service_record=lambda _service, **_kwargs: True, + _project_name_from_service=lambda _name: "Main", + ) + state = RunState( + run_id="run-prior", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + project="Main", + pid=1, + actual_port=8000, + port_lock_session="prior-session", + ) + }, + ) + + failed = terminate_services_from_state( + runtime, + state, + selected_services=None, + aggressive=False, + verify_ownership=True, + ) + + self.assertEqual(failed, set()) + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + + def test_terminated_rebound_frontend_releases_launch_owner_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(5173, owner="Main:frontend") + prior.reserve_next(5174, owner="Main:services:frontend-launch") + runtime = SimpleNamespace( + port_planner=current, + _terminate_service_record=lambda _service, **_kwargs: True, + _project_name_from_service=lambda _name: "Main", + ) + state = RunState( + run_id="run-prior", + mode="main", + services={ + "Main Frontend": ServiceRecord( + name="Main Frontend", + type="frontend", + cwd=".", + project="Main", + pid=1, + requested_port=5173, + actual_port=5174, + port_lock_session="prior-session", + ) + }, + ) + + failed = terminate_services_from_state( + runtime, + state, + selected_services=None, + aggressive=False, + verify_ownership=True, + ) + + self.assertEqual(failed, set()) + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + + def test_stale_service_state_cannot_release_newer_live_same_owner_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + newer = PortPlanner( + lock_dir=tmpdir, + session_id="newer-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + newer.reserve_next(8000, owner="Main:backend") + runtime = SimpleNamespace( + port_planner=cleanup, + _terminate_service_record=lambda _service, **_kwargs: True, + ) + state = RunState( + run_id="stale-run", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + project="Main", + pid=1, + actual_port=8000, + port_lock_session="stale-session", + ) + }, + ) + + failed = terminate_services_from_state( + runtime, + state, + selected_services=None, + aggressive=False, + verify_ownership=True, + ) + + self.assertEqual(failed, set()) + self.assertTrue((Path(tmpdir) / "8000.lock").exists()) + + def test_startup_rollback_releases_confirmed_service_ports_but_preserves_failed_service_ports(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="startup-session", + availability_checker=lambda _port: True, + ) + planner.reserve_next(5173, owner="Main:frontend") + planner.reserve_next(5174, owner="Main:services:frontend-launch") + planner.reserve_next(8000, owner="Main:backend") + frontend = ServiceRecord( + name="Main Frontend", + type="frontend", + cwd=".", + project="Main", + pid=1, + requested_port=5173, + actual_port=5174, + ) + backend = ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + project="Main", + pid=2, + requested_port=8000, + actual_port=8000, + ) + runtime = SimpleNamespace( + port_planner=planner, + _terminate_service_record=lambda service, **_kwargs: service is frontend, + ) + + failed = terminate_started_services( + runtime, + {frontend.name: frontend, backend.name: backend}, + ) + + self.assertEqual(failed, {backend.name}) + remaining = list(Path(tmpdir).glob("*.lock")) + self.assertEqual([path.name for path in remaining], ["8000.lock"]) + + def test_terminate_services_reports_failures_without_releasing_their_ports(self) -> None: + released: list[int] = [] + runtime = SimpleNamespace( + port_planner=SimpleNamespace(release=lambda port: released.append(port)), + _terminate_service_record=lambda service, **_kwargs: service.name != "Main Backend", + ) + state = RunState( + run_id="run-1", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + pid=1, + actual_port=8000, + ), + "Main Frontend": ServiceRecord( + name="Main Frontend", + type="frontend", + cwd=".", + pid=2, + actual_port=3000, + ), + }, + ) + + failed = terminate_services_from_state( + runtime, + state, + selected_services=None, + aggressive=False, + verify_ownership=True, + ) + + self.assertEqual(failed, {"Main Backend"}) + self.assertEqual(released, [3000]) def test_terminate_service_record_skips_self_parent_and_missing_ownership_port(self) -> None: events: list[dict[str, object]] = [] runtime = SimpleNamespace( _emit=lambda _event, **payload: events.append(payload), - process_runner=SimpleNamespace(), + process_runner=SimpleNamespace(is_pid_running=lambda _pid: True), ) with patch.object(os, "getpid", return_value=100), patch.object(os, "getppid", return_value=200): @@ -96,6 +359,7 @@ def test_terminate_service_record_skips_self_parent_and_missing_ownership_port(s def test_terminate_service_record_accepts_verified_listener_child_and_prefers_process_group(self) -> None: calls: list[tuple[str, int]] = [] + live_pids = {111, 222} def pid_owns_port(pid: int, port: int) -> bool: return pid == 222 and port == 9000 @@ -103,8 +367,13 @@ def pid_owns_port(pid: int, port: int) -> bool: runtime = SimpleNamespace( _emit=lambda *_args, **_kwargs: None, process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, pid_owns_port=pid_owns_port, - terminate_process_group=lambda pid, **_kwargs: calls.append(("group", pid)) or True, + terminate_process_group=lambda pid, **_kwargs: ( + calls.append(("group", pid)), + live_pids.discard(pid), + True, + )[-1], terminate=lambda pid, **_kwargs: calls.append(("single", pid)) or True, ), ) @@ -117,7 +386,428 @@ def pid_owns_port(pid: int, port: int) -> bool: ) self.assertTrue(terminated) - self.assertEqual(calls, [("group", 111)]) + self.assertEqual(calls, [("group", 111), ("group", 222)]) + + def test_terminate_service_record_stops_non_listener_with_matching_recorded_cwd(self) -> None: + calls: list[int] = [] + live_pids = {333} + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + process_cwd=lambda _pid: "/repo/worker", + terminate_process_group=lambda pid, **_kwargs: (calls.append(pid), live_pids.discard(pid), True)[-1], + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Worker", + type="worker", + cwd="/repo/worker", + pid=333, + listener_expected=False, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertTrue(terminated) + self.assertEqual(calls, [333]) + + def test_terminate_service_record_keeps_non_listener_when_cwd_cannot_be_verified(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + runtime = SimpleNamespace( + _emit=lambda event, **payload: events.append((event, payload)), + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + process_cwd=lambda _pid: "/foreign/worker", + terminate_process_group=lambda *_args, **_kwargs: self.fail("foreign PID must not be signalled"), + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Worker", + type="worker", + cwd="/repo/worker", + pid=334, + listener_expected=False, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertFalse(terminated) + self.assertEqual(events[0][1]["reason"], "missing_port_for_ownership") + + def test_parent_dead_live_listener_child_is_still_terminated(self) -> None: + calls: list[int] = [] + live_pids = {402} + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + pid_owns_port=lambda pid, port: pid == 402 and port == 8000, + terminate_process_group=lambda pid, **_kwargs: (calls.append(pid), live_pids.discard(pid), True)[-1], + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=401, + listener_pids=[402], + actual_port=8000, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertTrue(terminated) + self.assertEqual(calls, [402]) + + def test_missing_primary_pid_does_not_hide_live_listener_child(self) -> None: + calls: list[int] = [] + live_pids = {403} + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + pid_owns_port=lambda pid, port: pid == 403 and port == 8001, + terminate_process_group=lambda pid, **_kwargs: (calls.append(pid), live_pids.discard(pid), True)[-1], + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=None, + listener_pids=[403], + actual_port=8001, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertTrue(terminated) + self.assertEqual(calls, [403]) + + def test_dead_parent_with_unrecorded_live_listener_keeps_state_without_signalling_foreign_pid(self) -> None: + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _listener_pids_for_port=lambda _port: [406], + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid == 406, + pid_owns_port=lambda *_args, **_kwargs: self.fail("no recorded live PID should be probed"), + terminate_process_group=lambda *_args, **_kwargs: self.fail("foreign PID must not be signalled"), + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=404, + listener_pids=None, + actual_port=8002, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertFalse(terminated) + + def test_pidless_unconfirmed_launch_remains_tracked(self) -> None: + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace(), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Worker", + type="worker", + cwd="/repo/worker", + pid=None, + listener_expected=False, + status="termination_failed", + ), + aggressive=False, + verify_ownership=False, + ) + + self.assertFalse(terminated) + + def test_live_port_listener_after_termination_keeps_service_tracked(self) -> None: + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _listener_pids_for_port=lambda _port: [405], + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + pid_owns_port=lambda pid, port: pid == 404 and port == 8002, + terminate_process_group=lambda *_args, **_kwargs: True, + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=404, + actual_port=8002, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertFalse(terminated) + + def test_malformed_listener_probe_result_fails_closed(self) -> None: + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _listener_pids_for_port=lambda _port: None, + process_runner=SimpleNamespace(is_pid_running=lambda _pid: False), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=999, + actual_port=8003, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertFalse(terminated) + + def test_terminate_service_record_treats_already_dead_pid_as_success_before_ownership_check(self) -> None: + ownership_checks: list[tuple[int, int]] = [] + terminate_calls: list[int] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: False, + pid_owns_port=lambda pid, port: ownership_checks.append((pid, port)) or False, + terminate_process_group=lambda pid, **_kwargs: terminate_calls.append(pid) or True, + ), + ) + + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Shadow Backend", pid=777, actual_port=8000), + aggressive=False, + verify_ownership=True, + ) + + self.assertTrue(terminated) + self.assertEqual(ownership_checks, []) + self.assertEqual(terminate_calls, []) + + def test_terminate_service_record_keeps_live_non_owner_fail_closed(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + runtime = SimpleNamespace( + _emit=lambda event, **payload: events.append((event, payload)), + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + pid_owns_port=lambda _pid, _port: False, + terminate_process_group=lambda *_args, **_kwargs: self.fail("non-owner must not be signalled"), + ), + ) + + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Foreign Backend", pid=778, actual_port=8001), + aggressive=False, + verify_ownership=True, + ) + + self.assertFalse(terminated) + self.assertEqual( + events, + [ + ( + "cleanup.skip", + { + "service": "Foreign Backend", + "pid": 778, + "port": 8001, + "reason": "ownership_mismatch", + }, + ) + ], + ) + + def test_raw_signal_fallback_escalates_and_fails_when_pid_survives(self) -> None: + sent_signals: list[tuple[int, signal.Signals]] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + terminate=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + ), + ) + + with ( + patch( + "envctl_engine.shared.process_termination.os.kill", + side_effect=lambda pid, requested_signal: sent_signals.append((pid, requested_signal)), + ), + patch( + "envctl_engine.shared.process_termination.wait_for_pid_exit", + side_effect=[False, False], + ) as wait_for_exit, + ): + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Main Backend", pid=779, actual_port=8002), + aggressive=False, + verify_ownership=False, + ) + + self.assertFalse(terminated) + self.assertEqual(sent_signals, [(779, signal.SIGTERM), (779, signal.SIGKILL)]) + self.assertEqual( + [call.kwargs["timeout"] for call in wait_for_exit.call_args_list], + [2.0, 1.0], + ) + + def test_raw_signal_fallback_does_not_escalate_after_confirmed_sigterm_exit(self) -> None: + sent_signals: list[tuple[int, signal.Signals]] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + terminate=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + ), + ) + + with ( + patch( + "envctl_engine.shared.process_termination.os.kill", + side_effect=lambda pid, requested_signal: sent_signals.append((pid, requested_signal)), + ), + patch( + "envctl_engine.shared.process_termination.wait_for_pid_exit", + return_value=True, + ), + ): + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Main Backend", pid=780, actual_port=8003), + aggressive=True, + verify_ownership=False, + ) + + self.assertTrue(terminated) + self.assertEqual(sent_signals, [(780, signal.SIGTERM)]) + + def test_raw_signal_fallback_accepts_process_lookup_race_as_clean_exit(self) -> None: + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + terminate=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + ), + ) + + with patch( + "envctl_engine.shared.process_termination.os.kill", + side_effect=ProcessLookupError, + ): + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Main Backend", pid=781, actual_port=8004), + aggressive=False, + verify_ownership=False, + ) + + self.assertTrue(terminated) + + def test_raw_signal_exit_wait_polls_until_pid_is_confirmed_dead(self) -> None: + runtime = SimpleNamespace( + process_runner=SimpleNamespace(is_pid_running=Mock(side_effect=[True, False])), + ) + + with ( + patch( + "envctl_engine.shared.process_termination.time.monotonic", + side_effect=[10.0, 10.0], + ), + patch("envctl_engine.shared.process_termination.time.sleep") as sleep_mock, + ): + exited = _wait_for_pid_exit( + runtime, + 782, + timeout=1.0, + initial_identity=None, + ) + + self.assertTrue(exited) + sleep_mock.assert_called_once_with(0.05) + + def test_raw_signal_fallback_really_escalates_sigterm_resistant_process(self) -> None: + process = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "import signal,time; " + "signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "print('ready', flush=True); " + "time.sleep(30)" + ), + ], + stdout=subprocess.PIPE, + text=True, + ) + try: + assert process.stdout is not None + self.assertEqual(process.stdout.readline().strip(), "ready") + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: process.poll() is None, + terminate_process_group=lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("unsupported") + ), + terminate=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + ), + ) + + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Main Backend", pid=process.pid, actual_port=8005), + aggressive=True, + verify_ownership=False, + ) + + self.assertTrue(terminated) + self.assertEqual(process.wait(timeout=2.0), -signal.SIGKILL) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=2.0) def test_service_port_prefers_actual_then_requested(self) -> None: self.assertEqual(service_port(SimpleNamespace(actual_port=9000, requested_port=8000)), 9000) diff --git a/tests/python/runtime/test_lifecycle_worktree_cleanup.py b/tests/python/runtime/test_lifecycle_worktree_cleanup.py index 79773b828..759666a97 100644 --- a/tests/python/runtime/test_lifecycle_worktree_cleanup.py +++ b/tests/python/runtime/test_lifecycle_worktree_cleanup.py @@ -86,6 +86,56 @@ def test_blast_worktree_before_delete_uses_injected_cleanup_dependencies(self) - self.assertEqual(warnings, []) self.assertEqual(calls, ["ports", "cwd", "artifacts", "containers"]) + def test_worktree_delete_aborts_and_retains_authority_when_service_exit_is_unconfirmed(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + saved_states: list[RunState] = [] + released: list[int] = [] + service = ServiceRecord( + name="Feature Backend", + type="backend", + cwd="/repo/feature/backend", + pid=10, + actual_port=8000, + ) + requirements = RequirementsResult( + project="Feature", + db={"enabled": True, "final": 5432}, + ) + state = RunState( + run_id="run-1", + mode="trees", + services={service.name: service}, + requirements={"Feature": requirements}, + metadata={"project_roots": {"Feature": "/repo/feature"}}, + ) + runtime = SimpleNamespace( + _emit=lambda event, **payload: events.append((event, payload)), + _try_load_existing_state=lambda **kwargs: state if kwargs["mode"] == "trees" else None, + _project_name_from_service=lambda name: "Feature" if "Feature" in name else "", + _terminate_services_from_state=lambda *args, **kwargs: {service.name}, + port_planner=SimpleNamespace(release=lambda port: released.append(port)), + state_repository=SimpleNamespace( + save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) + ), + process_runner=SimpleNamespace(), + env={}, + ) + + with self.assertRaisesRegex(RuntimeError, "could not confirm service exit"): + blast_worktree_before_delete( + runtime, + project_name="Feature", + project_root=Path("/repo/feature"), + source_command="delete-worktree", + ) + + self.assertEqual(len(saved_states), 1) + self.assertIs(saved_states[0].services[service.name], service) + self.assertIs(saved_states[0].requirements["Feature"], requirements) + self.assertEqual(saved_states[0].metadata["project_roots"], {"Feature": "/repo/feature"}) + self.assertEqual(released, []) + self.assertEqual(events[-1][0], "cleanup.worktree.warning") + def test_prune_project_metadata_returns_test_artifacts_and_removes_empty_keys(self) -> None: state = RunState( run_id="run-1", @@ -152,7 +202,9 @@ def test_blast_tree_listener_ports_skips_docker_processes_and_kills_owned_listen self.assertEqual(warnings, []) self.assertEqual(killed, [222]) - self.assertEqual([event for event, _payload in events], ["cleanup.worktree.port.skip", "cleanup.worktree.port.kill"]) + self.assertEqual( + [event for event, _payload in events], ["cleanup.worktree.port.skip", "cleanup.worktree.port.kill"] + ) def test_remove_tree_containers_matches_hashed_and_legacy_names_and_removes_volumes(self) -> None: calls: list[list[str]] = [] diff --git a/tests/python/runtime/test_runtime_scope_isolation.py b/tests/python/runtime/test_runtime_scope_isolation.py index 503af616a..e2e3cc96c 100644 --- a/tests/python/runtime/test_runtime_scope_isolation.py +++ b/tests/python/runtime/test_runtime_scope_isolation.py @@ -13,6 +13,29 @@ class RuntimeScopeIsolationTests(unittest.TestCase): + def test_runtime_uses_repository_canonical_root_through_symlinked_parent(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + real_runtime_dir = root / "real-runtime" + runtime_alias = root / "runtime-alias" + repo = root / "repo" + real_runtime_dir.mkdir() + runtime_alias.symlink_to(real_runtime_dir, target_is_directory=True) + (repo / ".git").mkdir(parents=True) + config = load_config( + { + "RUN_REPO_ROOT": str(repo), + "RUN_SH_RUNTIME_DIR": str(runtime_alias), + } + ) + + runtime = PythonEngineRuntime(config, env={}) + + self.assertEqual(runtime.runtime_root, runtime.state_repository.runtime_root) + self.assertEqual(runtime.runtime_legacy_root, runtime.state_repository.runtime_legacy_root) + self.assertEqual(runtime.runtime_root, config.runtime_scope_dir.resolve()) + self.assertEqual(runtime.port_planner.lock_dir, runtime.runtime_root / "locks") + def test_managed_linked_worktree_uses_main_repo_runtime_scope_and_lock_dir(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -27,9 +50,7 @@ def test_managed_linked_worktree_uses_main_repo_runtime_scope_and_lock_dir(self) (worktree / ".git").write_text(f"gitdir: {gitdir}\n", encoding="utf-8") main_config = load_config({"RUN_REPO_ROOT": str(repo), "RUN_SH_RUNTIME_DIR": str(runtime_dir)}) - worktree_config = load_config( - {"RUN_REPO_ROOT": str(worktree), "RUN_SH_RUNTIME_DIR": str(runtime_dir)} - ) + worktree_config = load_config({"RUN_REPO_ROOT": str(worktree), "RUN_SH_RUNTIME_DIR": str(runtime_dir)}) main_runtime = PythonEngineRuntime(main_config, env={}) worktree_runtime = PythonEngineRuntime(worktree_config, env={"ENVCTL_INVOCATION_CWD": str(worktree)}) @@ -205,6 +226,83 @@ def test_stale_legacy_lock_symlink_is_replaced(self) -> None: (runtime.runtime_root / "locks").resolve(strict=False), ) + def test_scoped_and_read_only_construction_preserve_legacy_artifacts_and_lock_view(self) -> None: + for compat_mode in ("scoped_only", "compat_read_only"): + with self.subTest(compat_mode=compat_mode), tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime_dir = root / "runtime" + repo = root / "repo" + (repo / ".git").mkdir(parents=True) + legacy_root = runtime_dir / "python-engine" + legacy_root.mkdir(parents=True) + sentinel = legacy_root / "run_state.json" + sentinel.write_text("legacy-sentinel", encoding="utf-8") + config = load_config( + { + "RUN_REPO_ROOT": str(repo), + "RUN_SH_RUNTIME_DIR": str(runtime_dir), + } + ) + + runtime = PythonEngineRuntime( + config, + env={"ENVCTL_STATE_COMPAT_MODE": compat_mode}, + ) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "legacy-sentinel") + self.assertFalse((legacy_root / "locks").exists()) + self.assertTrue(runtime.runtime_root.is_dir()) + + def test_construction_rejects_preexisting_scoped_root_symlink_before_external_write(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime_dir = root / "runtime" + repo = root / "repo" + (repo / ".git").mkdir(parents=True) + config = load_config( + { + "RUN_REPO_ROOT": str(repo), + "RUN_SH_RUNTIME_DIR": str(runtime_dir), + } + ) + external = root / "external" + external.mkdir() + sentinel = external / "sentinel.txt" + sentinel.write_text("untouched", encoding="utf-8") + config.runtime_scope_dir.parent.mkdir(parents=True, exist_ok=True) + config.runtime_scope_dir.symlink_to(external, target_is_directory=True) + + with self.assertRaisesRegex(ValueError, "symlink"): + PythonEngineRuntime(config, env={}) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "untouched") + self.assertEqual(list(external.iterdir()), [sentinel]) + + def test_construction_rejects_preexisting_legacy_root_symlink_before_external_write(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime_dir = root / "runtime" + repo = root / "repo" + (repo / ".git").mkdir(parents=True) + external = root / "external" + external.mkdir() + sentinel = external / "sentinel.txt" + sentinel.write_text("untouched", encoding="utf-8") + runtime_dir.mkdir() + (runtime_dir / "python-engine").symlink_to(external, target_is_directory=True) + config = load_config( + { + "RUN_REPO_ROOT": str(repo), + "RUN_SH_RUNTIME_DIR": str(runtime_dir), + } + ) + + with self.assertRaisesRegex(ValueError, "symlink"): + PythonEngineRuntime(config, env={}) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "untouched") + self.assertEqual(list(external.iterdir()), [sentinel]) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/shared/test_ports_lock_reclamation.py b/tests/python/shared/test_ports_lock_reclamation.py index c6fb9f436..af200b89f 100644 --- a/tests/python/shared/test_ports_lock_reclamation.py +++ b/tests/python/shared/test_ports_lock_reclamation.py @@ -1,9 +1,13 @@ from __future__ import annotations +import os import json import tempfile +import threading import unittest +from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from unittest.mock import patch REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" @@ -177,6 +181,305 @@ def test_release_session_isolated_to_own_session(self) -> None: self.assertFalse((Path(tmpdir) / f"{a_port}.lock").exists()) self.assertTrue((Path(tmpdir) / f"{b_port}.lock").exists()) + def test_concurrent_threads_never_publish_duplicate_reservations(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + worker_count = 64 + barrier = threading.Barrier(worker_count) + + def reserve(index: int) -> int: + planner = PortPlanner( + lock_dir=tmpdir, + session_id=f"session-{index}", + availability_checker=lambda _port: True, + ) + barrier.wait() + return planner.reserve_next(12000, owner=f"worker-{index}") + + with ThreadPoolExecutor(max_workers=worker_count) as executor: + reserved = list(executor.map(reserve, range(worker_count))) + + self.assertEqual(len(set(reserved)), worker_count) + self.assertEqual(len(list(Path(tmpdir).glob("*.lock"))), worker_count) + PortPlanner(lock_dir=tmpdir, session_id="cleanup").release_all() + + def test_concurrent_stale_reclaim_is_serialized(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "13000.lock" + lock_path.write_text( + json.dumps({"owner": "dead", "session": "dead", "pid": 99999999, "created_at": 0}), + encoding="utf-8", + ) + worker_count = 32 + barrier = threading.Barrier(worker_count) + + def reserve(index: int) -> int: + planner = PortPlanner( + lock_dir=tmpdir, + session_id=f"session-{index}", + availability_checker=lambda _port: True, + ) + barrier.wait() + return planner.reserve_next(13000, owner=f"worker-{index}") + + with ThreadPoolExecutor(max_workers=worker_count) as executor: + reserved = list(executor.map(reserve, range(worker_count))) + + self.assertEqual(len(set(reserved)), worker_count) + self.assertEqual(len(list(Path(tmpdir).glob("*.lock"))), worker_count) + PortPlanner(lock_dir=tmpdir, session_id="cleanup").release_all() + + def test_lock_path_is_linked_only_after_payload_is_complete(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + observed: dict[str, object] = {} + real_link = os.link + + def inspected_link(source: str | Path, destination: str | Path) -> None: + source_path = Path(source) + destination_path = Path(destination) + observed["destination_preexisted"] = destination_path.exists() + observed["payload"] = json.loads(source_path.read_text(encoding="utf-8")) + real_link(source_path, destination_path) + + planner = PortPlanner( + lock_dir=tmpdir, + session_id="complete-publication", + availability_checker=lambda _port: True, + ) + with ( + patch("envctl_engine.shared.ports.os.link", side_effect=inspected_link), + patch("envctl_engine.shared.ports.fsync_directory") as fsync_directory, + ): + reserved = planner.reserve_next(14000, owner="worker") + + self.assertEqual(reserved, 14000) + self.assertFalse(observed["destination_preexisted"]) + self.assertEqual(observed["payload"], json.loads((Path(tmpdir) / "14000.lock").read_text())) + self.assertEqual(list((Path(tmpdir) / ".port-guards").glob("*.pending")), []) + fsync_directory.assert_called_once_with(Path(tmpdir)) + + def test_corrupt_lock_gets_a_grace_period_then_becomes_stale(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + now = [1000.0] + lock_path = Path(tmpdir) / "15000.lock" + lock_path.write_text("{}", encoding="utf-8") + os.utime(lock_path, (now[0], now[0])) + planner = PortPlanner( + lock_dir=tmpdir, + session_id="new", + availability_checker=lambda _port: True, + time_provider=lambda: now[0], + corrupt_lock_grace_seconds=2.0, + ) + + self.assertEqual(planner.reserve_next(15000, owner="new"), 15001) + planner.release_session() + now[0] += 3.0 + self.assertEqual(planner.reserve_next(15000, owner="new"), 15000) + + def test_permission_error_means_lock_owner_is_still_live(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "16000.lock" + lock_path.write_text( + json.dumps({"owner": "foreign", "session": "foreign", "pid": 424242, "created_at": 0}), + encoding="utf-8", + ) + planner = PortPlanner( + lock_dir=tmpdir, + session_id="new", + availability_checker=lambda _port: True, + ) + + with patch("envctl_engine.shared.ports.os.kill", side_effect=PermissionError("denied")): + self.assertEqual(planner.reserve_next(16000, owner="new"), 16001) + + def test_boolean_pid_is_corrupt_data_not_a_live_pid_one_reservation(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "16010.lock" + lock_path.write_text( + json.dumps({"owner": "broken", "session": "broken", "pid": True, "created_at": False}), + encoding="utf-8", + ) + os.utime(lock_path, (0, 0)) + planner = PortPlanner( + lock_dir=tmpdir, + session_id="new", + availability_checker=lambda _port: True, + pid_checker=lambda _pid: self.fail("a JSON boolean must not be treated as PID 1"), + corrupt_lock_grace_seconds=0.0, + ) + + self.assertEqual(planner.reserve_next(16010, owner="new"), 16010) + + def test_failed_pid_probe_preserves_a_foreign_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "16020.lock" + lock_path.write_text( + json.dumps({"owner": "foreign", "session": "foreign", "pid": 424242, "created_at": 0}), + encoding="utf-8", + ) + planner = PortPlanner( + lock_dir=tmpdir, + session_id="new", + availability_checker=lambda _port: True, + pid_checker=lambda _pid: (_ for _ in ()).throw(OSError("probe unavailable")), + ) + + self.assertEqual(planner.reserve_next(16020, owner="new"), 16021) + self.assertEqual(json.loads(lock_path.read_text(encoding="utf-8"))["owner"], "foreign") + + def test_non_finite_corrupt_timestamp_cannot_pin_a_port_forever(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "16030.lock" + lock_path.write_text( + '{"owner":"broken","session":"broken","pid":false,"created_at":"nan"}', + encoding="utf-8", + ) + os.utime(lock_path, (0, 0)) + planner = PortPlanner( + lock_dir=tmpdir, + session_id="new", + availability_checker=lambda _port: True, + corrupt_lock_grace_seconds=0.0, + ) + + self.assertEqual(planner.reserve_next(16030, owner="new"), 16030) + + def test_normal_release_is_session_qualified_but_release_all_is_explicitly_scoped(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + owner = PortPlanner( + lock_dir=tmpdir, + session_id="session-a", + availability_checker=lambda _port: True, + ) + foreign = PortPlanner( + lock_dir=tmpdir, + session_id="session-b", + availability_checker=lambda _port: True, + ) + port = owner.reserve_next(17000, owner="Main:backend") + + foreign.release(port, owner="Main:backend") + self.assertTrue((Path(tmpdir) / f"{port}.lock").exists()) + foreign.release_all() + self.assertFalse((Path(tmpdir) / f"{port}.lock").exists()) + + def test_reap_stale_never_removes_a_live_foreign_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "18000.lock" + lock_path.write_text( + json.dumps({"owner": "worker", "session": "old", "pid": os.getpid(), "created_at": 0}), + encoding="utf-8", + ) + planner = PortPlanner(lock_dir=tmpdir, session_id="new", availability_checker=lambda _port: True) + + self.assertFalse(planner.reap_stale(18000, owner="worker")) + self.assertTrue(lock_path.exists()) + lock_path.write_text( + json.dumps({"owner": "worker", "session": "old", "pid": 99999999, "created_at": 0}), + encoding="utf-8", + ) + self.assertTrue(planner.reap_stale(18000, owner="worker")) + self.assertFalse(lock_path.exists()) + + def test_release_owned_requires_exact_owner_and_original_session(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + original = PortPlanner( + lock_dir=tmpdir, + session_id="old-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + port = original.reserve_next(19000, owner="Main:backend") + + self.assertFalse(cleanup.release_owned(port, "Other:backend", expected_session="old-session")) + self.assertTrue((Path(tmpdir) / f"{port}.lock").exists()) + self.assertFalse(cleanup.release_owned(port, "Main:backend", expected_session="cleanup-session")) + self.assertFalse(cleanup.release_owned(port, "Main:backend", expected_session="")) + self.assertTrue((Path(tmpdir) / f"{port}.lock").exists()) + self.assertTrue(cleanup.release_owned(port, "Main:backend", expected_session="old-session")) + self.assertFalse((Path(tmpdir) / f"{port}.lock").exists()) + + def test_bulk_stale_reap_preserves_live_and_non_numeric_locks(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + (root / "19100.lock").write_text( + json.dumps({"owner": "dead", "session": "old", "pid": 99999999, "created_at": 0}), + encoding="utf-8", + ) + (root / "19101.lock").write_text( + json.dumps({"owner": "live", "session": "live", "pid": os.getpid(), "created_at": 0}), + encoding="utf-8", + ) + (root / "19102.lock").write_text("{}", encoding="utf-8") + os.utime(root / "19102.lock", (0, 0)) + (root / "not-a-port.lock").write_text("invalid", encoding="utf-8") + planner = PortPlanner(lock_dir=tmpdir, session_id="cleanup", corrupt_lock_grace_seconds=1.0) + + self.assertEqual(planner.reap_stale_locks(), 2) + self.assertFalse((root / "19100.lock").exists()) + self.assertTrue((root / "19101.lock").exists()) + self.assertFalse((root / "19102.lock").exists()) + self.assertTrue((root / "not-a-port.lock").exists()) + + def test_event_handler_failure_cannot_hide_a_successful_acquire_or_release(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="event-failure", + availability_checker=lambda _port: True, + event_handler=lambda _name, _payload: (_ for _ in ()).throw(RuntimeError("telemetry failed")), + ) + + port = planner.reserve_next(19200, owner="worker") + + self.assertEqual(port, 19200) + self.assertTrue((Path(tmpdir) / "19200.lock").exists()) + planner.release(port, owner="worker") + self.assertFalse((Path(tmpdir) / "19200.lock").exists()) + + def test_lock_and_guard_directories_reject_symlinks(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + external = root / "external" + external.mkdir() + lock_link = root / "lock-link" + lock_link.symlink_to(external, target_is_directory=True) + + with self.assertRaisesRegex(OSError, "not a real directory"): + PortPlanner(lock_dir=str(lock_link)) + + lock_dir = root / "locks" + lock_dir.mkdir() + (lock_dir / ".port-guards").symlink_to(external, target_is_directory=True) + planner = PortPlanner( + lock_dir=str(lock_dir), + session_id="symlink-guard", + availability_checker=lambda _port: True, + ) + with self.assertRaisesRegex(OSError, "not a real directory"): + planner.reserve_next(19300, owner="worker") + self.assertEqual(list(external.iterdir()), []) + + def test_guard_artifacts_are_bounded_by_the_fixed_shard_count(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="bounded", + availability_checker=lambda _port: True, + ) + for port in range(20000, 20128): + self.assertEqual(planner.reserve_next(port, owner=f"worker-{port}"), port) + planner.release(port, owner=f"worker-{port}") + + guard_dir = Path(tmpdir) / ".port-guards" + self.assertLessEqual(len(list(guard_dir.glob("*.guard"))), 64) + self.assertEqual(list(guard_dir.glob("*.pending")), []) + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/shared/test_process_cwd.py b/tests/python/shared/test_process_cwd.py new file mode 100644 index 000000000..3843c6d14 --- /dev/null +++ b/tests/python/shared/test_process_cwd.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import subprocess +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from envctl_engine.shared.process_cwd import parse_lsof_cwd, process_cwd + + +class ProcessCwdTests(unittest.TestCase): + def test_parse_lsof_cwd_ignores_pid_record_and_returns_name_record(self) -> None: + self.assertEqual(parse_lsof_cwd("p123\nfcwd\nn/repo/backend\n"), "/repo/backend") + self.assertIsNone(parse_lsof_cwd("p123\nfcwd\n")) + + def test_process_cwd_prefers_proc_symlink_without_spawning_lsof(self) -> None: + with ( + patch("envctl_engine.shared.process_cwd.os.readlink", return_value="/repo/from-proc"), + patch("envctl_engine.shared.process_cwd.subprocess.run") as run_mock, + ): + result = process_cwd(123) + + self.assertEqual(result, "/repo/from-proc") + run_mock.assert_not_called() + + def test_process_cwd_falls_back_to_bounded_noninteractive_lsof(self) -> None: + with ( + patch("envctl_engine.shared.process_cwd.os.readlink", side_effect=OSError), + patch( + "envctl_engine.shared.process_cwd.subprocess.run", + return_value=SimpleNamespace(returncode=0, stdout="p123\nfcwd\nn/repo/from-lsof\n"), + ) as run_mock, + ): + result = process_cwd(123) + + self.assertEqual(result, "/repo/from-lsof") + self.assertEqual( + run_mock.call_args.args[0], + ["lsof", "-a", "-p", "123", "-d", "cwd", "-Fn"], + ) + self.assertEqual(run_mock.call_args.kwargs["timeout"], 1.0) + self.assertIs(run_mock.call_args.kwargs["stdin"], subprocess.DEVNULL) + + def test_process_cwd_returns_none_when_lsof_times_out(self) -> None: + with ( + patch("envctl_engine.shared.process_cwd.os.readlink", side_effect=OSError), + patch( + "envctl_engine.shared.process_cwd.subprocess.run", + side_effect=subprocess.TimeoutExpired(["lsof"], 1.0), + ), + ): + self.assertIsNone(process_cwd(123)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/shared/test_process_runner_listener_detection.py b/tests/python/shared/test_process_runner_listener_detection.py index cfa9824d7..45ff0d0b7 100644 --- a/tests/python/shared/test_process_runner_listener_detection.py +++ b/tests/python/shared/test_process_runner_listener_detection.py @@ -1,7 +1,9 @@ from __future__ import annotations +import os import signal import subprocess +import sys import unittest from pathlib import Path from unittest.mock import patch @@ -274,7 +276,9 @@ def _fake_killpg(pid: int, sig: int) -> None: with ( patch("envctl_engine.shared.process_runner.os.killpg", side_effect=_fake_killpg), - patch.object(runner, "is_pid_running", return_value=False), + patch("envctl_engine.shared.process_runner.os.getpgid", side_effect=lambda pid: pid), + patch.object(runner, "_process_group_member_pids", return_value={4321}), + patch.object(runner, "_wait_for_process_group_exit", return_value=True), patch.object(runner, "_pid_identity", return_value="orig", create=True), ): result = runner.terminate_process_group(4321, term_timeout=0.1, kill_timeout=0.1) @@ -291,13 +295,10 @@ def _fake_killpg(pid: int, sig: int) -> None: with ( patch("envctl_engine.shared.process_runner.os.killpg", side_effect=_fake_killpg), - patch.object(runner, "is_pid_running", side_effect=[True, True, False]), + patch("envctl_engine.shared.process_runner.os.getpgid", side_effect=lambda pid: pid), + patch.object(runner, "_process_group_member_pids", return_value={5678}), + patch.object(runner, "_wait_for_process_group_exit", side_effect=[False, True]), patch.object(runner, "_pid_identity", side_effect=["orig"] * 8, create=True), - patch( - "envctl_engine.shared.process_runner.time.monotonic", - side_effect=[0.0, 0.0, 0.2, 0.2, 0.2, 0.4], - ), - patch("envctl_engine.shared.process_runner.time.sleep", return_value=None), ): result = runner.terminate_process_group(5678, term_timeout=0.1, kill_timeout=0.1) @@ -313,19 +314,120 @@ def _fake_killpg(pid: int, sig: int) -> None: with ( patch("envctl_engine.shared.process_runner.os.killpg", side_effect=_fake_killpg), - patch.object(runner, "is_pid_running", return_value=True), + patch("envctl_engine.shared.process_runner.os.getpgid", side_effect=lambda pid: pid), + patch.object(runner, "_process_group_member_pids", return_value={8765}), + patch.object(runner, "_wait_for_process_group_exit", return_value=False), patch.object(runner, "_pid_identity", side_effect=["orig", "reused"], create=True), - patch( - "envctl_engine.shared.process_runner.time.monotonic", - side_effect=[0.0, 0.0, 0.2, 0.2, 0.2, 0.4], - ), - patch("envctl_engine.shared.process_runner.time.sleep", return_value=None), ): result = runner.terminate_process_group(8765, term_timeout=0.1, kill_timeout=0.1) - self.assertTrue(result) + self.assertFalse(result) self.assertEqual(killpg_calls, [(8765, signal.SIGTERM)]) + def test_terminate_process_group_kills_term_ignoring_descendant_after_leader_exits(self) -> None: + runner = ProcessRunner() + parent = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "import subprocess,sys,time; " + "subprocess.Popen([sys.executable, '-c', " + "'import os,signal,time; signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "print(os.getpid(), flush=True); time.sleep(60)']); " + "time.sleep(60)" + ), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) + child_pid = 0 + try: + assert parent.stdout is not None + child_pid = int(parent.stdout.readline().strip()) + + self.assertTrue(runner.terminate_process_group(parent.pid, term_timeout=0.2, kill_timeout=2.0)) + parent.wait(timeout=2.0) + self.assertFalse(runner.is_pid_running(child_pid)) + finally: + try: + os.killpg(parent.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + def test_terminate_process_group_resolves_dead_leaders_group_from_live_orphan_member(self) -> None: + runner = ProcessRunner() + parent = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "import os,time; " + "child=os.fork(); " + "print(child, flush=True) if child else None; " + "os._exit(0) if child else time.sleep(60)" + ), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) + child_pid = 0 + try: + assert parent.stdout is not None + child_pid = int(parent.stdout.readline().strip()) + parent.wait(timeout=2.0) + self.assertEqual(os.getpgid(child_pid), parent.pid) + + self.assertTrue(runner.terminate_process_group(child_pid, term_timeout=0.5, kill_timeout=2.0)) + self.assertFalse(runner.is_pid_running(child_pid)) + finally: + try: + os.killpg(parent.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + def test_terminate_process_group_does_not_treat_live_eperm_as_success(self) -> None: + runner = ProcessRunner() + + with ( + patch("envctl_engine.shared.process_runner.os.killpg", side_effect=PermissionError), + patch("envctl_engine.shared.process_runner.os.getpgid", return_value=8766), + patch.object(runner, "is_pid_running", return_value=True), + patch.object(runner, "_pid_identity", return_value="orig", create=True), + ): + result = runner.terminate_process_group(8766, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + + def test_terminate_process_group_does_not_treat_missing_group_for_live_pid_as_success(self) -> None: + runner = ProcessRunner() + + with ( + patch("envctl_engine.shared.process_runner.os.killpg", side_effect=ProcessLookupError), + patch("envctl_engine.shared.process_runner.os.getpgid", return_value=8767), + patch.object(runner, "is_pid_running", return_value=True), + patch.object(runner, "_pid_identity", return_value="orig", create=True), + ): + result = runner.terminate_process_group(8767, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + + def test_terminate_does_not_treat_live_eperm_as_success(self) -> None: + runner = ProcessRunner() + + with ( + patch("envctl_engine.shared.process_runner.os.kill", side_effect=PermissionError), + patch.object(runner, "is_pid_running", return_value=True), + patch.object(runner, "_pid_identity", return_value="orig", create=True), + ): + result = runner.terminate(8768, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + def test_is_pid_running_treats_zombie_process_as_not_running(self) -> None: runner = ProcessRunner() @@ -340,6 +442,12 @@ def test_is_pid_running_treats_zombie_process_as_not_running(self) -> None: ): self.assertFalse(runner.is_pid_running(4321)) + def test_is_pid_running_treats_permission_denied_as_still_running(self) -> None: + runner = ProcessRunner() + + with patch("envctl_engine.shared.process_runner.os.kill", side_effect=PermissionError): + self.assertTrue(runner.is_pid_running(4321)) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/shared/test_process_termination.py b/tests/python/shared/test_process_termination.py new file mode 100644 index 000000000..dcd09238a --- /dev/null +++ b/tests/python/shared/test_process_termination.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import os +import signal +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from envctl_engine.shared.process_termination import terminate_pid + + +class ProcessTerminationTests(unittest.TestCase): + def test_terminate_pid_refuses_self_and_parent_without_signalling(self) -> None: + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: self.fail("protected PID must not be signalled"), + ) + + self.assertFalse(terminate_pid(os.getpid(), process_runner=runner, term_timeout=0.0, kill_timeout=0.0)) + self.assertFalse(terminate_pid(os.getppid(), process_runner=runner, term_timeout=0.0, kill_timeout=0.0)) + + def test_failed_group_termination_does_not_kill_only_the_leader(self) -> None: + runner_calls: list[str] = [] + sent: list[signal.Signals] = [] + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: runner_calls.append("group") or False, + terminate=lambda *_args, **_kwargs: runner_calls.append("single") or False, + ) + + with ( + patch( + "envctl_engine.shared.process_termination._send_signal", + side_effect=lambda _pid, requested_signal: sent.append(requested_signal) or True, + ), + patch("envctl_engine.shared.process_termination.wait_for_pid_exit", side_effect=[False, False]), + ): + result = terminate_pid(987654, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + self.assertEqual(runner_calls, ["group"]) + self.assertEqual(sent, []) + + def test_runner_without_group_support_falls_back_to_raw_signals(self) -> None: + sent: list[signal.Signals] = [] + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate=lambda *_args, **_kwargs: False, + ) + + with ( + patch( + "envctl_engine.shared.process_termination._send_signal", + side_effect=lambda _pid, requested_signal: sent.append(requested_signal) or True, + ), + patch("envctl_engine.shared.process_termination.wait_for_pid_exit", side_effect=[False, False]), + ): + result = terminate_pid(987654, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + self.assertEqual(sent, [signal.SIGTERM, signal.SIGKILL]) + + def test_runner_success_is_not_trusted_while_same_identified_pid_remains_live(self) -> None: + sent: list[signal.Signals] = [] + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + _pid_identity=lambda _pid: "original", + terminate_process_group=lambda *_args, **_kwargs: True, + ) + + with patch( + "envctl_engine.shared.process_termination._send_signal", + side_effect=lambda _pid, requested_signal: sent.append(requested_signal) or True, + ): + result = terminate_pid(987655, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + self.assertEqual(sent, []) + + def test_identity_incapable_runner_success_is_not_trusted_while_pid_remains_live(self) -> None: + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: True, + ) + + result = terminate_pid(987657, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + + def test_identity_replacement_after_failed_group_cleanup_is_not_reported_as_success(self) -> None: + identities = iter(("original", "replacement")) + sent: list[signal.Signals] = [] + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + _pid_identity=lambda _pid: next(identities), + terminate_process_group=lambda *_args, **_kwargs: False, + ) + + with patch( + "envctl_engine.shared.process_termination._send_signal", + side_effect=lambda _pid, requested_signal: sent.append(requested_signal) or True, + ): + result = terminate_pid(987656, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + self.assertEqual(sent, []) + + def test_group_cleanup_exception_is_not_reported_as_success_after_leader_replacement(self) -> None: + identities = iter(("original", "replacement")) + sent: list[signal.Signals] = [] + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + _pid_identity=lambda _pid: next(identities), + terminate_process_group=lambda *_args, **_kwargs: (_ for _ in ()).throw( + OSError("post-kill reporting failed") + ), + ) + + with patch( + "envctl_engine.shared.process_termination._send_signal", + side_effect=lambda _pid, requested_signal: sent.append(requested_signal) or True, + ): + result = terminate_pid(987658, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + self.assertEqual(sent, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/startup/test_requirements_project_startup.py b/tests/python/startup/test_requirements_project_startup.py new file mode 100644 index 000000000..5769e0f70 --- /dev/null +++ b/tests/python/startup/test_requirements_project_startup.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from envctl_engine.requirements.orchestrator import FailureClass, RequirementOutcome +from envctl_engine.shared.ports import PortPlanner +from envctl_engine.startup.requirements_component_startup import RequirementComponentStarter +from envctl_engine.startup.requirements_project_startup import RequirementProjectStarter +from envctl_engine.state.models import PortPlan + + +class RequirementProjectStarterTests(unittest.TestCase): + @staticmethod + def _plan(port: int) -> PortPlan: + return PortPlan( + project="Main", + requested=port, + assigned=port, + final=port, + source="test", + ) + + def _starter(self, *, parallel: bool) -> RequirementProjectStarter: + postgres = SimpleNamespace( + id="postgres", + resources=(SimpleNamespace(name="db", legacy_port_key="db"),), + ) + redis = SimpleNamespace( + id="redis", + resources=(SimpleNamespace(name="redis", legacy_port_key="redis"),), + ) + + def start_component(_context, component, plan, _reserve_next, **_kwargs): # noqa: ANN001, ANN202 + if component == "redis": + raise RuntimeError("redis native startup crashed") + return RequirementOutcome( + service_name=component, + success=True, + requested_port=plan.requested, + final_port=plan.final, + retries=0, + container_name="postgres-main", + ) + + runtime = SimpleNamespace( + env={"ENVCTL_REQUIREMENTS_PARALLEL": "true" if parallel else "false"}, + config=SimpleNamespace(raw={}, strict_n8n_bootstrap=False), + port_planner=SimpleNamespace(session_id="requirements-session"), + _emit=lambda *_args, **_kwargs: None, + _start_requirement_component=start_component, + ) + return RequirementProjectStarter( + orchestrator=SimpleNamespace(runtime=runtime), + context=SimpleNamespace( + name="Main", + ports={"db": self._plan(5432), "redis": self._plan(6379)}, + ), + mode="main", + definitions=[postgres, redis], + ) + + def test_sequential_component_exception_becomes_partial_failed_outcome(self) -> None: + starter = self._starter(parallel=False) + + starter._run_enabled_components(starter.definitions) + + self.assertTrue(starter.outcomes["postgres"].success) + self.assertFalse(starter.outcomes["redis"].success) + self.assertEqual(starter.outcomes["redis"].final_port, 6379) + self.assertIn("native startup crashed", str(starter.outcomes["redis"].error)) + + def test_parallel_component_exception_waits_for_and_preserves_other_outcomes(self) -> None: + starter = self._starter(parallel=True) + + with patch( + "envctl_engine.startup.requirements_project_startup.requirements_parallel_enabled", + return_value=True, + ): + starter._run_enabled_components(starter.definitions) + + self.assertEqual(set(starter.outcomes), {"postgres", "redis"}) + self.assertTrue(starter.outcomes["postgres"].success) + self.assertFalse(starter.outcomes["redis"].success) + + def test_managed_components_record_exact_port_planner_session(self) -> None: + starter = self._starter(parallel=False) + starter._run_enabled_components(starter.definitions) + + components = starter._build_components( + {"postgres": True, "redis": True}, + {"postgres": False, "redis": False}, + ) + + self.assertEqual( + components["postgres"]["port_lock_session"], + "requirements-session", + ) + self.assertEqual( + components["redis"]["port_lock_session"], + "requirements-session", + ) + + def test_multiple_requirement_rebinds_release_every_failed_port_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="startup-session", + availability_checker=lambda _port: True, + ) + planner.reserve_next(5432, owner="Main:db") + starter = object.__new__(RequirementComponentStarter) + starter.runtime = SimpleNamespace( + port_planner=planner, + requirements=SimpleNamespace(reason_code_for_failure=lambda *_args, **_kwargs: "port_in_use"), + _emit=lambda *_args, **_kwargs: None, + ) + starter.context = SimpleNamespace(name="Main") + starter.name = "postgres" + + retry_one = planner.reserve_next(5433, owner="Main:requirements") + starter._on_retry( + "postgres", + 5432, + retry_one, + 1, + FailureClass.BIND_CONFLICT_RETRYABLE, + "address already in use", + ) + retry_two = planner.reserve_next(5434, owner="Main:requirements") + starter._on_retry( + "postgres", + retry_one, + retry_two, + 2, + FailureClass.BIND_CONFLICT_RETRYABLE, + "address already in use", + ) + + locks = list(Path(tmpdir).glob("*.lock")) + self.assertEqual([path.name for path in locks], ["5434.lock"]) + payload = json.loads(locks[0].read_text(encoding="utf-8")) + self.assertEqual(payload["owner"], "Main:requirements") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/startup/test_restart_prestop_support.py b/tests/python/startup/test_restart_prestop_support.py index 49bc46997..b7c94dc3f 100644 --- a/tests/python/startup/test_restart_prestop_support.py +++ b/tests/python/startup/test_restart_prestop_support.py @@ -1,9 +1,12 @@ from __future__ import annotations import unittest +import tempfile +from pathlib import Path from types import SimpleNamespace from envctl_engine.runtime.command_router import Route +from envctl_engine.shared.ports import PortPlanner from envctl_engine.startup.restart_prestop_support import ( RestartOrphanListenerScan, apply_restart_port_assignments, @@ -105,11 +108,14 @@ def test_handle_restart_prestop_stops_selected_services_and_preserves_remaining_ restart_state=None, preserved_requirements={}, preserved_services={}, + base_metadata={"new": True}, ) state = SimpleNamespace( + run_id="run-old", mode="main", services={"Main Backend": object(), "Other Backend": object()}, requirements={"Main": object(), "Other": object()}, + metadata={"state_source_run_ids": ["run-ancestor"], "old": True}, ) events: list[tuple[str, dict[str, object]]] = [] released: list[object] = [] @@ -122,8 +128,8 @@ def test_handle_restart_prestop_stops_selected_services_and_preserves_remaining_ _effective_start_mode=lambda route: "main", _try_load_existing_state=lambda *, mode: state, _project_name_from_service=lambda name: str(name).removesuffix(" Backend"), - _terminate_services_from_state=lambda state, *, selected_services, aggressive, verify_ownership: terminated.append( - (state, set(selected_services)) + _terminate_services_from_state=lambda state, *, selected_services, aggressive, verify_ownership: ( + terminated.append((state, set(selected_services))) ), _release_requirement_ports=released.append, _emit=lambda event, **payload: events.append((event, payload)), @@ -148,10 +154,112 @@ def test_handle_restart_prestop_stops_selected_services_and_preserves_remaining_ self.assertEqual(released, [state.requirements["Main"]]) self.assertEqual(session.preserved_services, {"Other Backend": state.services["Other Backend"]}) self.assertEqual(session.preserved_requirements, {"Other": state.requirements["Other"]}) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-ancestor", "run-old"]) + self.assertTrue(session.base_metadata["old"]) + self.assertTrue(session.base_metadata["new"]) self.assertEqual(session.effective_route.command, "start") self.assertEqual(session.effective_route.flags["_restart_selected_services"], ["Main Backend"]) self.assertEqual(spinner.events, ["enter", "succeed:Restart pre-stop complete", "exit"]) - self.assertIn(("restart.selection", {"include_requirements": True, "target_projects": ["Main"], "selected_services": ["Main Backend"]}), events) + self.assertIn( + ( + "restart.selection", + {"include_requirements": True, "target_projects": ["Main"], "selected_services": ["Main Backend"]}, + ), + events, + ) + + def test_handle_restart_prestop_aborts_without_mutating_session_when_termination_fails(self) -> None: + route = Route( + command="restart", + mode="main", + flags={"services": ["Main Backend"]}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="main", + restart_state=None, + preserved_requirements={}, + preserved_services={}, + base_metadata={}, + ) + state = SimpleNamespace( + run_id="run-old", + mode="main", + services={"Main Backend": object()}, + requirements={"Main": object()}, + metadata={}, + ) + runtime = SimpleNamespace( + env={}, + _effective_start_mode=lambda _route: "main", + _try_load_existing_state=lambda *, mode: state, + _project_name_from_service=lambda _name: "Main", + _terminate_services_from_state=lambda *_args, **_kwargs: {"Main Backend"}, + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(RuntimeError, "Main Backend"): + handle_restart_prestop( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: True, + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("orphans must not be reaped"), + spinner_factory=lambda *_args, **_kwargs: _RecordingSpinner(), + use_spinner_policy_fn=lambda _policy: _NullContext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + ) + + self.assertEqual(session.preserved_services, {}) + self.assertEqual(session.preserved_requirements, {}) + self.assertEqual(session.base_metadata, {}) + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertIs(session.restart_state, state) + self.assertIs(session.effective_route, route) + + def test_handle_restart_prestop_aborts_when_orphan_exit_is_unconfirmed(self) -> None: + route = Route(command="restart", mode="main", flags={"services": ["Main Backend"]}) + session = SimpleNamespace( + effective_route=route, + runtime_mode="main", + restart_state=None, + preserved_requirements={}, + preserved_services={}, + base_metadata={}, + ) + state = SimpleNamespace( + run_id="run-old", + mode="main", + services={"Main Backend": object()}, + requirements={"Main": object()}, + metadata={}, + ) + runtime = SimpleNamespace( + env={}, + _effective_start_mode=lambda _route: "main", + _try_load_existing_state=lambda *, mode: state, + _project_name_from_service=lambda _name: "Main", + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(RuntimeError, "orphan listeners.*4242"): + handle_restart_prestop( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: True, + terminate_restart_orphan_listeners=lambda **_kwargs: {4242}, + spinner_factory=lambda *_args, **_kwargs: _RecordingSpinner(), + use_spinner_policy_fn=lambda _policy: _NullContext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, {}) + self.assertIs(session.effective_route, route) def test_restart_fallback_start_route_preserves_request_context_and_marks_restart(self) -> None: route = Route( @@ -502,7 +610,7 @@ def test_terminate_restart_orphan_listeners_releases_successfully_terminated_por released: list[int] = [] terminated: list[tuple[int, float, float]] = [] - terminate_restart_orphan_listeners( + failed = terminate_restart_orphan_listeners( state=state, selected_services={"Main Backend"}, aggressive=True, @@ -511,15 +619,121 @@ def test_terminate_restart_orphan_listeners_releases_successfully_terminated_por port_spacing=0, listener_pids_for_port=lambda port: [42] if port == 8000 else [], process_cwd=lambda pid: "/repo/backend" if pid == 42 else None, - terminate_pid=lambda pid, *, term_timeout, kill_timeout: terminated.append( - (pid, term_timeout, kill_timeout) - ) - or True, - release_port=released.append, + terminate_pid=lambda pid, *, term_timeout, kill_timeout: ( + terminated.append((pid, term_timeout, kill_timeout)) or True + ), + port_planner=SimpleNamespace(release=released.append), ) self.assertEqual(terminated, [(42, 0.5, 1.0)]) self.assertEqual(released, [8000]) + self.assertEqual(failed, set()) + + def test_restart_orphan_cleanup_releases_verified_prior_session_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(8000, owner="Main:backend") + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace( + project="Main", + type="backend", + cwd="/repo/backend", + actual_port=8000, + port_lock_session="prior-session", + ) + } + ) + + failed = terminate_restart_orphan_listeners( + state=state, + selected_services={"Main Backend"}, + aggressive=True, + backend_port_base=8000, + frontend_port_base=3000, + port_spacing=0, + listener_pids_for_port=lambda port: [42] if port == 8000 else [], + process_cwd=lambda pid: "/repo/backend" if pid == 42 else None, + terminate_pid=lambda *_args, **_kwargs: True, + port_planner=current, + ) + + self.assertEqual(failed, set()) + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + + def test_stale_restart_state_cannot_release_newer_same_owner_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + newer = PortPlanner( + lock_dir=tmpdir, + session_id="newer-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + newer.reserve_next(8000, owner="Main:backend") + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace( + project="Main", + type="backend", + cwd="/repo/backend", + actual_port=8000, + port_lock_session="stale-session", + ) + } + ) + + failed = terminate_restart_orphan_listeners( + state=state, + selected_services={"Main Backend"}, + aggressive=True, + backend_port_base=8000, + frontend_port_base=3000, + port_spacing=0, + listener_pids_for_port=lambda port: [42] if port == 8000 else [], + process_cwd=lambda pid: "/repo/backend" if pid == 42 else None, + terminate_pid=lambda *_args, **_kwargs: True, + port_planner=cleanup, + ) + + self.assertEqual(failed, set()) + self.assertTrue((Path(tmpdir) / "8000.lock").exists()) + + def test_terminate_restart_orphan_listeners_reports_unconfirmed_exits_without_releasing_ports(self) -> None: + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace(type="backend", cwd="/repo/backend", actual_port=8000), + } + ) + released: list[int] = [] + + failed = terminate_restart_orphan_listeners( + state=state, + selected_services={"Main Backend"}, + aggressive=True, + backend_port_base=8000, + frontend_port_base=3000, + port_spacing=0, + listener_pids_for_port=lambda port: [42] if port == 8000 else [], + process_cwd=lambda pid: "/repo/backend" if pid == 42 else None, + terminate_pid=lambda *_args, **_kwargs: False, + port_planner=SimpleNamespace(release=released.append), + ) + + self.assertEqual(failed, {42}) + self.assertEqual(released, []) if __name__ == "__main__": diff --git a/tests/python/startup/test_run_reuse_identity.py b/tests/python/startup/test_run_reuse_identity.py index 15cd8531d..6496c5771 100644 --- a/tests/python/startup/test_run_reuse_identity.py +++ b/tests/python/startup/test_run_reuse_identity.py @@ -1,5 +1,7 @@ from __future__ import annotations +import hashlib +import json import unittest from pathlib import Path from types import SimpleNamespace @@ -49,6 +51,52 @@ def test_build_startup_identity_metadata_records_roots_and_fingerprint_payload(s self.assertEqual(identity["services"], {"backend": True, "frontend": False}) self.assertIn("fingerprint", identity) + def test_build_startup_identity_metadata_preserves_unselected_roots_without_case_duplicates(self) -> None: + runtime = SimpleNamespace( + config=SimpleNamespace( + backend_dir_name="api", + frontend_dir_name="web", + additional_services=(), + startup_enabled_for_mode=lambda _mode: True, + service_enabled_for_mode=lambda _mode, service: service == "backend", + requirement_enabled_for_mode=lambda _mode, _requirement: False, + ) + ) + + metadata = build_startup_identity_metadata( + runtime, + runtime_mode="trees", + project_contexts=[SimpleNamespace(name="featurea", root=Path("/tmp/new-a"))], + base_metadata={ + "project_roots": { + "FEATUREA": "/tmp/old-a", + "FeatureB": "/tmp/feature-b", + } + }, + route=Route(command="start", mode="trees", flags={}), + ) + + self.assertEqual( + metadata["project_roots"], + { + "featurea": normalize_project_root("/tmp/new-a"), + "FeatureB": normalize_project_root("/tmp/feature-b"), + }, + ) + identity = metadata["startup_identity"] + self.assertEqual( + identity["projects"], + [ + {"name": "featurea", "root": normalize_project_root("/tmp/new-a")}, + {"name": "FeatureB", "root": normalize_project_root("/tmp/feature-b")}, + ], + ) + fingerprint_payload = {key: value for key, value in identity.items() if key != "fingerprint"} + self.assertEqual( + identity["fingerprint"], + hashlib.sha256(json.dumps(fingerprint_payload, sort_keys=True).encode("utf-8")).hexdigest(), + ) + def test_project_identities_from_state_uses_metadata_requirements_and_services(self) -> None: state = RunState( run_id="run-1", @@ -83,7 +131,9 @@ def test_context_identity_sorting_and_weak_keys_are_stable(self) -> None: def test_startup_identity_mismatch_ignores_matching_fingerprints_and_reports_changed_fields(self) -> None: self.assertEqual( - _startup_identity_mismatch({"fingerprint": "same", "mode": "main"}, {"fingerprint": "same", "mode": "trees"}), + _startup_identity_mismatch( + {"fingerprint": "same", "mode": "main"}, {"fingerprint": "same", "mode": "trees"} + ), {}, ) diff --git a/tests/python/startup/test_run_reuse_support.py b/tests/python/startup/test_run_reuse_support.py index 566c0b45e..bbd34f3cc 100644 --- a/tests/python/startup/test_run_reuse_support.py +++ b/tests/python/startup/test_run_reuse_support.py @@ -44,7 +44,9 @@ def test_dashboard_stopped_service_entries_normalizes_valid_backend_frontend_ent def test_dashboard_stopped_service_entries_ignores_missing_or_invalid_metadata(self) -> None: self.assertEqual(dashboard_stopped_service_entries(SimpleNamespace(metadata={})), []) - self.assertEqual(dashboard_stopped_service_entries(SimpleNamespace(metadata={"dashboard_stopped_services": {}})), []) + self.assertEqual( + dashboard_stopped_service_entries(SimpleNamespace(metadata={"dashboard_stopped_services": {}})), [] + ) def test_metadata_without_dashboard_stopped_services_removes_restored_entries_only(self) -> None: metadata = { @@ -172,8 +174,7 @@ def test_prepare_dashboard_stopped_service_restore_ignores_active_or_unselected_ ) ) - def test_fresh_start_replacement_services_selects_configured_service_types_for_target_projects(self) -> None: - route = Route(command="start", mode="main", raw_args=["start"], flags={}) + def test_fresh_start_replacement_services_selects_every_service_for_target_projects(self) -> None: candidate_state = SimpleNamespace( services={ "Main Backend": SimpleNamespace(name="Main Backend", project="Main", type="backend"), @@ -184,20 +185,36 @@ def test_fresh_start_replacement_services_selects_configured_service_types_for_t self.assertEqual( fresh_start_replacement_services( - route=route, selected_contexts=[SimpleNamespace(name="Main")], candidate_state=candidate_state, - configured_service_types={"backend"}, - additional_services=(), project_name_from_service=lambda name: str(name).removesuffix(" Backend").removesuffix(" Frontend"), ), - {"Main Backend"}, + {"Main Backend", "Main Frontend"}, ) def test_replace_existing_project_services_for_fresh_start_terminates_selected_services_and_orphans(self) -> None: route = Route(command="start", mode="trees", raw_args=["start"], flags={}) - session = SimpleNamespace(effective_route=route, runtime_mode="trees") - candidate_state = SimpleNamespace(run_id="run-existing") + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="Main")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + candidate_state = SimpleNamespace( + run_id="run-existing", + services={ + "Main Backend": SimpleNamespace(project="Main"), + "Main Frontend": SimpleNamespace(project="Main"), + "Other Backend": SimpleNamespace(project="Other"), + }, + requirements={ + "Main": SimpleNamespace(project="Main"), + "Other": SimpleNamespace(project="Other"), + }, + metadata={"state_source_run_ids": ["run-ancestor"], "keep": True}, + ) events: list[tuple[str, dict[str, object]]] = [] announced: list[object] = [] progress: list[tuple[Route, str]] = [] @@ -205,17 +222,19 @@ def test_replace_existing_project_services_for_fresh_start_terminates_selected_s orphaned: list[tuple[object, set[str], bool]] = [] runtime = SimpleNamespace( _emit=lambda event, **payload: events.append((event, payload)), - _terminate_services_from_state=lambda state, *, selected_services, aggressive, verify_ownership: terminated.append( - (state, set(selected_services), aggressive, verify_ownership) + _terminate_services_from_state=lambda state, *, selected_services, aggressive, verify_ownership: ( + terminated.append((state, set(selected_services), aggressive, verify_ownership)) ), + _project_name_from_service=lambda name: str(name).removesuffix(" Backend").removesuffix(" Frontend"), + _release_requirement_ports=lambda requirements: released.append(requirements), ) + released: list[object] = [] replace_existing_project_services_for_fresh_start( runtime=runtime, session=session, candidate_state=candidate_state, reason="startup_fingerprint_mismatch", - fresh_start_replacement_services=lambda session, *, candidate_state: {"Main Backend", "Main Frontend"}, announce_session_identifiers=announced.append, report_progress=lambda route, message: progress.append((route, message)), terminate_restart_orphan_listeners=lambda *, state, selected_services, aggressive: orphaned.append( @@ -241,35 +260,251 @@ def test_replace_existing_project_services_for_fresh_start_terminates_selected_s ) self.assertEqual(terminated, [(candidate_state, {"Main Backend", "Main Frontend"}, False, True)]) self.assertEqual(orphaned, [(candidate_state, {"Main Backend", "Main Frontend"}, True)]) + self.assertEqual(released, [candidate_state.requirements["Main"]]) + self.assertEqual(session.preserved_services, {"Other Backend": candidate_state.services["Other Backend"]}) + self.assertEqual(session.preserved_requirements, {"Other": candidate_state.requirements["Other"]}) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-ancestor", "run-existing"]) + self.assertTrue(session.base_metadata["keep"]) + + def test_explicit_no_resume_replaces_matching_services_instead_of_leaking_a_second_copy(self) -> None: + route = Route(command="start", mode="trees", flags={"no_resume": True}) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": SimpleNamespace(project="feature-a")}, + requirements={}, + metadata={}, + ) + terminated: list[set[str]] = [] + progress: list[str] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _terminate_services_from_state=lambda _state, *, selected_services, **_kwargs: terminated.append( + set(selected_services) + ), + _project_name_from_service=lambda name: str(name).removesuffix(" Backend"), + _release_requirement_ports=lambda _requirements: None, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda _route, message: progress.append(message), + terminate_restart_orphan_listeners=lambda **_kwargs: None, + ) + + self.assertEqual(terminated, [{"feature-a Backend"}]) + self.assertEqual(progress, ["Auto-resume disabled; replacing 1 existing service(s)..."]) - def test_replace_existing_project_services_for_fresh_start_skips_non_mismatch_or_dependencies_scope(self) -> None: - runtime = SimpleNamespace(_emit=lambda *args, **kwargs: self.fail("replace should not emit")) - candidate_state = SimpleNamespace(run_id="run-existing") + def test_explicit_no_resume_failure_keeps_previous_state_authoritative(self) -> None: + route = Route(command="start", mode="trees", flags={"no_resume": True}) + original_service = SimpleNamespace(project="feature-a") + original_requirements = SimpleNamespace(project="feature-a") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": original_service}, + requirements={"feature-a": original_requirements}, + metadata={"keep": True}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={"sentinel": object()}, + preserved_requirements={"sentinel": object()}, + base_metadata={"sentinel": True}, + ) + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "feature-a", + _terminate_services_from_state=lambda *_args, **_kwargs: {"feature-a Backend"}, + _release_requirement_ports=lambda _requirements: self.fail("failed replacement must not release ports"), + ) - for reason, flags in ( - ("no_matching_state", {}), - ("startup_fingerprint_mismatch", {"runtime_scope": "dependencies"}), - ): - route = Route(command="start", mode="trees", raw_args=["start"], flags=flags) - session = SimpleNamespace(effective_route=route, runtime_mode="trees") + with self.assertRaisesRegex(RuntimeError, "feature-a Backend"): replace_existing_project_services_for_fresh_start( runtime=runtime, session=session, candidate_state=candidate_state, - reason=reason, - fresh_start_replacement_services=lambda session, *, candidate_state: self.fail("selection should skip"), - announce_session_identifiers=lambda session: self.fail("announce should skip"), - report_progress=lambda route, message: self.fail("progress should skip"), - terminate_restart_orphan_listeners=lambda **kwargs: self.fail("orphan cleanup should skip"), + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda _route, _message: None, + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("must not reap after failure"), ) - def test_fresh_start_replacement_services_honors_restart_service_type_filters(self) -> None: - route = Route( - command="restart", - mode="main", - raw_args=["restart"], - flags={"_restart_request": True, "restart_service_types": ["frontend"]}, + self.assertEqual(set(session.preserved_services), {"sentinel"}) + self.assertEqual(set(session.preserved_requirements), {"sentinel"}) + self.assertEqual(session.base_metadata, {"sentinel": True}) + self.assertTrue(session.preserve_existing_state_on_failure) + + def test_fresh_start_aborts_when_orphan_exit_is_unconfirmed(self) -> None: + route = Route(command="start", mode="trees", flags={"no_resume": True}) + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": SimpleNamespace(project="feature-a")}, + requirements={"feature-a": SimpleNamespace(project="feature-a")}, + metadata={}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "feature-a", + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), ) + + with self.assertRaisesRegex(RuntimeError, "orphan listeners.*4242"): + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda *_args: None, + terminate_restart_orphan_listeners=lambda **_kwargs: {4242}, + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, {}) + self.assertEqual(session.preserved_requirements, {}) + + def test_explicit_no_resume_supersedes_requirement_only_state(self) -> None: + route = Route(command="start", mode="trees", flags={"no_resume": True}) + target_requirements = SimpleNamespace(project="feature-a") + other_requirements = SimpleNamespace(project="feature-b") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={}, + requirements={"feature-a": target_requirements, "feature-b": other_requirements}, + metadata={}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + released: list[object] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "", + _terminate_services_from_state=lambda *_args, **_kwargs: self.fail("no service termination expected"), + _release_requirement_ports=released.append, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: self.fail("empty replacement needs no announcement"), + report_progress=lambda _route, _message: self.fail("empty replacement needs no progress"), + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("no orphan scan expected"), + ) + + self.assertEqual(released, [target_requirements]) + self.assertEqual(session.preserved_requirements, {"feature-b": other_requirements}) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-existing"]) + + def test_no_reusable_runtime_still_supersedes_requirement_only_state(self) -> None: + target_requirements = SimpleNamespace(project="feature-a") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={}, + requirements={"feature-a": target_requirements}, + metadata={"existing": True}, + ) + session = SimpleNamespace( + effective_route=Route(command="start", mode="trees", flags={}), + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + released: list[object] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "", + _terminate_services_from_state=lambda *_args, **_kwargs: self.fail("no services should be stopped"), + _release_requirement_ports=released.append, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="no_reusable_runtime", + announce_session_identifiers=lambda _session: self.fail("no services need an announcement"), + report_progress=lambda _route, _message: self.fail("no services need progress"), + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("no orphan scan expected"), + ) + + self.assertEqual(released, [target_requirements]) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-existing"]) + + def test_dependencies_only_fresh_start_preserves_services_but_replaces_selected_requirements(self) -> None: + target_service = SimpleNamespace(project="feature-a") + other_service = SimpleNamespace(project="feature-b") + target_requirements = SimpleNamespace(project="feature-a") + other_requirements = SimpleNamespace(project="feature-b") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": target_service, "feature-b Backend": other_service}, + requirements={"feature-a": target_requirements, "feature-b": other_requirements}, + metadata={}, + ) + session = SimpleNamespace( + effective_route=Route(command="start", mode="trees", flags={"runtime_scope": "dependencies"}), + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + released: list[object] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "feature-a", + _terminate_services_from_state=lambda *_args, **_kwargs: self.fail("app services must be preserved"), + _release_requirement_ports=released.append, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="startup_fingerprint_mismatch", + announce_session_identifiers=lambda _session: self.fail("app services must be preserved"), + report_progress=lambda _route, _message: self.fail("app services must be preserved"), + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("no orphan scan expected"), + ) + + self.assertEqual(session.preserved_services, candidate_state.services) + self.assertEqual(session.preserved_requirements, {"feature-b": other_requirements}) + self.assertEqual(released, [target_requirements]) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-existing"]) + + def test_fresh_start_replacement_services_does_not_leave_disabled_service_types_running(self) -> None: candidate_state = SimpleNamespace( services={ "Main Backend": SimpleNamespace(name="Main Backend", project="Main", type="backend"), @@ -279,14 +514,11 @@ def test_fresh_start_replacement_services_honors_restart_service_type_filters(se self.assertEqual( fresh_start_replacement_services( - route=route, selected_contexts=[SimpleNamespace(name="Main")], candidate_state=candidate_state, - configured_service_types={"backend", "frontend"}, - additional_services=(), project_name_from_service=lambda name: str(name).removesuffix(" Backend").removesuffix(" Frontend"), ), - {"Main Frontend"}, + {"Main Backend", "Main Frontend"}, ) diff --git a/tests/python/startup/test_selected_context_startup.py b/tests/python/startup/test_selected_context_startup.py index a752e410e..a3619d347 100644 --- a/tests/python/startup/test_selected_context_startup.py +++ b/tests/python/startup/test_selected_context_startup.py @@ -6,13 +6,15 @@ import unittest from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.service_manager import ServiceCleanupError +from envctl_engine.startup.startup_execution_support import ProjectStartupFailure from envctl_engine.startup.selected_context_startup import ( record_project_startup, start_selected_contexts, start_selected_contexts_with_runtime, ) -from envctl_engine.startup.session import ProjectStartupResult, StartupSession -from envctl_engine.state.models import RequirementsResult +from envctl_engine.startup.session import ProjectStartupResult, StartupSession, track_unterminated_services +from envctl_engine.state.models import RequirementsResult, ServiceRecord class _SpinnerStub: @@ -77,6 +79,109 @@ def _start_project_context(self, *, context, mode: str, route: Route, run_id: st class SelectedContextStartupTests(unittest.TestCase): + def test_parallel_renderer_error_occurs_only_after_successful_resources_are_recorded(self) -> None: + runtime = _RuntimeStub() + runtime._tree_parallel_startup_config = ( # type: ignore[method-assign] + lambda **_kwargs: (True, 2) + ) + route = Route(command="start", mode="trees", flags={}) + contexts = [self._context("Alpha"), self._context("Beta")] + session = self._session(route=route, contexts=contexts) + + with self.assertRaisesRegex(OSError, "renderer failed"): + start_selected_contexts( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: False, + resolved_run_id=lambda _session: "run-1", + record_project_startup=record_project_startup, + render_project_startup_warnings=lambda **_kwargs: (_ for _ in ()).throw(OSError("renderer failed")), + should_degrade_to_plan_agent_handoff=lambda _session, _error: False, + record_plan_agent_handoff_local_startup_failure=lambda **_kwargs: None, + spinner_factory=_SpinnerFactory(), + use_spinner_policy_fn=lambda _policy: nullcontext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False, backend="rich"), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + project_spinner_group_factory=lambda **_kwargs: SimpleNamespace(), + ) + + self.assertEqual(set(session.services_by_project), {"Alpha", "Beta"}) + self.assertEqual(set(session.requirements_by_project), {"Alpha", "Beta"}) + self.assertEqual(set(session.started_context_names), {"Alpha", "Beta"}) + + def test_parallel_failure_callback_error_does_not_hide_later_success(self) -> None: + runtime = _RuntimeStub() + runtime._tree_parallel_startup_config = lambda **_kwargs: (True, 2) # type: ignore[method-assign] + + def start_context(*, context, **_kwargs): # noqa: ANN001, ANN202 + if context.name == "Alpha": + raise RuntimeError("alpha failed") + return ProjectStartupResult( + requirements=RequirementsResult(project=context.name), + services={"Beta Backend": object()}, + ) + + runtime._start_project_context = start_context # type: ignore[method-assign] + original_emit = runtime._emit + + def emit(event: str, **payload: object) -> None: + if event == "startup.project.failed": + raise OSError("event sink failed") + original_emit(event, **payload) + + runtime._emit = emit # type: ignore[method-assign] + route = Route(command="start", mode="trees", flags={}) + session = self._session(route=route, contexts=[self._context("Alpha"), self._context("Beta")]) + + with self.assertRaisesRegex(RuntimeError, "alpha failed"): + start_selected_contexts( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: False, + resolved_run_id=lambda _session: "run-1", + record_project_startup=record_project_startup, + render_project_startup_warnings=lambda **_kwargs: None, + should_degrade_to_plan_agent_handoff=lambda _session, _error: False, + record_plan_agent_handoff_local_startup_failure=lambda **_kwargs: None, + spinner_factory=_SpinnerFactory(), + use_spinner_policy_fn=lambda _policy: nullcontext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False, backend="rich"), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + project_spinner_group_factory=lambda **_kwargs: SimpleNamespace(), + ) + + self.assertIn("Beta", session.services_by_project) + self.assertIn("Beta", session.requirements_by_project) + + def test_unterminated_service_error_is_carried_into_session_state(self) -> None: + route = Route(command="start", mode="main", flags={}) + session = self._session(route=route, contexts=[]) + service = ServiceRecord(name="Main Backend", type="backend", cwd="/repo", pid=63001) + failure = ServiceCleanupError("cleanup failed", {service.name: service}) + + track_unterminated_services(session, failure) + + self.assertIs(session.unterminated_services[service.name], service) + self.assertIs(session.merged_services[service.name], service) + + def test_project_startup_failure_carries_requirements_into_failure_state(self) -> None: + route = Route(command="start", mode="main", flags={}) + session = self._session(route=route, contexts=[]) + requirements = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6379}, + ) + failure = ProjectStartupFailure( + "backend failed", + project="Main", + requirements=requirements, + ) + + track_unterminated_services(session, failure) + + self.assertIs(session.requirements_by_project["Main"], requirements) + self.assertIs(session.merged_requirements["Main"], requirements) + @staticmethod def _context(name: str) -> SimpleNamespace: return SimpleNamespace(name=name, root=Path("/tmp") / name, ports={}) @@ -110,7 +215,9 @@ def test_start_selected_contexts_builds_execution_route_and_records_sequential_r session.services_by_project.__setitem__(context.name, result.services), session.started_context_names.append(context.name), ), - render_project_startup_warnings=lambda *, context, warnings, route, project_spinner_group: warnings and None, + render_project_startup_warnings=lambda *, context, warnings, route, project_spinner_group: ( + warnings and None + ), should_degrade_to_plan_agent_handoff=lambda session, error: False, record_plan_agent_handoff_local_startup_failure=lambda session, project_name, error: None, spinner_factory=_SpinnerFactory(), @@ -133,7 +240,9 @@ def test_start_selected_contexts_builds_execution_route_and_records_sequential_r self.assertIsNone(execution_route.flags["_spinner_update_project"]) self.assertTrue(execution_route.flags["debug_suppress_progress_output"]) self.assertEqual(runtime.events[0][0], "spinner.policy") - self.assertIn(("startup.execution", {"mode": "sequential", "workers": 1, "projects": ["Alpha"]}), runtime.events) + self.assertIn( + ("startup.execution", {"mode": "sequential", "workers": 1, "projects": ["Alpha"]}), runtime.events + ) def test_record_project_startup_updates_session_result_maps_and_started_names(self) -> None: route = Route(command="start", mode="trees", raw_args=["start"], passthrough_args=[], projects=[], flags={}) @@ -194,13 +303,12 @@ def test_runtime_bound_selected_context_startup_records_results_and_uses_owner_d runtime, session, resolved_run_id=lambda session: "run-1", - render_project_startup_warnings=lambda *, context, warnings, route, project_spinner_group: rendered_warnings.append( - (context.name, list(warnings)) + render_project_startup_warnings=lambda *, context, warnings, route, project_spinner_group: ( + rendered_warnings.append((context.name, list(warnings))) + ), + should_degrade_to_plan_agent_handoff=lambda session, error: ( + degradation_checks.append((session.requested_command, error)) or False ), - should_degrade_to_plan_agent_handoff=lambda session, error: degradation_checks.append( - (session.requested_command, error) - ) - or False, record_plan_agent_handoff_local_startup_failure=lambda session, project_name, error: local_failures.append( (project_name, error) ), diff --git a/tests/python/startup/test_service_attach_execution.py b/tests/python/startup/test_service_attach_execution.py index 0c61b31f4..6d5743117 100644 --- a/tests/python/startup/test_service_attach_execution.py +++ b/tests/python/startup/test_service_attach_execution.py @@ -1,12 +1,15 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Callable, Iterable +import json from pathlib import Path +import tempfile from types import SimpleNamespace from typing import cast import unittest -from envctl_engine.runtime.service_manager import ServiceStartDescriptor +from envctl_engine.runtime.service_manager import ServiceCleanupError, ServiceManager, ServiceStartDescriptor +from envctl_engine.shared.ports import PortPlanner from envctl_engine.startup.service_attach_execution import ServiceAttachRunner from envctl_engine.startup.service_execution_records import PreparedServiceLaunch from envctl_engine.state.models import PortPlan, ServiceRecord @@ -17,6 +20,204 @@ def port_plan(port: int) -> PortPlan: class ServiceAttachExecutionTests(unittest.TestCase): + def test_later_layer_failure_rolls_back_and_tracks_prior_layer_records(self) -> None: + project_root = Path("/tmp/envctl-project") + voice = SimpleNamespace( + name="voice-runtime", + start_cmd="python voice.py", + depends_on=("backend",), + start_order=25, + expect_listener=True, + critical=True, + env_suffix="VOICE_RUNTIME", + ) + worker = SimpleNamespace( + name="worker", + start_cmd="python worker.py", + depends_on=("voice-runtime",), + start_order=50, + expect_listener=False, + critical=True, + env_suffix="WORKER", + ) + attach_layers: list[tuple[str, ...]] = [] + backend_record = ServiceRecord( + name="Main Backend", + type="backend", + cwd=str(project_root / "backend"), + pid=61001, + actual_port=8000, + ) + + def generic_attach(**kwargs: object) -> dict[str, ServiceRecord]: + descriptors = tuple(cast(Iterable[ServiceStartDescriptor], kwargs["descriptors"])) + layer = tuple(descriptor.service_type for descriptor in descriptors) + attach_layers.append(layer) + if "worker" not in layer: + return {backend_record.name: backend_record} + raise RuntimeError("worker startup failed") + + runtime = SimpleNamespace( + services=SimpleNamespace(start_services_with_attach=generic_attach), + _terminate_started_services=lambda _services: (_ for _ in ()).throw(OSError("layer terminator crashed")), + config=SimpleNamespace( + app_service_by_name=lambda name: {"voice-runtime": voice, "worker": worker}.get(name) + ), + _conflict_remaining={}, + _emit=lambda *_args, **_kwargs: None, + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(), + port_allocator=SimpleNamespace( + reserve_next=lambda port, owner: port, + session_id="attach-session", + ), + project_name="Main", + project_root=project_root, + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=project_root / "backend", + frontend_cwd=project_root / "frontend", + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=project_root / "backend", + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ), + "worker": PreparedServiceLaunch( + service_name="worker", + cwd=project_root / "worker", + log_path="/logs/worker.txt", + requested_port=0, + env={}, + command_source="configured", + listener_expected=False, + ), + "voice-runtime": PreparedServiceLaunch( + service_name="voice-runtime", + cwd=project_root / "voice-runtime", + log_path="/logs/voice.txt", + requested_port=8010, + env={}, + command_source="configured", + ), + }, + selected_service_types={"backend", "voice-runtime", "worker"}, + additional_services=(voice, worker), + backend_listener_expected=True, + rebound_delta=0, + ) + + with self.assertRaises(ServiceCleanupError) as raised: + runner.start(attach_parallel=False, on_service_retry=lambda *_args: None) + + self.assertEqual(attach_layers, [("backend", "voice-runtime"), ("worker",)]) + self.assertIs(raised.exception.unterminated_services["Main Backend"], backend_record) + self.assertEqual(backend_record.port_lock_session, "attach-session") + self.assertIn("cleanup error: layer terminator crashed", str(raised.exception)) + + def test_process_launch_without_pid_never_invents_envctl_or_adjacent_pid(self) -> None: + project_root = Path("/tmp/envctl-project") + runtime = SimpleNamespace( + _conflict_remaining={}, + _emit=lambda *_args, **_kwargs: None, + _service_start_command_resolved=lambda **_kwargs: (["backend"], "configured"), + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(start_background=lambda *_args, **_kwargs: object()), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="Main", + project_root=project_root, + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=project_root / "backend", + frontend_cwd=project_root / "frontend", + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=project_root / "backend", + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + success, error, pid = runner.start_backend(8000) + + self.assertTrue(success) + self.assertIsNone(error) + self.assertIsNone(pid) + + def test_post_spawn_telemetry_failure_still_returns_real_pid_to_manager(self) -> None: + project_root = Path("/tmp/envctl-project") + + def emit(event: str, **payload: object) -> None: + if event == "service.start" or payload.get("phase") == "process_launch": + raise OSError("event sink failed") + + runtime = SimpleNamespace( + _conflict_remaining={}, + _emit=emit, + _service_start_command_resolved=lambda **_kwargs: (["backend"], "configured"), + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(start_background=lambda *_args, **_kwargs: SimpleNamespace(pid=62001)), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="Main", + project_root=project_root, + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=project_root / "backend", + frontend_cwd=project_root / "frontend", + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=project_root / "backend", + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + success, error, pid = runner.start_backend(8000) + + self.assertTrue(success) + self.assertIsNone(error) + self.assertEqual(pid, 62001) + def test_runner_builds_layered_descriptors_and_preserves_additional_urls(self) -> None: project_root = Path("/tmp/envctl-project") events: list[tuple[str, dict[str, object]]] = [] @@ -92,14 +293,16 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: ) process_runtime = SimpleNamespace( start_background=lambda command, cwd, env, stdout_path, stderr_path: ( - process_commands.append((list(command), str(cwd), int(env["PORT"]))) - or SimpleNamespace(pid=4321) + process_commands.append((list(command), str(cwd), int(env["PORT"]))) or SimpleNamespace(pid=4321) ) ) runner = ServiceAttachRunner( runtime=runtime, process_runtime=process_runtime, - port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + port_allocator=SimpleNamespace( + reserve_next=lambda port, owner: port, + session_id="layered-session", + ), project_name="Main", project_root=project_root, backend_plan=port_plan(8000), @@ -152,12 +355,26 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: self.assertEqual(layers, [("backend", "voice-runtime"), ("worker",)]) self.assertEqual(records["Main Backend"].actual_port, 8101) + self.assertEqual( + {record.port_lock_session for record in records.values()}, + {"layered-session"}, + ) self.assertEqual(records["Main Voice Runtime"].actual_port, 8022) self.assertIsNone(records["Main Worker"].actual_port) self.assertEqual(records["Main Voice Runtime"].public_url, "http://localhost:8010") self.assertIn((["backend", "8000"], str(project_root / "backend"), 8000), process_commands) - self.assertIn((["voice.py", "8010", "voice-runtime", str(project_root / "voice-runtime")], str(project_root / "voice-runtime"), 8010), process_commands) - self.assertIn(("service.bind.skipped", {"project": "Main", "service": "worker", "reason": "listener_not_expected"}), events) + self.assertIn( + ( + ["voice.py", "8010", "voice-runtime", str(project_root / "voice-runtime")], + str(project_root / "voice-runtime"), + 8010, + ), + process_commands, + ) + self.assertIn( + ("service.bind.skipped", {"project": "Main", "service": "worker", "reason": "listener_not_expected"}), + events, + ) self.assertTrue( any( event == "service.attach.phase" @@ -182,7 +399,10 @@ def test_additional_listener_truth_failure_emits_failure_event(self) -> None: services=SimpleNamespace(), _conflict_remaining={}, _emit=lambda event, **payload: events.append((event, payload)), - _service_start_command_resolved=lambda service_name, project_root, port: ([service_name, str(port)], "configured"), + _service_start_command_resolved=lambda service_name, project_root, port: ( + [service_name, str(port)], + "configured", + ), _detect_service_actual_port=lambda **kwargs: None, _listener_truth_enforced=lambda: True, _service_listener_failure_detail=lambda **kwargs: "process exited", @@ -242,12 +462,17 @@ def test_core_only_runner_uses_legacy_service_manager_when_generic_attach_is_mis attach_kwargs: dict[str, object] = {} runtime = SimpleNamespace( services=SimpleNamespace( - start_project_with_attach=lambda **kwargs: attach_kwargs.update(kwargs) - or {"Main Backend": ServiceRecord(name="Main Backend", type="backend", cwd="/repo")} + start_project_with_attach=lambda **kwargs: ( + attach_kwargs.update(kwargs) + or {"Main Backend": ServiceRecord(name="Main Backend", type="backend", cwd="/repo")} + ) ), _conflict_remaining={}, _emit=lambda event, **payload: None, - _service_start_command_resolved=lambda service_name, project_root, port: ([service_name, str(port)], "configured"), + _service_start_command_resolved=lambda service_name, project_root, port: ( + [service_name, str(port)], + "configured", + ), _detect_service_actual_port=lambda **kwargs: kwargs["requested_port"], _listener_truth_enforced=lambda: True, _service_listener_failure_detail=lambda **kwargs: "", @@ -255,7 +480,10 @@ def test_core_only_runner_uses_legacy_service_manager_when_generic_attach_is_mis runner = ServiceAttachRunner( runtime=runtime, process_runtime=SimpleNamespace(start=lambda *args, **kwargs: SimpleNamespace(pid=123)), - port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + port_allocator=SimpleNamespace( + reserve_next=lambda port, owner: port, + session_id="legacy-attach-session", + ), project_name="Main", project_root=Path("/repo"), backend_plan=port_plan(8000), @@ -286,9 +514,135 @@ def test_core_only_runner_uses_legacy_service_manager_when_generic_attach_is_mis records = runner.start(attach_parallel=False, on_service_retry=lambda *args: None) self.assertIn("Main Backend", records) + self.assertEqual( + records["Main Backend"].port_lock_session, + "legacy-attach-session", + ) self.assertTrue(callable(attach_kwargs["start_backend"])) self.assertFalse(attach_kwargs["parallel_start"]) + def test_multiple_service_retries_release_every_failed_port_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="startup-session", + availability_checker=lambda _port: True, + ) + planner.reserve_next(8000, owner="Main:backend") + + def generic_attach(**kwargs: object) -> dict[str, ServiceRecord]: + reserve_next = cast(Callable[[int], int], kwargs["reserve_next"]) + on_retry = cast(Callable[[str, int, int, int, str | None], None], kwargs["on_retry"]) + retry_one = reserve_next(8001) + on_retry("backend", 8000, retry_one, 1, "address already in use") + retry_two = reserve_next(retry_one + 1) + on_retry("backend", retry_one, retry_two, 2, "address already in use") + return {} + + runtime = SimpleNamespace( + services=SimpleNamespace(start_services_with_attach=generic_attach), + _conflict_remaining={}, + _emit=lambda *_args, **_kwargs: None, + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(), + port_allocator=planner, + project_name="Main", + project_root=Path("/repo"), + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=Path("/repo/backend"), + frontend_cwd=Path("/repo/frontend"), + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=Path("/repo/backend"), + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + runner.start(attach_parallel=False, on_service_retry=lambda *_args: None) + + locks = list(Path(tmpdir).glob("*.lock")) + self.assertEqual([path.name for path in locks], ["8002.lock"]) + payload = json.loads(locks[0].read_text(encoding="utf-8")) + self.assertEqual(payload["owner"], "Main:services") + + def test_failed_rebound_frontend_releases_attempt_locks_without_bulk_session_release(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="startup-session", + availability_checker=lambda _port: True, + ) + planner.reserve_next(5173, owner="Main:frontend") + live_pids = {123} + process_runtime = SimpleNamespace( + start_background=lambda *_args, **_kwargs: SimpleNamespace(pid=123), + is_pid_running=lambda pid: pid in live_pids, + terminate_process_group=lambda pid, **_kwargs: live_pids.discard(pid) or True, + ) + runtime = SimpleNamespace( + services=ServiceManager(), + _conflict_remaining={}, + _emit=lambda *_args, **_kwargs: None, + _service_start_command_resolved=lambda **_kwargs: (["frontend"], "configured"), + _detect_service_actual_port=lambda **_kwargs: None, + _listener_truth_enforced=lambda: True, + _service_listener_failure_detail=lambda **_kwargs: "listener missing", + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=process_runtime, + port_allocator=planner, + project_name="Main", + project_root=Path("/repo"), + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=Path("/repo/backend"), + frontend_cwd=Path("/repo/frontend"), + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "frontend": PreparedServiceLaunch( + service_name="frontend", + cwd=Path("/repo/frontend"), + log_path="/logs/frontend.txt", + requested_port=5173, + env={}, + command_source="configured", + ) + }, + selected_service_types={"frontend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=2, + ) + + with self.assertRaisesRegex(RuntimeError, "listener not detected"): + runner.start(attach_parallel=False, on_service_retry=lambda *_args: None) + + # Failure finalization keeps the whole session when managed + # requirements remain, so per-attempt cleanup must be sufficient. + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/startup/test_service_manager.py b/tests/python/startup/test_service_manager.py index 06186f094..eaafd7182 100644 --- a/tests/python/startup/test_service_manager.py +++ b/tests/python/startup/test_service_manager.py @@ -3,10 +3,17 @@ import time import unittest from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" -from envctl_engine.runtime.service_manager import ServiceManager, ServiceStartDescriptor +from envctl_engine.runtime.service_manager import ( + ServiceCleanupError, + ServiceManager, + ServiceStartDescriptor, + ServiceStartError, +) class ServiceManagerTests(unittest.TestCase): @@ -149,11 +156,14 @@ def fail_backend(_pid: int | None, _requested: int) -> int: ) def test_listener_detection_failure_is_not_retried_as_port_conflict(self) -> None: - manager = ServiceManager() attempts: list[int] = [] terminated: list[int] = [] - - manager.terminate_process_group = lambda pid, **kwargs: terminated.append(pid) or True # type: ignore[attr-defined] + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda pid, **_kwargs: terminated.append(pid) or True, + ) + ) def start_backend(port: int) -> tuple[bool, str | None, int | None]: attempts.append(port) @@ -226,7 +236,6 @@ def test_non_listener_service_can_start_without_persisted_ports(self) -> None: self.assertIsNone(service.requested_port) self.assertIsNone(service.actual_port) - def test_start_services_with_attach_returns_degraded_record_for_noncritical_failure(self) -> None: manager = ServiceManager() descriptors = ( @@ -330,9 +339,13 @@ def test_start_services_with_attach_starts_arbitrary_services_and_non_listeners( self.assertIsNone(records["Main Worker"].actual_port) def test_start_services_with_attach_cleans_partial_records_on_failure(self) -> None: - manager = ServiceManager() terminated: list[int] = [] - manager.terminate_process_group = lambda pid, **kwargs: terminated.append(pid) or True # type: ignore[attr-defined] + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda pid, **_kwargs: terminated.append(pid) or True, + ) + ) descriptors = ( ServiceStartDescriptor( @@ -361,6 +374,186 @@ def test_start_services_with_attach_cleans_partial_records_on_failure(self) -> N self.assertEqual(terminated, [41001]) + def test_listener_failure_fails_closed_when_group_and_raw_cleanup_cannot_prove_exit(self) -> None: + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: False, + terminate=lambda *_args, **_kwargs: False, + ) + manager = ServiceManager(process_runner=runner) + + with ( + patch("envctl_engine.shared.process_termination._send_signal", return_value=True), + patch("envctl_engine.shared.process_termination.wait_for_pid_exit", side_effect=[False, False]), + self.assertRaises(ServiceCleanupError) as raised, + ): + manager.start_service_with_retry( + project="Main", + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=lambda _port: (True, None, 51001), + reserve_next=lambda port: port, + detect_actual=lambda *_args: (_ for _ in ()).throw(RuntimeError("backend listener not detected")), + ) + + record = raised.exception.unterminated_services["Main Backend"] + self.assertEqual(record.pid, 51001) + self.assertEqual(record.status, "termination_failed") + + def test_non_runtime_detection_exception_still_terminates_started_pid(self) -> None: + live_pids = {51501} + terminated: list[int] = [] + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + terminate_process_group=lambda pid, **_kwargs: (terminated.append(pid), live_pids.discard(pid), True)[ + -1 + ], + ) + ) + + with self.assertRaisesRegex(ServiceStartError, "listener probe crashed"): + manager.start_service_with_retry( + project="Main", + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=lambda _port: (True, None, 51501), + reserve_next=lambda port: port, + detect_actual=lambda *_args: (_ for _ in ()).throw(OSError("listener probe crashed")), + ) + + self.assertEqual(terminated, [51501]) + + def test_failed_launcher_pid_is_terminated_before_bind_retry(self) -> None: + attempts: list[int] = [] + terminated: list[int] = [] + live_pids = {52001} + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + terminate_process_group=lambda pid, **_kwargs: (terminated.append(pid), live_pids.discard(pid), True)[ + -1 + ], + ) + ) + + def start(port: int) -> tuple[bool, str | None, int | None]: + attempts.append(port) + if len(attempts) == 1: + return False, "address already in use", 52001 + return True, None, 52002 + + record = manager.start_service_with_retry( + project="Main", + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=start, + reserve_next=lambda port: port, + detect_actual=lambda _pid, requested: requested, + ) + + self.assertEqual(attempts, [8000, 8001]) + self.assertEqual(terminated, [52001]) + self.assertEqual(record.pid, 52002) + + def test_parallel_rollback_reports_every_service_whose_exit_is_unconfirmed(self) -> None: + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: False, + terminate=lambda *_args, **_kwargs: False, + ) + ) + descriptors = ( + ServiceStartDescriptor( + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=lambda _port: (True, None, 53001), + detect_actual=lambda _pid, requested: requested, + ), + ServiceStartDescriptor( + service_type="frontend", + cwd="/repo/frontend", + requested_port=3000, + start=lambda _port: (False, "boot failed", None), + max_retries=0, + ), + ) + + with ( + patch("envctl_engine.shared.process_termination._send_signal", return_value=True), + patch("envctl_engine.shared.process_termination.wait_for_pid_exit", side_effect=[False, False]), + self.assertRaises(ServiceCleanupError) as raised, + ): + manager.start_services_with_attach( + project="Main", + descriptors=descriptors, + reserve_next=lambda port: port, + parallel_start=True, + ) + + self.assertEqual(set(raised.exception.unterminated_services), {"Main Backend"}) + + def test_noncritical_listener_cleanup_failure_is_never_downgraded_to_pidless_record(self) -> None: + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: False, + terminate=lambda *_args, **_kwargs: False, + ) + ) + descriptor = ServiceStartDescriptor( + service_type="worker", + cwd="/repo/worker", + requested_port=0, + start=lambda _port: (True, None, 54001), + detect_actual=lambda *_args: (_ for _ in ()).throw(RuntimeError("worker attach failed")), + listener_expected=False, + critical=False, + ) + + with ( + patch("envctl_engine.shared.process_termination._send_signal", return_value=True), + patch("envctl_engine.shared.process_termination.wait_for_pid_exit", side_effect=[False, False]), + self.assertRaises(ServiceCleanupError) as raised, + ): + manager.start_services_with_attach( + project="Main", + descriptors=(descriptor,), + reserve_next=lambda port: port, + ) + + record = raised.exception.unterminated_services["Main Worker"] + self.assertEqual(record.pid, 54001) + self.assertEqual(record.status, "termination_failed") + + def test_success_without_valid_pid_is_tracked_as_unconfirmed_launch_even_for_non_listener(self) -> None: + manager = ServiceManager() + descriptor = ServiceStartDescriptor( + service_type="worker", + cwd="/repo/worker", + requested_port=0, + start=lambda _port: (True, None, None), + listener_expected=False, + critical=False, + ) + + with self.assertRaises(ServiceCleanupError) as raised: + manager.start_services_with_attach( + project="Main", + descriptors=(descriptor,), + reserve_next=lambda port: port, + ) + + record = raised.exception.unterminated_services["Main Worker"] + self.assertIsNone(record.pid) + self.assertEqual(record.status, "termination_failed") + self.assertFalse(record.listener_expected) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/startup/test_startup_dashboard_stopped_restore.py b/tests/python/startup/test_startup_dashboard_stopped_restore.py index af1dea6ad..ad6401b81 100644 --- a/tests/python/startup/test_startup_dashboard_stopped_restore.py +++ b/tests/python/startup/test_startup_dashboard_stopped_restore.py @@ -77,7 +77,9 @@ def test_startup_restores_dashboard_stopped_services_instead_of_auto_resuming_pa ) captured_state: dict[str, RunState] = {} - engine._try_load_existing_state = lambda mode=None, strict_mode_match=False: previous_state # type: ignore[method-assign] + engine._try_load_existing_state = ( # type: ignore[method-assign] + lambda mode=None, strict_mode_match=False, project_names=None: previous_state + ) engine._discover_projects = lambda mode: [context] # type: ignore[method-assign] engine._reserve_project_ports = lambda _context, route=None: None # type: ignore[method-assign] engine._resume = lambda route: self.fail("startup should start stopped services, not auto-resume") # type: ignore[method-assign] diff --git a/tests/python/startup/test_startup_finalization.py b/tests/python/startup/test_startup_finalization.py index 72cc316b0..db90245f4 100644 --- a/tests/python/startup/test_startup_finalization.py +++ b/tests/python/startup/test_startup_finalization.py @@ -34,7 +34,7 @@ from envctl_engine.planning.plan_agent.models import CreatedPlanWorktree, PlanAgentLaunchOutcome, PlanAgentLaunchResult from envctl_engine.startup.finalization_failure import StartupFailureFinalizer from envctl_engine.startup.session import LocalStartupFailure, StartupSession -from envctl_engine.state.models import RunState +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord def _session(*, contexts: list[object], contexts_to_start: list[object] | None = None) -> StartupSession: @@ -72,6 +72,233 @@ def _runtime_stub(**overrides: object) -> SimpleNamespace: class StartupFinalizationTests(unittest.TestCase): + def test_failure_finalization_persists_service_when_exit_remains_unconfirmed(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=64001, + actual_port=8000, + ) + session.unterminated_services[service.name] = service + writes: list[RunState] = [] + released = {"value": False} + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("event sink failed")), + _terminate_started_services=lambda _services: {service.name}, + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + result = finalize_failed_startup( + runtime=runtime, + session=session, + error="listener truth failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace( + release_session=lambda: released.__setitem__("value", True) + ), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual(result, 1) + self.assertEqual(writes[0].metadata["termination_failed_services"], [service.name]) + self.assertIs(writes[0].services[service.name], service) + self.assertEqual(service.status, "termination_failed") + self.assertFalse(released["value"]) + + def test_failure_finalization_preserves_port_session_for_started_requirements(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + requirements = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6379}, + ) + session.requirements_by_project["Main"] = requirements + writes: list[RunState] = [] + released = {"value": False} + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: set(), + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="backend launch failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace( + release_session=lambda: released.__setitem__("value", True) + ), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertIs(writes[0].requirements["Main"], requirements) + self.assertFalse(released["value"]) + + def test_successful_rollback_is_removed_from_failure_state_when_requirements_remain(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=64002, + requested_port=8000, + actual_port=8000, + ) + session.started_context_names.append("Main") + session.services_by_project["Main"] = {service.name: service} + session.requirements_by_project["Main"] = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6379}, + ) + writes: list[RunState] = [] + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: set(), + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="frontend launch failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual(writes[0].services, {}) + self.assertEqual(session.services_by_project, {}) + self.assertEqual(session.unterminated_services, {}) + + def test_missing_termination_result_retains_service_and_port_session(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=64003, + requested_port=8000, + actual_port=8000, + ) + session.started_context_names.append("Main") + session.services_by_project["Main"] = {service.name: service} + writes: list[RunState] = [] + released = {"value": False} + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: None, + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="frontend launch failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace( + release_session=lambda: released.__setitem__("value", True) + ), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertIs(writes[0].services[service.name], service) + self.assertEqual(writes[0].metadata["termination_failed_services"], [service.name]) + self.assertEqual(service.status, "termination_failed") + self.assertFalse(released["value"]) + + def test_failure_finalization_releases_port_session_for_external_only_requirements(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + requirements = RequirementsResult( + project="Main", + redis={ + "enabled": True, + "success": True, + "external": True, + "final": 6379, + }, + ) + session.requirements_by_project["Main"] = requirements + released = {"value": False} + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: set(), + _write_artifacts=lambda *_args, **_kwargs: None, + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="backend launch failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace( + release_session=lambda: released.__setitem__("value", True) + ), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertTrue(released["value"]) + + def test_failure_finalization_releases_port_session_for_clean_internal_start_failure(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + session.requirements_by_project["Main"] = RequirementsResult( + project="Main", + redis={"enabled": True, "success": False, "final": 6379, "error": "command missing"}, + ) + released = {"value": False} + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: set(), + _write_artifacts=lambda *_args, **_kwargs: None, + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="requirements failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace( + release_session=lambda: released.__setitem__("value", True) + ), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertTrue(released["value"]) + + def test_failure_event_exception_does_not_skip_cleanup_or_failure_state_write(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + service = ServiceRecord(name="Main Backend", type="backend", cwd="/repo", pid=65001) + session.started_context_names.append("Main") + session.services_by_project["Main"] = {service.name: service} + terminated: list[dict[str, ServiceRecord]] = [] + writes: list[RunState] = [] + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("event sink failed")), + _terminate_started_services=lambda services: terminated.append(dict(services)) or set(), + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + result = finalize_failed_startup( + runtime=runtime, + session=session, + error="backend failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual(result, 1) + self.assertEqual(terminated, [{service.name: service}]) + self.assertEqual(len(writes), 1) + def test_failure_finalization_uses_named_owner(self) -> None: self.assertTrue(callable(StartupFailureFinalizer.finalize)) @@ -91,7 +318,7 @@ def test_finalize_failed_startup_writes_error_artifacts_and_releases_ports(self) env={}, _emit=lambda event, **payload: events.append((event, payload)), _write_artifacts=lambda state, contexts, *, errors: writes.append((state, list(contexts), list(errors))), - _terminate_started_services=lambda services: terminated.append(dict(services)), + _terminate_started_services=lambda services: terminated.append(dict(services)) or set(), ) result = finalize_failed_startup( @@ -101,8 +328,9 @@ def test_finalize_failed_startup_writes_error_artifacts_and_releases_ports(self) ensure_run_id=lambda session: None, port_allocator=lambda runtime: SimpleNamespace(release_session=lambda: released.__setitem__("value", True)), emit_phase=lambda session, phase, started_at, **extra: events.append((f"phase:{phase}", dict(extra))), - render_final_failure_status=lambda runtime, session, error, *, interactive_tty: rendered_statuses.append(error) - or error, + render_final_failure_status=lambda runtime, session, error, *, interactive_tty: ( + rendered_statuses.append(error) or error + ), ) self.assertEqual(result, 1) @@ -170,6 +398,39 @@ def test_finalize_failed_startup_preserves_existing_startup_failure_prefix_and_s events, ) + def test_finalize_failed_startup_does_not_shadow_existing_state_after_failed_replacement(self) -> None: + session = _session(contexts=[SimpleNamespace(name="feature-a")]) + session.preserve_existing_state_on_failure = True + events: list[tuple[str, dict[str, object]]] = [] + runtime = _runtime_stub( + _emit=lambda event, **payload: events.append((event, payload)), + _write_artifacts=lambda *_args, **_kwargs: self.fail("existing state must remain authoritative"), + _terminate_started_services=lambda _services: None, + ) + + result = finalize_failed_startup( + runtime=runtime, + session=session, + error="existing service did not terminate", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda _session, phase, _started_at, **extra: events.append((f"phase:{phase}", extra)), + render_final_failure_status=lambda _runtime, _session, error, *, interactive_tty: error, + ) + + self.assertEqual(result, 1) + self.assertIn( + ( + "state.failure_write.skipped", + { + "reason": "existing_runtime_state_remains_authoritative", + "run_id": "run-finalization", + }, + ), + events, + ) + self.assertIn(("phase:artifacts_write", {"status": "skipped"}), events) + def test_finalize_successful_startup_writes_artifacts_summary_snapshot_and_returns_zero(self) -> None: context = SimpleNamespace(name="Main") session = _session(contexts=[context]) @@ -275,7 +536,10 @@ def test_runtime_bound_success_finalization_writes_artifacts_and_snapshot(self) self.assertEqual(writes[0][1], [context]) self.assertEqual(summaries[0][1], [context]) self.assertIn(("validate", {"phase": "success_finalization"}), events) - self.assertIn("before_dashboard_entry", [payload.get("checkpoint") for event, payload in events if event == "ui.plan_handoff.snapshot"]) + self.assertIn( + "before_dashboard_entry", + [payload.get("checkpoint") for event, payload in events if event == "ui.plan_handoff.snapshot"], + ) def test_finalize_plan_agent_degraded_handoff_writes_artifacts_and_attach_result(self) -> None: session = _session(contexts=[]) @@ -319,9 +583,7 @@ def test_finalize_plan_agent_degraded_handoff_artifacts_returns_run_state(self) result = finalize_plan_agent_degraded_handoff_artifacts( runtime=SimpleNamespace( - _write_artifacts=lambda state, contexts, *, errors: writes.append( - (state, list(contexts), list(errors)) - ) + _write_artifacts=lambda state, contexts, *, errors: writes.append((state, list(contexts), list(errors))) ), session=session, ensure_run_id=lambda session: events.append(("ensure", {})), @@ -583,7 +845,9 @@ def test_print_headless_plan_session_summary_uses_attach_target_override_without print_headless_plan_session_summary( session, attach_target=attach_target, - validate_plan_agent_handoff=lambda session, *, phase: self.fail("override attach target should skip validation"), + validate_plan_agent_handoff=lambda session, *, phase: self.fail( + "override attach target should skip validation" + ), print_fn=lines.append, ) @@ -771,7 +1035,13 @@ def test_restart_port_rebound_summary_lines_deduplicate_port_changes(self) -> No startup_event_index=1, ) events = [ - {"event": "port.rebound", "project": "ignored", "service": "backend", "restart_preferred_port": 1, "port": 2}, + { + "event": "port.rebound", + "project": "ignored", + "service": "backend", + "restart_preferred_port": 1, + "port": 2, + }, { "event": "port.rebound", "project": "feature-a-1", diff --git a/tests/python/startup/test_startup_fingerprint_replacement.py b/tests/python/startup/test_startup_fingerprint_replacement.py index d5d9f6df9..86ab97806 100644 --- a/tests/python/startup/test_startup_fingerprint_replacement.py +++ b/tests/python/startup/test_startup_fingerprint_replacement.py @@ -73,7 +73,9 @@ def test_start_fingerprint_mismatch_replaces_existing_project_services_before_ne terminate_calls: list[tuple[set[str] | None, bool]] = [] call_order: list[str] = [] - engine._try_load_existing_state = lambda mode=None, strict_mode_match=False: previous_state # type: ignore[method-assign] + engine._try_load_existing_state = ( # type: ignore[method-assign] + lambda mode=None, strict_mode_match=False, project_names=None: previous_state + ) engine._discover_projects = lambda mode: [context] # type: ignore[method-assign] engine._terminate_services_from_state = ( # type: ignore[method-assign] lambda state, selected_services, aggressive, verify_ownership: ( @@ -109,7 +111,7 @@ def test_start_fingerprint_mismatch_replaces_existing_project_services_before_ne warnings=[], ), )[1] - ) + ) code = engine.dispatch(parse_route(["--main", "--managed-deps", "--batch"], env={})) diff --git a/tests/python/startup/test_startup_orchestrator_flow_reuse.py b/tests/python/startup/test_startup_orchestrator_flow_reuse.py index 86c8e7ce6..d9c159aa3 100644 --- a/tests/python/startup/test_startup_orchestrator_flow_reuse.py +++ b/tests/python/startup/test_startup_orchestrator_flow_reuse.py @@ -1,10 +1,108 @@ # ruff: noqa: F403,F405 from __future__ import annotations +import json +import threading + +from envctl_engine.runtime.lifecycle_operation_lease import ( + release_lifecycle_operation, + run_exclusive_lifecycle_operation, +) +from envctl_engine.state.run_index import StateSelector + from tests.python.startup.startup_orchestrator_flow_test_support import * class StartupOrchestratorFlowReuseTests(StartupOrchestratorFlowTestCase): + def test_start_heals_multiple_historical_active_owners_for_main(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repo(root) + runtime = root / "runtime" + historical_candidates = [] + + for _index in range(2): + engine = self._engine(repo, runtime) + with ( + patch.object(engine, "_should_enter_post_start_interactive", return_value=False), + redirect_stdout(StringIO()), + ): + code = engine.dispatch( + parse_route(["start", "--batch", "--no-infra"], env={"ENVCTL_DEFAULT_MODE": "main"}) + ) + self.assertEqual(code, 0) + historical_candidates.append( + engine.state_repository.run_index.candidates(StateSelector("main", ("main",)))[0] + ) + + engine.state_repository.run_index.replace_all(historical_candidates) + for offset, candidate in enumerate(historical_candidates): + stale_lock = engine.port_planner.lock_dir / f"{44000 + offset}.lock" + stale_lock.write_text( + json.dumps( + { + "owner": "Main:db", + "session": f"historical-{candidate.run_id}", + "pid": 99999999, + "created_at": 0, + } + ), + encoding="utf-8", + ) + third_engine = self._engine(repo, runtime) + with ( + patch.object(third_engine, "_should_enter_post_start_interactive", return_value=False), + redirect_stdout(StringIO()), + ): + third_code = third_engine.dispatch( + parse_route(["start", "--batch", "--no-infra"], env={"ENVCTL_DEFAULT_MODE": "main"}) + ) + + self.assertEqual(third_code, 0) + index_payload = json.loads((third_engine.runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(len(index_payload["entries"]), 1) + self.assertEqual( + set(index_payload["retired_run_ids"]), + {candidate.run_id for candidate in historical_candidates}, + ) + self.assertEqual(list(third_engine.port_planner.lock_dir.glob("*.lock")), []) + + def test_repeated_no_infra_start_supersedes_old_authority_without_port_locks(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repo(root) + runtime = root / "runtime" + first_engine = self._engine(repo, runtime) + + with ( + patch.object(first_engine, "_should_enter_post_start_interactive", return_value=False), + redirect_stdout(StringIO()), + ): + first_code = first_engine.dispatch( + parse_route(["start", "--batch", "--no-infra"], env={"ENVCTL_DEFAULT_MODE": "main"}) + ) + + self.assertEqual(first_code, 0) + self.assertEqual(list(first_engine.port_planner.lock_dir.glob("*.lock")), []) + first_index = json.loads((first_engine.runtime_root / "run_index.json").read_text(encoding="utf-8")) + first_run_id = first_index["entries"][0]["run_id"] + + second_engine = self._engine(repo, runtime) + with ( + patch.object(second_engine, "_should_enter_post_start_interactive", return_value=False), + redirect_stdout(StringIO()), + ): + second_code = second_engine.dispatch( + parse_route(["start", "--batch", "--no-infra"], env={"ENVCTL_DEFAULT_MODE": "main"}) + ) + + self.assertEqual(second_code, 0) + self.assertEqual(list(second_engine.port_planner.lock_dir.glob("*.lock")), []) + second_index = json.loads((second_engine.runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(len(second_index["entries"]), 1) + self.assertNotEqual(second_index["entries"][0]["run_id"], first_run_id) + self.assertIn(first_run_id, second_index["retired_run_ids"]) + def test_disabled_startup_reopens_existing_dashboard_run_when_identity_matches(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -33,7 +131,11 @@ def test_disabled_startup_reopens_existing_dashboard_run_when_identity_matches(s with ( patch.object(engine, "_discover_projects", return_value=[context]), patch.object(engine, "_try_load_existing_state", return_value=existing_state), - patch.object(engine, "_write_artifacts", side_effect=AssertionError("fresh dashboard state should not be written")), + patch.object( + engine, + "_write_artifacts", + side_effect=AssertionError("fresh dashboard state should not be written"), + ), patch.object(engine, "_should_enter_post_start_interactive", return_value=False), patch.object( engine.state_repository, @@ -146,17 +248,27 @@ def _record_launch(_runtime: object, *, route: object, created_worktrees: tuple[ patch.object( engine.planning_worktree_orchestrator, "last_plan_selection_result", - return_value=PlanSelectionResult(raw_projects=[], selected_contexts=[context], created_worktrees=()), + return_value=PlanSelectionResult( + raw_projects=[], selected_contexts=[context], created_worktrees=() + ), ), patch("envctl_engine.startup.lifecycle.launch_plan_agent_terminals", side_effect=_record_launch), patch("envctl_engine.planning.plan_agent.omx_transport._tmux_session_exists", return_value=True), - patch("envctl_engine.planning.plan_agent.omx_transport._tmux_display_message_succeeds", return_value=(True, "%42")), + patch( + "envctl_engine.planning.plan_agent.omx_transport._tmux_display_message_succeeds", + return_value=(True, "%42"), + ), patch.object(engine, "_write_artifacts"), patch.object(engine, "_should_enter_post_start_interactive", return_value=False), ): out = StringIO() with redirect_stdout(out): - code = engine.dispatch(parse_route(["--plan", "feature-a", "--omx", "--codex", "--headless"], env={"ENVCTL_DEFAULT_MODE": "trees"})) + code = engine.dispatch( + parse_route( + ["--plan", "feature-a", "--omx", "--codex", "--headless"], + env={"ENVCTL_DEFAULT_MODE": "trees"}, + ) + ) self.assertEqual(code, 0) self.assertEqual(captured_created_worktrees, [["feature-a-1"]]) @@ -166,7 +278,10 @@ def _record_launch(_runtime: object, *, route: object, created_worktrees: tuple[ rendered, ) self.assertIn("attach: tmux attach -t omx-feature-session", rendered) - self.assertIn("new session: ENVCTL_USE_REPO_WRAPPER=1 /tmp/repo/bin/envctl --plan feature-a --omx --codex --new-session --headless", rendered) + self.assertIn( + "new session: ENVCTL_USE_REPO_WRAPPER=1 /tmp/repo/bin/envctl --plan feature-a --omx --codex --new-session --headless", + rendered, + ) self.assertIn("kill: tmux kill-session -t omx-feature-session", rendered) def test_explicit_cmux_plan_launch_reuses_selected_existing_worktree(self) -> None: @@ -207,13 +322,17 @@ def _record_launch(_runtime: object, *, route: object, created_worktrees: tuple[ patch.object( engine.planning_worktree_orchestrator, "last_plan_selection_result", - return_value=PlanSelectionResult(raw_projects=[], selected_contexts=[context], created_worktrees=()), + return_value=PlanSelectionResult( + raw_projects=[], selected_contexts=[context], created_worktrees=() + ), ), patch("envctl_engine.startup.lifecycle.launch_plan_agent_terminals", side_effect=_record_launch), patch.object(engine, "_write_artifacts"), patch.object(engine, "_should_enter_post_start_interactive", return_value=False), ): - code = engine.dispatch(parse_route(["--plan", "feature-a", "--cmux", "--headless"], env={"ENVCTL_DEFAULT_MODE": "trees"})) + code = engine.dispatch( + parse_route(["--plan", "feature-a", "--cmux", "--headless"], env={"ENVCTL_DEFAULT_MODE": "trees"}) + ) self.assertEqual(code, 0) self.assertEqual(captured_created_worktrees, [["feature-a-1"]]) @@ -252,7 +371,9 @@ def test_resume_dashboard_exact_headless_plan_prints_attach_command(self) -> Non patch.object( engine.planning_worktree_orchestrator, "last_plan_selection_result", - return_value=PlanSelectionResult(raw_projects=[], selected_contexts=[context], created_worktrees=()), + return_value=PlanSelectionResult( + raw_projects=[], selected_contexts=[context], created_worktrees=() + ), ), patch( "envctl_engine.startup.lifecycle.launch_plan_agent_terminals", @@ -273,7 +394,10 @@ def test_resume_dashboard_exact_headless_plan_prints_attach_command(self) -> Non out = StringIO() with redirect_stdout(out): code = engine.dispatch( - parse_route(["--plan", "feature-a", "--tmux", "--opencode", "--headless"], env={"ENVCTL_DEFAULT_MODE": "trees"}) + parse_route( + ["--plan", "feature-a", "--tmux", "--opencode", "--headless"], + env={"ENVCTL_DEFAULT_MODE": "trees"}, + ) ) self.assertEqual(code, 0) @@ -335,6 +459,73 @@ def test_resume_reuse_failure_falls_back_to_fresh_run_id_before_failure_write(se self.assertEqual(written_state.run_id, "run-fresh-after-resume-failure") self.assertNotEqual(written_state.run_id, existing_state.run_id) + def test_resume_failure_after_interactive_release_is_terminal_instead_of_racing_fresh_start(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repo(root) + runtime = root / "runtime" + engine = self._engine(repo, runtime) + context = self._main_context(repo) + metadata = startup_support.build_startup_identity_metadata( + engine, + runtime_mode="main", + project_contexts=[context], + ) + existing_state = RunState( + run_id="run-existing", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=str(repo / "backend"), + pid=1111, + requested_port=8000, + actual_port=8000, + status="running", + ) + }, + metadata=metadata, + ) + competing_results: list[int] = [] + + def released_resume_failure(_route: object) -> int: + self.assertTrue(release_lifecycle_operation(engine)) + competitor = threading.Thread( + target=lambda: competing_results.append( + run_exclusive_lifecycle_operation( + engine, + command="stop", + operation=lambda: 0, + ) + ) + ) + competitor.start() + competitor.join(timeout=2) + self.assertFalse(competitor.is_alive()) + return 1 + + with ( + patch.object(engine, "_discover_projects", return_value=[context]), + patch.object(engine, "_try_load_existing_state", return_value=existing_state), + patch.object(engine, "_resume", side_effect=released_resume_failure), + patch.object( + engine, + "_start_project_context", + side_effect=AssertionError("fresh fallback must not mutate after releasing its lease"), + ) as start_project, + patch.object( + engine, + "_write_artifacts", + side_effect=AssertionError("terminal resume failure must not write fresh state"), + ), + ): + code = engine.dispatch(parse_route(["start", "--batch"], env={"ENVCTL_DEFAULT_MODE": "main"})) + + self.assertEqual(code, 1) + self.assertEqual(competing_results, [0]) + start_project.assert_not_called() + def test_reuse_expand_failure_writes_failed_state_to_fresh_run_id(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -481,4 +672,5 @@ def _start_context(context: object, **_kwargs: object) -> ProjectStartupResult: orphan_mock.assert_not_called() written_state = cast(RunState, captured["state"]) self.assertEqual(set(written_state.services), {"feature-a-1 Backend", "feature-b-1 Backend"}) + self.assertEqual(written_state.metadata["state_source_run_ids"], ["run-existing-expand"]) self.assertEqual(captured["errors"], []) diff --git a/tests/python/startup/test_startup_selection_support.py b/tests/python/startup/test_startup_selection_support.py index 2c2a40cce..d99dcfac1 100644 --- a/tests/python/startup/test_startup_selection_support.py +++ b/tests/python/startup/test_startup_selection_support.py @@ -7,6 +7,7 @@ from envctl_engine.runtime.command_router import Route, parse_route from envctl_engine.startup.startup_selection_support import ( + _restart_service_types_for_project, restart_include_requirements, select_start_tree_projects, trees_start_selection_required, @@ -68,7 +69,9 @@ def test_trees_start_selection_required_for_default_or_explicit_trees_start(self self.assertTrue(trees_start_selection_required(route=self._trees_route(raw_args=[]), runtime_mode="trees")) self.assertFalse( trees_start_selection_required( - route=Route(command="plan", mode="trees", raw_args=["--plan"], passthrough_args=[], projects=[], flags={}), + route=Route( + command="plan", mode="trees", raw_args=["--plan"], passthrough_args=[], projects=[], flags={} + ), runtime_mode="trees", ) ) @@ -152,6 +155,22 @@ def test_runtime_scope_flags_control_restart_requirements(self) -> None: self.assertTrue(restart_include_requirements(parse_route(["restart"], env={}))) self.assertFalse(restart_include_requirements(parse_route(["restart", "--service", "Main Frontend"], env={}))) + def test_no_infra_selects_no_additional_application_services(self) -> None: + route = Route( + command="start", + mode="main", + flags={"launch_backend": False, "launch_frontend": False, "launch_dependencies": False}, + ) + + self.assertEqual( + _restart_service_types_for_project( + route=route, + project_name="Main", + default_service_types={"backend", "frontend", "worker"}, + ), + set(), + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/startup/test_support_module_decoupling.py b/tests/python/startup/test_support_module_decoupling.py index f58d1e238..10ac54eb0 100644 --- a/tests/python/startup/test_support_module_decoupling.py +++ b/tests/python/startup/test_support_module_decoupling.py @@ -26,6 +26,7 @@ _maybe_prewarm_docker, _requirements_failure_message, _shared_main_requirements, + ProjectStartupFailure, start_requirements_for_project, start_project_services, start_project_context, @@ -67,6 +68,132 @@ def release(self, port: int) -> None: class StartupSupportModuleDecouplingTests(unittest.TestCase): + def test_post_start_truth_failure_propagates_unterminated_service_records(self) -> None: + requirements = RequirementsResult(project="Main", health="healthy") + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=62001, + actual_port=8000, + ) + runtime = SimpleNamespace( + config=SimpleNamespace( + supabase_auth_users=(), + supabase_auth_user_errors=(), + supabase_auth_users_strict=True, + ), + runtime_root=Path("/tmp/runtime"), + _reserve_project_ports=lambda _context, route=None: None, + _requirements_ready=lambda _value: True, + _consume_project_startup_warnings=lambda _project: [], + _emit=lambda *_args, **_kwargs: None, + _assert_project_services_post_start_truth=lambda **_kwargs: (_ for _ in ()).throw( + RuntimeError("backend truth failed") + ), + _terminate_started_services=lambda _services: None, + ) + orchestrator = SimpleNamespace( + runtime=runtime, + _report_progress=lambda *_args, **_kwargs: None, + start_requirements_for_project=lambda *_args, **_kwargs: requirements, + start_project_services=lambda *_args, **_kwargs: {"Main Backend": service}, + ) + + with self.assertRaises(ProjectStartupFailure) as raised: + start_project_context( + orchestrator, + context=SimpleNamespace(name="Main", root=Path("/repo"), ports={}), + mode="main", + route=parse_route(["start"], env={}), + run_id="run-1", + ) + + self.assertIs(raised.exception.unterminated_services["Main Backend"], service) + self.assertIs(raised.exception.requirements, requirements) + + def test_service_attach_failure_preserves_already_started_requirements(self) -> None: + requirements = RequirementsResult( + project="Main", + health="healthy", + redis={"enabled": True, "success": True, "final": 6379}, + ) + runtime = SimpleNamespace( + config=SimpleNamespace( + supabase_auth_users=(), + supabase_auth_user_errors=(), + supabase_auth_users_strict=True, + ), + runtime_root=Path("/tmp/runtime"), + _reserve_project_ports=lambda _context, route=None: None, + _requirements_ready=lambda _value: True, + _emit=lambda *_args, **_kwargs: None, + ) + orchestrator = SimpleNamespace( + runtime=runtime, + _report_progress=lambda *_args, **_kwargs: None, + start_requirements_for_project=lambda *_args, **_kwargs: requirements, + start_project_services=lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("backend launch failed") + ), + ) + + with self.assertRaises(ProjectStartupFailure) as raised: + start_project_context( + orchestrator, + context=SimpleNamespace(name="Main", root=Path("/repo"), ports={}), + mode="main", + route=parse_route(["start"], env={}), + run_id="run-1", + ) + + self.assertEqual(raised.exception.project, "Main") + self.assertIs(raised.exception.requirements, requirements) + + def test_cleanup_exception_still_transports_all_started_resources(self) -> None: + requirements = RequirementsResult(project="Main", health="healthy") + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=62002, + actual_port=8000, + ) + runtime = SimpleNamespace( + config=SimpleNamespace( + supabase_auth_users=(), + supabase_auth_user_errors=(), + supabase_auth_users_strict=True, + ), + runtime_root=Path("/tmp/runtime"), + _reserve_project_ports=lambda _context, route=None: None, + _requirements_ready=lambda _value: True, + _consume_project_startup_warnings=lambda _project: [], + _emit=lambda *_args, **_kwargs: None, + _assert_project_services_post_start_truth=lambda **_kwargs: (_ for _ in ()).throw( + RuntimeError("truth failed") + ), + _terminate_started_services=lambda _services: (_ for _ in ()).throw(OSError("terminator crashed")), + ) + orchestrator = SimpleNamespace( + runtime=runtime, + _report_progress=lambda *_args, **_kwargs: None, + start_requirements_for_project=lambda *_args, **_kwargs: requirements, + start_project_services=lambda *_args, **_kwargs: {service.name: service}, + ) + + with self.assertRaisesRegex(ProjectStartupFailure, "cleanup error: terminator crashed") as raised: + start_project_context( + orchestrator, + context=SimpleNamespace(name="Main", root=Path("/repo"), ports={}), + mode="main", + route=parse_route(["start"], env={}), + run_id="run-1", + ) + + self.assertIs(raised.exception.requirements, requirements) + self.assertIs(raised.exception.unterminated_services[service.name], service) + def test_startup_execution_support_reexports_new_owner_modules(self) -> None: self.assertIs(start_requirements_for_project, requirements_start_impl) self.assertIs(start_project_services, service_start_impl) @@ -325,7 +452,9 @@ def reconcile(project, req, *, project_root=None): # noqa: ANN001, ANN202 self.assertEqual(project_root, Path("/tmp/repo")) for dependency_id in ("redis", "supabase", "n8n"): component = req.component(dependency_id) - component["runtime_status"] = "healthy" if component.get("final") in {6557, 5610, 5856} else "unreachable" + component["runtime_status"] = ( + "healthy" if component.get("final") in {6557, 5610, 5856} else "unreachable" + ) return [ {"project": project, "component": dependency_id, "status": "unreachable"} for dependency_id in ("redis", "supabase", "n8n") @@ -352,7 +481,7 @@ def reconcile(project, req, *, project_root=None): # noqa: ANN001, ANN202 _reserve_project_ports=lambda context: reserved.append(context.name), _emit=lambda event, **payload: events.append((event, payload)), ) - ) + ), ) with mock.patch( @@ -366,7 +495,9 @@ def reconcile(project, req, *, project_root=None): # noqa: ANN001, ANN202 self.assertEqual(reused.component("n8n")["final"], 5856) self.assertEqual(reserved, ["Main"]) self.assertEqual(start_requirements.call_count, 1) - self.assertIn(("requirements.shared.reuse_stale", {"project": "Main", "source_run_id": "run-stale-main"}), events) + self.assertIn( + ("requirements.shared.reuse_stale", {"project": "Main", "source_run_id": "run-stale-main"}), events + ) def test_start_project_context_rejects_invalid_supabase_auth_user_config_before_services(self) -> None: order: list[str] = [] @@ -403,7 +534,7 @@ def test_start_project_context_rejects_invalid_supabase_auth_user_config_before_ start_requirements_for_project=lambda context, mode, route: requirements, ) - with self.assertRaisesRegex(RuntimeError, "Invalid Supabase Auth user config"): + with self.assertRaisesRegex(ProjectStartupFailure, "Invalid Supabase Auth user config") as raised: start_project_context( orchestrator, context=SimpleNamespace(name="Main", root=Path("/tmp/repo"), ports={}), @@ -413,6 +544,7 @@ def test_start_project_context_rejects_invalid_supabase_auth_user_config_before_ ) self.assertNotIn("services", order) + self.assertIs(raised.exception.requirements, requirements) def test_start_project_context_syncs_supabase_auth_users_before_services(self) -> None: order: list[str] = [] @@ -444,8 +576,10 @@ def project_env(**kwargs): # noqa: ANN003, ANN201 _record_project_startup_warning=lambda project, warning: None, _consume_project_startup_warnings=lambda project: [], _emit=lambda event, **payload: ( - order.append(event), events.append((event, payload)) - ) if event.startswith("supabase.auth_users") else None, + (order.append(event), events.append((event, payload))) + if event.startswith("supabase.auth_users") + else None + ), _assert_project_services_post_start_truth=lambda **kwargs: None, _terminate_started_services=lambda services: None, ) @@ -685,7 +819,9 @@ def start_project_services(_context, *, requirements, run_id, route): # noqa: A _terminate_service_record=lambda service, **kwargs: terminated_services.append(service.name) or True, _service_port=lambda service: service.actual_port, _release_requirement_ports=lambda req: self.fail("healthy reused requirements should not be released"), - _reserve_project_ports=lambda ctx: self.fail("partial service restore should reserve only selected services"), + _reserve_project_ports=lambda ctx: self.fail( + "partial service restore should reserve only selected services" + ), _start_requirements_for_project=lambda *args, **kwargs: self.fail("healthy requirements should be reused"), _start_project_services=start_project_services, ) @@ -884,7 +1020,9 @@ def start_project_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: _project_service_env=lambda context, requirements, route=None: {}, _resolve_backend_env_file=lambda context, backend_cwd: (None, False), _resolve_frontend_env_file=lambda context, frontend_cwd: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service: True, process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: None), _prepare_backend_runtime=prepare_backend_runtime, @@ -963,7 +1101,9 @@ def test_start_project_services_skips_default_backend_frontend_when_no_local_sys _project_service_env=lambda context, requirements, route=None, service_name=None: {}, _resolve_backend_env_file=lambda context, backend_cwd: (None, False), _resolve_frontend_env_file=lambda context, frontend_cwd: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service: True, process_runner=SimpleNamespace( start_background=lambda *args, **kwargs: background_calls.append((args, kwargs)) @@ -1039,7 +1179,9 @@ def test_start_project_services_keeps_missing_command_failure_for_explicit_enabl _project_service_env=lambda context, requirements, route=None, service_name=None: {}, _resolve_backend_env_file=lambda context, backend_cwd: (None, False), _resolve_frontend_env_file=lambda context, frontend_cwd: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service: True, process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: None), _prepare_backend_runtime=lambda **kwargs: None, @@ -1101,7 +1243,9 @@ def test_start_project_services_keeps_missing_command_failure_for_explicit_addit _project_service_env=lambda context, requirements, route=None, service_name=None: {}, _resolve_backend_env_file=lambda context, backend_cwd: (None, False), _resolve_frontend_env_file=lambda context, frontend_cwd: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service: True, process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: None), _prepare_backend_runtime=lambda **kwargs: None, @@ -1182,12 +1326,16 @@ def test_start_project_services_keeps_autodetectable_default_services_startable( _project_service_env=lambda context, requirements, route=None, service_name=None: {}, _resolve_backend_env_file=lambda context, backend_cwd: (None, False), _resolve_frontend_env_file=lambda context, frontend_cwd: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service: True, process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: None), _prepare_backend_runtime=lambda **kwargs: None, _prepare_frontend_runtime=lambda **kwargs: None, - _service_command_source=lambda service_name, **kwargs: source_calls.append(service_name) or "autodetected", + _service_command_source=lambda service_name, **kwargs: ( + source_calls.append(service_name) or "autodetected" + ), _service_start_command_resolved=lambda service_name, **kwargs: (["npm", "run", "dev"], "autodetected"), _command_exists=lambda command: command in {"npm", "npx"}, _detect_service_actual_port=lambda pid, requested_port, service_name, log_path: requested_port, @@ -1332,15 +1480,17 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: _resolve_frontend_env_file=lambda context, frontend_cwd: None, _prepare_backend_runtime=lambda **kwargs: None, _prepare_frontend_runtime=lambda **kwargs: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=service_enabled_for_mode, _service_command_source=lambda **kwargs: "configured", _service_start_command_resolved=lambda service_name, **kwargs: ("python -c pass", "configured"), _split_command=lambda command, **kwargs: ["python", "-c", "pass"], process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: SimpleNamespace(pid=1234)), - _detect_service_actual_port=lambda service_name, pid, requested_port, **kwargs: 8022 - if service_name == "voice-runtime" - else requested_port, + _detect_service_actual_port=lambda service_name, pid, requested_port, **kwargs: ( + 8022 if service_name == "voice-runtime" else requested_port + ), _listener_truth_enforced=lambda: True, _service_listener_failure_detail=lambda **kwargs: "", _conflict_remaining={}, @@ -1348,7 +1498,9 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: ) orchestrator = SimpleNamespace( runtime=runtime, - _restart_service_types_for_project=lambda route, project_name, default_service_types: set(default_service_types), + _restart_service_types_for_project=lambda route, project_name, default_service_types: set( + default_service_types + ), _suppress_timing_output=lambda route: True, ) context = SimpleNamespace( @@ -1444,13 +1596,18 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: _resolve_frontend_env_file=lambda context, frontend_cwd: None, _prepare_backend_runtime=lambda **kwargs: None, _prepare_frontend_runtime=lambda **kwargs: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service_name: service_name == "voice-runtime", _service_command_source=lambda **kwargs: "configured", - _split_command=lambda command, **kwargs: split_calls.append({"command": command, **kwargs}) or [ - "scripts/envctl/start-voice-runtime.sh", - str(kwargs["port"]), - ], + _split_command=lambda command, **kwargs: ( + split_calls.append({"command": command, **kwargs}) + or [ + "scripts/envctl/start-voice-runtime.sh", + str(kwargs["port"]), + ] + ), process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: SimpleNamespace(pid=1234)), _detect_service_actual_port=lambda **kwargs: 8010, _listener_truth_enforced=lambda: True, @@ -1556,7 +1713,9 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: _resolve_frontend_env_file=lambda context, frontend_cwd: None, _prepare_backend_runtime=lambda **kwargs: None, _prepare_frontend_runtime=lambda **kwargs: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service_name: service_name == "voice-runtime", _service_command_source=lambda **kwargs: "configured", _split_command=lambda command, **kwargs: ["python", "-c", "pass"], diff --git a/tests/python/state/test_fingerprints.py b/tests/python/state/test_fingerprints.py new file mode 100644 index 000000000..e210b7afd --- /dev/null +++ b/tests/python/state/test_fingerprints.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from pathlib import Path + +from envctl_engine.runtime.state_fingerprint_support import ( + state_fingerprint as runtime_state_fingerprint, +) +from envctl_engine.state.fingerprints import file_fingerprint, state_fingerprint, text_fingerprint +from envctl_engine.state.models import RunState + + +def test_shared_fingerprint_helpers_agree_for_text_file_and_runtime_facade(tmp_path: Path) -> None: + payload = "deterministic payload\n" + path = tmp_path / "payload.txt" + path.write_text(payload, encoding="utf-8") + + assert file_fingerprint(path) == text_fingerprint(payload) + + state = RunState(run_id="run-1", mode="trees", metadata={"project_names": ["alpha"]}) + assert runtime_state_fingerprint(state) == state_fingerprint(state) + + +def test_state_fingerprint_changes_with_serialized_state() -> None: + first = RunState(run_id="run-1", mode="main") + second = RunState(run_id="run-1", mode="main", metadata={"changed": True}) + + assert state_fingerprint(first) != state_fingerprint(second) diff --git a/tests/python/state/test_lookup.py b/tests/python/state/test_lookup.py new file mode 100644 index 000000000..5e1fc5be9 --- /dev/null +++ b/tests/python/state/test_lookup.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import unittest + +from envctl_engine.state.lookup import call_state_loader + + +class StateLookupTests(unittest.TestCase): + def test_passes_all_supported_selection_arguments(self) -> None: + captured: dict[str, object] = {} + + def loader( + *, + mode: str | None, + strict_mode_match: bool, + project_names: tuple[str, ...] | None, + ) -> str: + captured.update( + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ) + return "selected" + + result = call_state_loader( + loader, + mode="trees", + strict_mode_match=True, + project_names=("feature-a",), + ) + + self.assertEqual(result, "selected") + self.assertEqual( + captured, + { + "mode": "trees", + "strict_mode_match": True, + "project_names": ("feature-a",), + }, + ) + + def test_omits_unsupported_compatibility_keywords_for_untargeted_lookup(self) -> None: + calls = 0 + + def legacy_loader() -> str: + nonlocal calls + calls += 1 + return "legacy" + + self.assertEqual(call_state_loader(legacy_loader, mode="main"), "legacy") + self.assertEqual(calls, 1) + + def test_targeted_lookup_fails_closed_when_loader_cannot_select_projects(self) -> None: + called = False + + def legacy_loader(*, mode: str | None = None) -> str: + nonlocal called + called = True + return str(mode) + + with self.assertRaisesRegex(TypeError, "accepts project_names"): + call_state_loader(legacy_loader, mode="trees", project_names=("feature-a",)) + self.assertFalse(called) + + def test_var_keyword_loader_receives_selection_arguments(self) -> None: + captured: dict[str, object] = {} + + def flexible_loader(**kwargs: object) -> dict[str, object]: + captured.update(kwargs) + return captured + + self.assertIs( + call_state_loader( + flexible_loader, + mode="trees", + strict_mode_match=True, + project_names=("feature-a",), + ), + captured, + ) + self.assertEqual(captured["project_names"], ("feature-a",)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/state/test_persistence.py b/tests/python/state/test_persistence.py new file mode 100644 index 000000000..2844c6bc4 --- /dev/null +++ b/tests/python/state/test_persistence.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import errno +import os +import tempfile +import threading +import unittest +from pathlib import Path +from unittest.mock import patch + +from envctl_engine.state.persistence import ( + advisory_file_lock, + atomic_write_text, + durable_mkdir, + require_path_component, + scavenge_atomic_write_temps, +) + + +class StatePersistenceTests(unittest.TestCase): + def test_failed_atomic_replaces_preserve_old_value_and_leave_no_temp_files_or_descriptors(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + target = root / "state.json" + target.write_text("old", encoding="utf-8") + descriptor_root = Path("/dev/fd") + descriptors_before = len(list(descriptor_root.iterdir())) if descriptor_root.is_dir() else None + + with patch("envctl_engine.state.persistence.os.replace", side_effect=OSError("forced failure")): + for _ in range(25): + with self.assertRaisesRegex(OSError, "forced failure"): + atomic_write_text(target, "new") + + descriptors_after = len(list(descriptor_root.iterdir())) if descriptor_root.is_dir() else None + self.assertEqual(target.read_text(encoding="utf-8"), "old") + self.assertEqual(list(root.glob(".*.tmp")), []) + if descriptors_before is not None: + self.assertEqual(descriptors_after, descriptors_before) + + def test_target_local_scavenge_removes_only_exact_regular_atomic_temps(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + target = root / "state.json" + stale = root / ".state.json.abcdefgh.tmp" + unrelated_target = root / ".other.json.abcdefgh.tmp" + malformed_token = root / ".state.json.not-atomic.tmp" + generic_tmp = root / "notes.tmp" + external = root / "external" + symlink_temp = root / ".state.json.ijklmnop.tmp" + for path in (stale, unrelated_target, malformed_token, generic_tmp): + path.write_text(path.name, encoding="utf-8") + external.write_text("sentinel", encoding="utf-8") + symlink_temp.symlink_to(external) + + removed = scavenge_atomic_write_temps(target) + + self.assertEqual(removed, 1) + self.assertFalse(stale.exists()) + self.assertTrue(unrelated_target.is_file()) + self.assertTrue(malformed_token.is_file()) + self.assertTrue(generic_tmp.is_file()) + self.assertTrue(symlink_temp.is_symlink()) + self.assertEqual(external.read_text(encoding="utf-8"), "sentinel") + + def test_target_local_scavenge_rejects_symlink_parent_without_touching_target(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + external = root / "external" + external.mkdir() + stale = external / ".state.json.abcdefgh.tmp" + stale.write_text("sentinel", encoding="utf-8") + linked_parent = root / "linked-parent" + linked_parent.symlink_to(external, target_is_directory=True) + + with self.assertRaises(OSError): + scavenge_atomic_write_temps(linked_parent / "state.json") + + self.assertEqual(stale.read_text(encoding="utf-8"), "sentinel") + + def test_path_component_validation_accepts_safe_names_and_rejects_traversal(self) -> None: + self.assertEqual(require_path_component("run-2026_07.11", label="run_id"), "run-2026_07.11") + for value in ("", " ", ".", "..", "../run", "nested/run", "nested\\run", "/tmp/run", "bad\0id"): + with self.subTest(value=value): + with self.assertRaisesRegex(ValueError, "path component"): + require_path_component(value, label="run_id") + + def test_lock_descriptor_is_closed_if_stream_construction_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "state.lock" + opened_descriptors: list[int] = [] + closed_descriptors: list[int] = [] + real_open = os.open + real_close = os.close + + def tracked_open(*args: object, **kwargs: object) -> int: + descriptor = real_open(*args, **kwargs) # type: ignore[arg-type] + opened_descriptors.append(descriptor) + return descriptor + + def tracked_close(descriptor: int) -> None: + closed_descriptors.append(descriptor) + real_close(descriptor) + + with ( + patch("envctl_engine.state.persistence.os.open", side_effect=tracked_open), + patch("envctl_engine.state.persistence.os.fdopen", side_effect=OSError("fdopen failed")), + patch("envctl_engine.state.persistence.os.close", side_effect=tracked_close), + ): + with self.assertRaisesRegex(OSError, "fdopen failed"): + with advisory_file_lock(lock_path, exclusive=True): + self.fail("lock acquisition unexpectedly succeeded") + + self.assertEqual(len(opened_descriptors), 1) + self.assertEqual(closed_descriptors, opened_descriptors) + + def test_lock_rejects_symlink_without_touching_its_target(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + target = root / "external-target" + target.write_text("sentinel", encoding="utf-8") + lock_path = root / "state.lock" + lock_path.symlink_to(target) + + with self.assertRaises(OSError): + with advisory_file_lock(lock_path, exclusive=True): + self.fail("symlink lock unexpectedly succeeded") + + self.assertEqual(target.read_text(encoding="utf-8"), "sentinel") + + def test_lock_timeout_does_not_leak_or_enter_critical_section(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "state.lock" + holder_entered = threading.Event() + release_holder = threading.Event() + + def hold_lock() -> None: + with advisory_file_lock(lock_path, exclusive=True): + holder_entered.set() + release_holder.wait(timeout=2) + + holder = threading.Thread(target=hold_lock) + holder.start() + self.assertTrue(holder_entered.wait(timeout=1)) + try: + with self.assertRaisesRegex(TimeoutError, "Timed out acquiring lock"): + with advisory_file_lock(lock_path, exclusive=True, timeout_seconds=0.02): + self.fail("contended lock unexpectedly succeeded") + finally: + release_holder.set() + holder.join(timeout=1) + self.assertFalse(holder.is_alive()) + + def test_unsupported_directory_fsync_does_not_undo_committed_replace(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + target = Path(tmpdir) / "state.json" + with patch( + "envctl_engine.state.persistence.os.fsync", + side_effect=[None, OSError(errno.EINVAL, "directory fsync unsupported")], + ): + atomic_write_text(target, "committed") + + self.assertEqual(target.read_text(encoding="utf-8"), "committed") + + def test_durable_mkdir_fsyncs_each_new_directory_and_its_parent(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + target = root / "runs" / "run-a" / "revisions" + synced: list[Path] = [] + + with patch( + "envctl_engine.state.persistence.fsync_directory", + side_effect=lambda path: synced.append(path), + ): + durable_mkdir(target) + + self.assertTrue(target.is_dir()) + for directory in (root / "runs", root / "runs" / "run-a", target): + self.assertIn(directory, synced) + self.assertIn(directory.parent, synced) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/state/test_project_runtime_resolution.py b/tests/python/state/test_project_runtime_resolution.py index 0c6cc5435..1502a9e09 100644 --- a/tests/python/state/test_project_runtime_resolution.py +++ b/tests/python/state/test_project_runtime_resolution.py @@ -8,6 +8,7 @@ active_project_names, cwd_runtime_warnings, dependency_mode_summary, + filter_project_scoped_metadata, infer_cwd_project, requested_project_not_running_payload, resolve_requested_project_state, @@ -55,6 +56,80 @@ def test_exact_requested_project_match_filters_state_to_canonical_project(self) self.assertEqual(set(resolution.state.requirements), {"Alpha"}) self.assertEqual(resolution.state.metadata["project_roots"], {"Alpha": "/tmp/a"}) + def test_project_selection_filters_every_project_scoped_metadata_surface(self) -> None: + state = RunState( + run_id="run-metadata", + mode="trees", + services={ + "Alpha Backend": ServiceRecord(name="Alpha Backend", type="backend", cwd="/tmp/a", project="Alpha"), + "Beta Voice": ServiceRecord(name="Beta Voice", type="voice", cwd="/tmp/b", project="Beta"), + }, + metadata={ + "project_roots": {"Alpha": "/tmp/a", "Beta": "/tmp/b"}, + "project_pr_links": {"Alpha": "pr-a", "Beta": "pr-b"}, + "project_test_summaries": {"Alpha": {"failed": 0}, "Beta": {"failed": 1}}, + "project_test_results_root": "/tmp/results", + "project_test_results_updated_at": "now", + "project_action_reports": {"Alpha": {"test": {}}, "Beta": {"test": {}}}, + "dashboard_project_configured_services": { + "Alpha": ["backend"], + "Beta": ["voice"], + }, + "dashboard_configured_service_types": ["backend", "voice"], + "dashboard_stopped_services": [ + {"name": "Alpha Backend", "project": "Alpha", "type": "backend"}, + {"name": "Beta Voice", "project": "Beta", "type": "voice"}, + {"project": "Alpha", "type": "frontend"}, + ], + "startup_identity": { + "mode": "trees", + "projects": [ + {"name": "Alpha", "root": "/tmp/a"}, + {"name": "Beta", "root": "/tmp/b"}, + ], + "fingerprint": "stale", + }, + "unrelated": "preserved", + }, + ) + + resolution = resolve_requested_project_state(state, ["Alpha"], command="health") + + self.assertTrue(resolution.ok) + assert resolution.state is not None + metadata = resolution.state.metadata + for key in ( + "project_roots", + "project_pr_links", + "project_test_summaries", + "project_action_reports", + "dashboard_project_configured_services", + ): + self.assertEqual(set(metadata[key]), {"Alpha"}) + self.assertEqual( + metadata["dashboard_stopped_services"], + [{"name": "Alpha Backend", "project": "Alpha", "type": "backend"}], + ) + self.assertEqual(metadata["dashboard_configured_service_types"], ["backend"]) + self.assertEqual(metadata["startup_identity"]["projects"], [{"name": "Alpha", "root": "/tmp/a"}]) + self.assertNotEqual(metadata["startup_identity"]["fingerprint"], "stale") + self.assertNotIn("project_test_results_root", metadata) + self.assertNotIn("project_test_results_updated_at", metadata) + self.assertEqual(metadata["unrelated"], "preserved") + + def test_filter_removes_test_companion_fields_when_no_selected_summary_remains(self) -> None: + metadata = filter_project_scoped_metadata( + { + "project_test_summaries": {"Beta": {"failed": 1}}, + "project_test_results_root": "/tmp/results", + "project_test_results_updated_at": "now", + }, + {"Alpha"}, + ) + + self.assertNotIn("project_test_summaries", metadata) + self.assertNotIn("project_test_results_root", metadata) + self.assertNotIn("project_test_results_updated_at", metadata) def test_case_mismatched_project_selector_fails_closed(self) -> None: state = RunState( @@ -148,14 +223,18 @@ def test_dependency_mode_summary_prefers_explicit_metadata_and_derives_legacy_sh mode="trees", metadata={"dependency_mode": "isolated", "shared_dependencies": False}, ) - self.assertEqual(dependency_mode_summary(explicit), {"dependency_mode": "isolated", "shared_dependencies": False}) + self.assertEqual( + dependency_mode_summary(explicit), {"dependency_mode": "isolated", "shared_dependencies": False} + ) legacy_shared = RunState( run_id="run-legacy", mode="trees", metadata={"dashboard_dependency_scope": "shared"}, ) - self.assertEqual(dependency_mode_summary(legacy_shared), {"dependency_mode": "shared", "shared_dependencies": True}) + self.assertEqual( + dependency_mode_summary(legacy_shared), {"dependency_mode": "shared", "shared_dependencies": True} + ) unknown = RunState(run_id="run-unknown", mode="trees") self.assertEqual(dependency_mode_summary(unknown), {"dependency_mode": "unknown", "shared_dependencies": None}) diff --git a/tests/python/state/test_run_index.py b/tests/python/state/test_run_index.py new file mode 100644 index 000000000..db8a61471 --- /dev/null +++ b/tests/python/state/test_run_index.py @@ -0,0 +1,437 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import patch + +from envctl_engine.state.persistence import atomic_write_text +from envctl_engine.state.run_index import RunIndex, RunIndexCandidate, StateSelector + + +class RunIndexTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.runtime_dir = Path(self.temporary_directory.name) / "runtime" + self.runtime_root = self.runtime_dir / "repo-scope" + self.index = RunIndex( + runtime_root=self.runtime_root, + runtime_dir=self.runtime_dir, + runtime_scope_id="repo-scope", + ) + self.revision_number = 0 + + def _state_path(self, run_id: str) -> Path: + self.revision_number += 1 + path = self.runtime_root / "runs" / run_id / "revisions" / f"revision-{self.revision_number}" / "run_state.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{}\n", encoding="utf-8") + return path + + def _record(self, run_id: str, *, mode: str = "trees", projects: list[str]) -> Path: + path = self._state_path(run_id) + self.index.record( + state_path=path, + run_id=run_id, + mode=mode, + project_names=projects, + ) + return path.resolve() + + def test_new_records_replace_overlapping_project_owners_without_crossing_modes(self) -> None: + _ = self._record("exact-old", projects=["Feature-A"]) + containing = self._record("containing", projects=["feature-a", "feature-b"]) + partial = self._record("partial-overlap", projects=["feature-c", "feature-d"]) + exact_new = self._record("exact-new", projects=["FEATURE-A"]) + _ = self._record("disjoint", projects=["feature-z"]) + wrong_mode = self._record("wrong-mode", mode="main", projects=["feature-a"]) + + self.assertEqual( + self.index.candidate_paths(StateSelector(mode="TREES", project_names=("feature-a",))), + [exact_new], + ) + self.assertEqual( + self.index.candidate_paths(StateSelector(mode="trees", project_names=("feature-b",))), + [containing], + ) + self.assertEqual( + self.index.candidate_paths(StateSelector(mode="main", project_names=("feature-a",))), + [wrong_mode], + ) + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-a", "feature-c"))), + [exact_new, partial], + ) + + def test_record_heals_multiple_historical_owners_for_the_same_project(self) -> None: + run_a = self._state_path("run-a") + run_b = self._state_path("run-b") + self.index.replace_all( + [ + RunIndexCandidate(run_a, "run-a", "main", ("main",), 1), + RunIndexCandidate(run_b, "run-b", "main", ("main",), 2), + ] + ) + + run_c = self._record("run-c", mode="main", projects=["Main"]) + + self.assertEqual( + self.index.candidate_paths(StateSelector("main", ("main",))), + [run_c], + ) + payload = json.loads(self.index.index_path.read_text(encoding="utf-8")) + self.assertEqual(payload["retired_run_ids"], ["run-a", "run-b"]) + + def test_record_trims_mixed_historical_owner_without_losing_unrelated_project(self) -> None: + run_a = self._state_path("run-a") + run_b = self._state_path("run-b") + self.index.replace_all( + [ + RunIndexCandidate(run_a, "run-a", "trees", ("feature-x", "feature-y"), 1), + RunIndexCandidate(run_b, "run-b", "trees", ("feature-x",), 2), + ] + ) + + run_c = self._record("run-c", projects=["feature-x"]) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-x",))), + [run_c], + ) + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-y",))), + [run_a.resolve()], + ) + payload = json.loads(self.index.index_path.read_text(encoding="utf-8")) + entries = {entry["run_id"]: entry["project_names"] for entry in payload["entries"]} + self.assertEqual(entries, {"run-a": ["feature-y"], "run-c": ["feature-x"]}) + self.assertEqual(payload["retired_run_ids"], ["run-b"]) + + def test_preserves_independent_entries_during_concurrent_records(self) -> None: + run_ids = [f"run-{index:02d}" for index in range(32)] + state_paths = {run_id: self._state_path(run_id).resolve() for run_id in run_ids} + + def record(run_id: str) -> None: + self.index.record( + state_path=state_paths[run_id], + run_id=run_id, + mode="trees", + project_names=[run_id], + ) + + with ThreadPoolExecutor(max_workers=12) as executor: + list(executor.map(record, run_ids)) + + candidates = self.index.candidate_paths(StateSelector(mode="trees", project_names=())) + self.assertEqual(len(candidates), len(run_ids)) + self.assertEqual(set(candidates), set(state_paths.values())) + payload = json.loads((self.runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(len(payload["entries"]), len(run_ids)) + + def test_corrupt_index_fails_closed_and_later_record_replaces_it(self) -> None: + self.runtime_root.mkdir(parents=True, exist_ok=True) + index_path = self.runtime_root / "run_index.json" + index_path.write_text('{"entries":', encoding="utf-8") + + self.assertEqual(self.index.candidate_paths(StateSelector(mode=None, project_names=())), []) + + recovered = self._record("recovered", projects=["feature-a"]) + self.assertEqual( + self.index.candidate_paths(StateSelector(mode="trees", project_names=("feature-a",))), + [recovered], + ) + payload = json.loads(index_path.read_text(encoding="utf-8")) + self.assertEqual(payload["version"], 4) + self.assertGreaterEqual(payload["generation"], 1) + + def test_clean_registry_write_scavenges_only_its_crash_temp_targets(self) -> None: + self.runtime_root.mkdir(parents=True, exist_ok=True) + stale_primary = self.runtime_root / ".run_index.json.abcdefgh.tmp" + stale_backup = self.runtime_root / ".run_index.backup.json.ijklmnop.tmp" + unrelated = self.runtime_root / ".run_state.json.abcdefgh.tmp" + for path in (stale_primary, stale_backup, unrelated): + path.write_text("crash debris", encoding="utf-8") + + self.index.initialize_empty() + + self.assertFalse(stale_primary.exists()) + self.assertFalse(stale_backup.exists()) + self.assertTrue(unrelated.is_file()) + + def test_rejects_record_paths_outside_runtime_dir(self) -> None: + outside = Path(self.temporary_directory.name) / "runtime-sibling" / "run_state.json" + outside.parent.mkdir(parents=True, exist_ok=True) + outside.write_text("{}\n", encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "escapes runtime_root"): + self.index.record( + state_path=outside, + run_id="outside", + mode="trees", + project_names=["feature-a"], + ) + + def test_rejects_state_path_from_another_runtime_scope(self) -> None: + foreign = self.runtime_dir / "another-scope" / "runs" / "foreign" / "run_state.json" + foreign.parent.mkdir(parents=True, exist_ok=True) + foreign.write_text("{}\n", encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "escapes runtime_root"): + self.index.record( + state_path=foreign, + run_id="foreign", + mode="trees", + project_names=["feature-a"], + ) + + def test_rejects_symlinked_or_noncanonical_revision_paths(self) -> None: + real_path = self._state_path("safe") + alias_path = real_path.parent / "alias.json" + alias_path.symlink_to(real_path) + + for path in (alias_path, real_path.parent / ".." / real_path.parent.name / real_path.name): + with self.subTest(path=path): + with self.assertRaisesRegex(ValueError, "canonical"): + self.index.record( + state_path=path, + run_id="safe", + mode="trees", + project_names=["feature-a"], + ) + + def test_runtime_root_retarget_after_construction_is_rejected_without_external_writes(self) -> None: + external = Path(self.temporary_directory.name) / "external" + external.mkdir() + sentinel = external / "sentinel.txt" + sentinel.write_text("untouched", encoding="utf-8") + self.runtime_root.parent.mkdir(parents=True, exist_ok=True) + self.runtime_root.symlink_to(external, target_is_directory=True) + + with self.assertRaisesRegex(RuntimeError, "symlink"): + self.index.candidates(StateSelector(mode=None, project_names=())) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "untouched") + self.assertEqual(list(external.iterdir()), [sentinel]) + + def test_traversal_entry_fails_closed_and_wrong_scope_is_not_loaded(self) -> None: + safe = self._record("safe", projects=["feature-a"]) + index_path = self.runtime_root / "run_index.json" + payload = json.loads(index_path.read_text(encoding="utf-8")) + payload["entries"].append( + { + "state_path": str(self.runtime_root / ".." / ".." / "escaped.json"), + "run_id": "escaped", + "mode": "trees", + "project_names": ["feature-a"], + "sequence": 2, + "runtime_scope_id": "repo-scope", + } + ) + index_path.write_text(json.dumps(payload), encoding="utf-8") + self.assertEqual(self.index.candidate_paths(StateSelector("trees", ("feature-a",))), [safe]) + + payload["entries"] = [entry for entry in payload["entries"] if entry["run_id"] == "safe"] + payload["runtime_scope_id"] = "another-scope" + index_path.write_text(json.dumps(payload), encoding="utf-8") + (self.runtime_root / "run_index.backup.json").write_text(json.dumps(payload), encoding="utf-8") + self.assertEqual(self.index.candidate_paths(StateSelector("trees", ("feature-a",))), []) + self.assertTrue(safe.is_file()) + + def test_purge_removes_index_without_preventing_future_records(self) -> None: + before = self._record("before-purge", projects=["feature-a"]) + self.index.purge() + payload = json.loads((self.runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(payload["entries"], []) + self.assertEqual(payload["retired_run_ids"], ["before-purge"]) + self.assertEqual(self.index.candidate_paths(StateSelector(None, ())), []) + + with self.assertRaisesRegex(RuntimeError, "retired"): + self.index.record( + state_path=before, + run_id="before-purge", + mode="trees", + project_names=["feature-a"], + ) + + after = self._record("after-purge", projects=["feature-b"]) + self.assertEqual(self.index.candidate_paths(StateSelector("trees", ("feature-b",))), [after]) + + def test_empty_update_preserves_existing_project_ownership(self) -> None: + state_path = self._record("run-a", projects=["feature-a"]) + + self.index.record( + state_path=state_path, + run_id="run-a", + mode="trees", + project_names=[], + ) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-a",))), + [state_path], + ) + + def test_rejects_invalid_run_id_components_before_indexing(self) -> None: + for run_id in (".", "..", "../other", "nested/run", "nested\\run", "/absolute"): + with self.subTest(run_id=run_id): + with self.assertRaisesRegex(ValueError, "path component"): + self.index.record( + state_path=self.runtime_root / "runs" / "safe" / "run_state.json", + run_id=run_id, + mode="trees", + project_names=["feature-a"], + ) + + def test_remove_deactivates_only_requested_run(self) -> None: + run_a = self._record("run-a", projects=["feature-a"]) + run_b = self._record("run-b", projects=["feature-b"]) + + self.assertTrue(self.index.remove("run-a")) + self.assertFalse(self.index.remove("run-a")) + self.assertEqual(self.index.candidate_paths(StateSelector("trees", ("feature-a",))), []) + self.assertEqual(self.index.candidate_paths(StateSelector("trees", ("feature-b",))), [run_b]) + self.assertTrue(run_a.is_file()) + + def test_superseded_run_is_durably_fenced_from_reclaiming_ownership(self) -> None: + run_a = self._record("run-a", projects=["feature-a"]) + run_b = self._state_path("run-b") + self.index.record( + state_path=run_b, + run_id="run-b", + mode="trees", + project_names=["feature-a"], + supersede_run_ids=["run-a"], + ) + + with self.assertRaisesRegex(RuntimeError, "retired"): + self.index.record( + state_path=run_a, + run_id="run-a", + mode="trees", + project_names=["feature-a"], + ) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-a",))), + [run_b.resolve()], + ) + + def test_removing_untracked_source_id_still_fences_late_writer(self) -> None: + self.index.initialize_empty() + self.assertFalse(self.index.remove("late-source")) + + with self.assertRaisesRegex(RuntimeError, "retired"): + self.index.record( + state_path=self._state_path("late-source"), + run_id="late-source", + mode="trees", + project_names=["feature-a"], + ) + + def test_single_copy_failure_keeps_newest_generation_readable_and_repairable(self) -> None: + run_a = self._record("run-a", projects=["feature-a"]) + run_b = self._state_path("run-b") + + def write_backup_only(path: Path, text: str) -> None: + if path == self.index.index_path: + raise OSError("primary unavailable") + atomic_write_text(path, text) + + with patch("envctl_engine.state.run_index.atomic_write_text", side_effect=write_backup_only): + self.index.record( + state_path=run_b, + run_id="run-b", + mode="trees", + project_names=["feature-b"], + ) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ())), + [run_b.resolve(), run_a], + ) + primary_before = json.loads(self.index.index_path.read_text(encoding="utf-8")) + backup_before = json.loads(self.index.backup_path.read_text(encoding="utf-8")) + self.assertGreater(backup_before["generation"], primary_before["generation"]) + + self.index.repair_copies() + + primary_after = json.loads(self.index.index_path.read_text(encoding="utf-8")) + backup_after = json.loads(self.index.backup_path.read_text(encoding="utf-8")) + self.assertEqual(primary_after, backup_after) + self.assertEqual( + {entry["run_id"] for entry in primary_after["entries"]}, + {"run-a", "run-b"}, + ) + + def test_equal_generation_divergence_fails_closed_without_overwriting_either_copy(self) -> None: + self._record("run-a", projects=["feature-a"]) + indexed_entry = self.index.candidates(StateSelector("trees", ()))[0] + primary_before = self.index.index_path.read_text(encoding="utf-8") + backup_payload = json.loads(self.index.backup_path.read_text(encoding="utf-8")) + backup_payload["entries"][0]["project_names"] = ["feature-b"] + backup_before = json.dumps(backup_payload, indent=2, sort_keys=True) + "\n" + self.index.backup_path.write_text(backup_before, encoding="utf-8") + + with self.assertRaisesRegex(RuntimeError, "diverged at generation"): + self.index.candidate_paths(StateSelector("trees", ())) + with self.assertRaisesRegex(RuntimeError, "diverged at generation"): + self.index.repair_copies() + with self.assertRaisesRegex(RuntimeError, "diverged at generation"): + self.index.initialize_empty() + with self.assertRaisesRegex(RuntimeError, "diverged at generation"): + self.index.replace_all([indexed_entry]) + + self.assertEqual(self.index.index_path.read_text(encoding="utf-8"), primary_before) + self.assertEqual(self.index.backup_path.read_text(encoding="utf-8"), backup_before) + + def test_total_registry_write_failure_preserves_last_committed_generation(self) -> None: + run_a = self._record("run-a", projects=["feature-a"]) + run_b = self._state_path("run-b") + + with patch( + "envctl_engine.state.run_index.atomic_write_text", + side_effect=OSError("storage unavailable"), + ): + with self.assertRaisesRegex(OSError, "storage unavailable"): + self.index.record( + state_path=run_b, + run_id="run-b", + mode="trees", + project_names=["feature-b"], + ) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ())), + [run_a], + ) + + def test_visible_registry_replace_is_commit_even_if_post_replace_fsync_reports_error(self) -> None: + state_path = self._state_path("run-visible") + + def replace_then_report_error(path: Path, text: str) -> None: + atomic_write_text(path, text) + raise OSError("directory fsync result unknown") + + with patch( + "envctl_engine.state.run_index.atomic_write_text", + side_effect=replace_then_report_error, + ): + self.index.record( + state_path=state_path, + run_id="run-visible", + mode="trees", + project_names=["feature-a"], + ) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-a",))), + [state_path.resolve()], + ) + self.assertTrue(state_path.is_file()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/state/test_state_aggregation.py b/tests/python/state/test_state_aggregation.py new file mode 100644 index 000000000..2274bf75e --- /dev/null +++ b/tests/python/state/test_state_aggregation.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +from pathlib import Path + +from envctl_engine.state.models import RunState, ServiceRecord +from envctl_engine.state.run_index import RunIndexCandidate +from envctl_engine.state.state_aggregation import ( + IndexedState, + select_indexed_owners, + state_from_indexed_owners, +) + + +def _candidate( + run_id: str, + *projects: str, + sequence: int, + activation_sequence: int, +) -> RunIndexCandidate: + return RunIndexCandidate( + state_path=Path("/runtime") / run_id / "run_state.json", + run_id=run_id, + mode="trees", + project_names=tuple(project.casefold() for project in projects), + sequence=sequence, + activation_sequence=activation_sequence, + ) + + +def _service_project_name(_name: str, service: ServiceRecord) -> str: + return str(service.project or "") + + +def _project_names(state: RunState) -> list[str]: + raw = state.metadata.get("project_names") + return [str(name) for name in raw] if isinstance(raw, list) else [] + + +def _source_run_ids(state: RunState) -> list[str]: + raw = state.metadata.get("state_source_run_ids") + return [str(run_id) for run_id in raw] if isinstance(raw, list) else [] + + +def test_owner_selection_is_total_for_an_empty_inventory() -> None: + assert ( + select_indexed_owners( + [], + selected_projects=(), + service_project_name=_service_project_name, + ) + == [] + ) + + +def test_owner_selection_filters_shadowed_run_to_projects_it_still_owns() -> None: + newest = IndexedState( + candidate=_candidate("new", "alpha", sequence=2, activation_sequence=3), + state=RunState( + run_id="new", + mode="trees", + services={ + "alpha backend": ServiceRecord( + name="alpha backend", + type="backend", + cwd="/alpha", + project="alpha", + ) + }, + metadata={"project_names": ["alpha"]}, + ), + ) + previous = IndexedState( + candidate=_candidate("old", "alpha", "beta", sequence=1, activation_sequence=1), + state=RunState( + run_id="old", + mode="trees", + services={ + project: ServiceRecord( + name=project, + type="backend", + cwd=f"/{project}", + project=project, + ) + for project in ("alpha", "beta") + }, + metadata={"project_names": ["alpha", "beta"]}, + ), + ) + + selected = select_indexed_owners( + [newest, previous], + selected_projects=(), + service_project_name=_service_project_name, + ) + + assert [indexed.candidate.run_id for indexed in selected] == ["new", "old"] + assert set(selected[0].state.services) == {"alpha backend"} + assert set(selected[1].state.services) == {"beta"} + assert selected[1].state.metadata["project_names"] == ["beta"] + + +def test_merged_state_uses_activation_owner_for_global_identity() -> None: + activated = IndexedState( + candidate=_candidate("activated", "alpha", sequence=1, activation_sequence=5), + state=RunState( + run_id="activated", + mode="trees", + metadata={ + "project_names": ["alpha"], + "project_roots": {"alpha": "/alpha"}, + "global_marker": "activated", + "state_source_run_ids": ["ancestor"], + }, + ), + ) + later_sequence = IndexedState( + candidate=_candidate("stable", "beta", sequence=2, activation_sequence=2), + state=RunState( + run_id="stable", + mode="trees", + metadata={ + "project_names": ["beta"], + "project_roots": {"beta": "/beta"}, + "global_marker": "stable", + }, + ), + ) + + merged = state_from_indexed_owners( + [activated, later_sequence], + project_names_from_state=_project_names, + source_run_ids=_source_run_ids, + ) + + assert merged is not None + assert merged.run_id == "activated" + assert merged.metadata["global_marker"] == "activated" + assert merged.metadata["project_names"] == ["alpha", "beta"] + assert merged.metadata["state_source_run_ids"] == ["activated", "ancestor", "stable"] diff --git a/tests/python/state/test_state_repository_contract.py b/tests/python/state/test_state_repository_contract.py index 1fc8b92d4..01dc3ca66 100644 --- a/tests/python/state/test_state_repository_contract.py +++ b/tests/python/state/test_state_repository_contract.py @@ -4,20 +4,25 @@ import json import tempfile import unittest +from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from threading import Event from types import MappingProxyType from types import SimpleNamespace +from unittest.mock import patch REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" _models = importlib.import_module("envctl_engine.state.models") _state = importlib.import_module("envctl_engine.state") _state_repository = importlib.import_module("envctl_engine.state.repository") +_run_index = importlib.import_module("envctl_engine.state.run_index") RunState = _models.RunState ServiceRecord = _models.ServiceRecord dump_state = _state.dump_state load_state = _state.load_state RuntimeStateRepository = _state_repository.RuntimeStateRepository +StateSelector = _run_index.StateSelector class StateRepositoryContractTests(unittest.TestCase): @@ -212,10 +217,10 @@ def test_save_resume_state_updates_scoped_and_legacy_runtime_files(self) -> None self.assertTrue((legacy_root / "runtime_map.json").is_file()) main_pointer = runtime_root / ".last_state.main" self.assertTrue(main_pointer.is_file()) - self.assertEqual( - main_pointer.read_text(encoding="utf-8").strip(), - str(runtime_root / "runs" / "run-1" / "run_state.json"), - ) + pointer_target = Path(main_pointer.read_text(encoding="utf-8").strip()) + self.assertEqual(pointer_target.name, "run_state.json") + self.assertEqual(pointer_target.parent.parent.name, "revisions") + self.assertEqual(pointer_target.parent.parent.parent.name, "run-1") def test_save_selected_stop_state_updates_scoped_and_legacy_runtime_files(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -230,7 +235,19 @@ def test_save_selected_stop_state_updates_scoped_and_legacy_runtime_files(self) runtime_scope_id="repo-123", compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, ) - state = RunState(run_id="run-2", mode="trees", metadata={"repo_scope_id": "repo-123"}) + state = RunState( + run_id="run-2", + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/tmp/feature-a/backend", + project="feature-a", + ) + }, + metadata={"repo_scope_id": "repo-123"}, + ) repo.save_selected_stop_state( state=state, @@ -290,10 +307,10 @@ def test_save_resume_main_pointer_survives_following_trees_runtime_update(self) ) main_pointer = runtime_root / ".last_state.main" main_pointer_target = main_pointer.read_text(encoding="utf-8").strip() - self.assertEqual( - main_pointer_target, - str(runtime_root / "runs" / "run-main-resume" / "run_state.json"), - ) + main_target = Path(main_pointer_target) + self.assertEqual(main_target.name, "run_state.json") + self.assertEqual(main_target.parent.parent.name, "revisions") + self.assertEqual(main_target.parent.parent.parent.name, "run-main-resume") tree_state = RunState( run_id="run-tree-update", @@ -368,7 +385,9 @@ def test_save_run_main_preserves_existing_tree_pointers_for_mode_switch_resume(s ) self.assertTrue(tree_pointer.is_file()) - self.assertEqual(tree_pointer.read_text(encoding="utf-8").strip(), str(tree_state_path)) + migrated_target = Path(tree_pointer.read_text(encoding="utf-8").strip()) + self.assertEqual(load_state(str(migrated_target)).run_id, "run-tree") + self.assertIn("revisions", migrated_target.parts) def test_load_latest_ignores_invalid_scoped_pointer_and_uses_next_valid_candidate(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -472,7 +491,9 @@ def test_save_run_trees_preserves_existing_main_pointer_for_mode_switch_resume(s ) self.assertTrue(main_pointer.is_file()) - self.assertEqual(main_pointer.read_text(encoding="utf-8").strip(), str(main_state_path)) + migrated_target = Path(main_pointer.read_text(encoding="utf-8").strip()) + self.assertEqual(load_state(str(migrated_target)).run_id, "run-main") + self.assertIn("revisions", migrated_target.parts) loaded_main = repo.load_latest(mode="main", strict_mode_match=True) self.assertIsNotNone(loaded_main) @@ -495,7 +516,9 @@ def test_save_run_trees_uses_safe_pointer_filename_for_project_name(self) -> Non compat_mode=RuntimeStateRepository.SCOPED_ONLY, ) - tree_context = SimpleNamespace(name="Feature/API v2", root=Path(tmpdir) / "trees" / "Feature_API_v2", ports={}) + tree_context = SimpleNamespace( + name="Feature/API v2", root=Path(tmpdir) / "trees" / "Feature_API_v2", ports={} + ) state = RunState(run_id="run-tree-safe-name", mode="trees", metadata={"repo_scope_id": "repo-123"}) repo.save_run( state=state, @@ -506,87 +529,1778 @@ def test_save_run_trees_uses_safe_pointer_filename_for_project_name(self) -> Non runtime_map_builder=lambda _state: {"projection": {}}, ) - safe_pointer = runtime_root / ".last_state.trees.Feature_API_v2" + pointer_name = repo._tree_pointer_name("Feature/API v2") + self.assertIsNotNone(pointer_name) + safe_pointer = runtime_root / str(pointer_name) unsafe_pointer_parent = runtime_root / ".last_state.trees.Feature" self.assertTrue(safe_pointer.is_file()) + self.assertIn("Feature_API_v2-", safe_pointer.name) self.assertFalse(unsafe_pointer_parent.exists()) loaded = repo.load_latest(mode="trees", strict_mode_match=True) self.assertIsNotNone(loaded) assert loaded is not None self.assertEqual(loaded.run_id, "run-tree-safe-name") - def test_resume_orchestrator_does_not_write_legacy_state_files_directly(self) -> None: - resume_path = REPO_ROOT / "python/envctl_engine/startup/resume_orchestrator.py" - raw = resume_path.read_text(encoding="utf-8") - self.assertNotIn('runtime_legacy_root / "run_state.json"', raw) - self.assertNotIn('runtime_legacy_root / "runtime_map.json"', raw) + def test_independent_tree_runs_remain_selection_addressable(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) - def test_lifecycle_cleanup_orchestrator_does_not_write_legacy_state_files_directly(self) -> None: - cleanup_path = REPO_ROOT / "python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py" - raw = cleanup_path.read_text(encoding="utf-8") - self.assertNotIn('runtime_legacy_root / "run_state.json"', raw) - self.assertNotIn('runtime_legacy_root / "runtime_map.json"', raw) + for project_name in ("feature-zeta", "feature-alpha"): + state = RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=f"/tmp/{project_name}", + pid=123, + requested_port=8000, + actual_port=8000, + status="running", + ) + }, + metadata={"repo_scope_id": "repo-123"}, + ) + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + self.assertTrue((runtime_root / ".last_state.trees.feature-zeta").is_file()) + self.assertTrue((runtime_root / ".last_state.trees.feature-alpha").is_file()) + zeta = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-zeta"], + ) + alpha = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-alpha"], + ) - def test_engine_runtime_legacy_compat_helper_is_removed(self) -> None: - runtime_path = REPO_ROOT / "python/envctl_engine/runtime/engine_runtime.py" - raw = runtime_path.read_text(encoding="utf-8") - self.assertNotIn("def _write_legacy_runtime_compat_files(", raw) + self.assertIsNotNone(zeta) + self.assertIsNotNone(alpha) + assert zeta is not None and alpha is not None + self.assertEqual(zeta.run_id, "run-feature-zeta") + self.assertEqual(alpha.run_id, "run-feature-alpha") - def test_aggressive_purge_removes_runtime_scoped_test_results_with_run(self) -> None: + def test_selection_aware_lookup_uses_new_owner_when_superset_supersedes_exact_run(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: runtime_dir = Path(tmpdir) / "runtime" runtime_root = runtime_dir / "scope" - legacy_root = runtime_dir / "python-engine" - runtime_root.mkdir(parents=True, exist_ok=True) - legacy_root.mkdir(parents=True, exist_ok=True) repo = RuntimeStateRepository( runtime_root=runtime_root, - runtime_legacy_root=legacy_root, + runtime_legacy_root=runtime_dir / "python-engine", runtime_dir=runtime_dir, runtime_scope_id="repo-123", compat_mode=RuntimeStateRepository.SCOPED_ONLY, ) - run_state = RunState(run_id="run-1", mode="main", metadata={"repo_scope_id": "repo-123"}) + exact_context = SimpleNamespace(name="feature-a", root=Path(tmpdir) / "feature-a", ports={}) + repo.save_run( + state=RunState(run_id="run-exact", mode="trees", metadata={"repo_scope_id": "repo-123"}), + contexts=[exact_context], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + superset_contexts = [ + exact_context, + SimpleNamespace(name="feature-b", root=Path(tmpdir) / "feature-b", ports={}), + ] + repo.save_run( + state=RunState(run_id="run-superset", mode="trees", metadata={"repo_scope_id": "repo-123"}), + contexts=superset_contexts, + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + loaded = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual(loaded.run_id, "run-superset") + + def test_save_run_atomically_supersedes_every_source_run(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + contexts = [ + SimpleNamespace(name="feature-a", root=Path(tmpdir) / "feature-a", ports={}), + SimpleNamespace(name="feature-b", root=Path(tmpdir) / "feature-b", ports={}), + ] + old_state = RunState( + run_id="run-old", + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/tmp/feature-a", + project="feature-a", + ), + "feature-b Backend": ServiceRecord( + name="feature-b Backend", + type="backend", + cwd="/tmp/feature-b", + project="feature-b", + ), + }, + metadata={"repo_scope_id": "repo-123"}, + ) + repo.save_run( + state=old_state, + contexts=contexts, + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + replacement = RunState( + run_id="run-new", + mode="trees", + services=dict(old_state.services), + metadata={ + "repo_scope_id": "repo-123", + "state_source_run_ids": ["run-old"], + }, + ) + + repo.save_run( + state=replacement, + contexts=[contexts[0]], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + self.assertEqual([state.run_id for state in repo.load_all(mode="trees")], ["run-new"]) + loaded_b = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-b"], + ) + self.assertIsNotNone(loaded_b) + assert loaded_b is not None + self.assertEqual(loaded_b.run_id, "run-new") + self.assertEqual(set(loaded_b.services), {"feature-a Backend", "feature-b Backend"}) + with self.assertRaisesRegex(RuntimeError, "retired"): + repo.save_run( + state=old_state, + contexts=contexts, + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + def test_merged_state_uses_most_recently_activated_run_for_global_metadata(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + def save(project: str, run_id: str, marker: str) -> None: + repo.save_resume_state( + state=RunState( + run_id=run_id, + mode="trees", + services={ + f"{project} Backend": ServiceRecord( + name=f"{project} Backend", + type="backend", + cwd=f"/tmp/{project}", + project=project, + ) + }, + metadata={"repo_scope_id": "repo-123", "activation_marker": marker}, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + save("feature-a", "run-a", "a-original") + save("feature-b", "run-b", "b-newer-sequence") + save("feature-a", "run-a", "a-reactivated") + loaded = repo.load_latest(mode="trees", strict_mode_match=True) + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual(loaded.run_id, "run-a") + self.assertEqual(loaded.metadata["activation_marker"], "a-reactivated") + self.assertEqual(set(loaded.services), {"feature-a Backend", "feature-b Backend"}) + + def test_selection_aware_lookup_does_not_return_disjoint_latest_state(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + context = SimpleNamespace(name="feature-a", root=Path(tmpdir) / "feature-a", ports={}) + repo.save_run( + state=RunState(run_id="run-a", mode="trees", metadata={"repo_scope_id": "repo-123"}), + contexts=[context], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + loaded = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-b"], + ) + + self.assertIsNone(loaded) + + def test_multi_project_lookup_consolidates_disjoint_active_runs_without_losing_history(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + for project_name in ("feature-a", "feature-b"): + repo.save_resume_state( + state=RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=f"/tmp/{project_name}/backend", + project=project_name, + pid=100 if project_name == "feature-a" else 200, + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + merged = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a", "feature-b"], + ) + self.assertIsNotNone(merged) + assert merged is not None + self.assertEqual(set(merged.services), {"feature-a Backend", "feature-b Backend"}) + self.assertEqual(set(merged.metadata["project_names"]), {"feature-a", "feature-b"}) + repo.save_resume_state( - state=run_state, + state=merged, emit=lambda *_args, **_kwargs: None, runtime_map_builder=lambda _state: {"projection": {}}, ) - test_summary = ( - repo.test_results_dir_path("run-1", "run_20260309_100000") / "Main" / "failed_tests_summary.txt" + active = repo.load_all(mode="trees") + self.assertEqual([state.run_id for state in active], ["run-feature-b"]) + self.assertTrue((runtime_root / "runs" / "run-feature-a" / "run_state.json").is_file()) + + def test_load_all_filters_projects_subtracted_from_a_mixed_historical_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + mixed_services = { + f"{project} Backend": ServiceRecord( + name=f"{project} Backend", + type="backend", + cwd=f"/tmp/{project}/backend", + project=project, + ) + for project in ("feature-a", "feature-b") + } + repo.save_resume_state( + state=RunState(run_id="run-mixed", mode="trees", services=mixed_services), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, + ) + repo.save_resume_state( + state=RunState( + run_id="run-new-a", + mode="trees", + services={"feature-a Backend": mixed_services["feature-a Backend"]}, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, ) - test_summary.parent.mkdir(parents=True, exist_ok=True) - test_summary.write_text("No failed tests.\n", encoding="utf-8") - repo.purge(aggressive=False) - self.assertTrue(test_summary.is_file()) + active = {state.run_id: set(state.services) for state in repo.load_all(mode="trees")} - repo.purge(aggressive=True) - self.assertFalse(repo.run_dir_path("run-1").exists()) - self.assertFalse(test_summary.exists()) + self.assertEqual( + active, + { + "run-mixed": {"feature-b Backend"}, + "run-new-a": {"feature-a Backend"}, + }, + ) - def test_tree_diffs_dir_path_scopes_to_run_and_runtime_root(self) -> None: + def test_aggregate_and_subset_saves_keep_project_metadata_owned_by_live_projects(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: runtime_dir = Path(tmpdir) / "runtime" runtime_root = runtime_dir / "scope" - legacy_root = runtime_dir / "python-engine" - runtime_root.mkdir(parents=True, exist_ok=True) - legacy_root.mkdir(parents=True, exist_ok=True) repo = RuntimeStateRepository( runtime_root=runtime_root, - runtime_legacy_root=legacy_root, + runtime_legacy_root=runtime_dir / "python-engine", runtime_dir=runtime_dir, runtime_scope_id="repo-123", compat_mode=RuntimeStateRepository.SCOPED_ONLY, ) - self.assertEqual(repo.tree_diffs_dir_path(), runtime_root / "tree-diffs") + for project_name, service_type in (("feature-a", "backend"), ("feature-b", "frontend")): + repo.save_resume_state( + state=RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} {service_type}": ServiceRecord( + name=f"{project_name} {service_type}", + type=service_type, + cwd=f"/tmp/{project_name}/{service_type}", + project=project_name, + ) + }, + metadata={ + "project_roots": { + project_name: f"/tmp/{project_name}", + "stale-project": "/tmp/stale", + }, + "project_pr_links": { + project_name: f"pr-{project_name}", + "stale-project": "pr-stale", + }, + "project_test_summaries": { + project_name: {"failed": 0}, + "stale-project": {"failed": 99}, + }, + "project_test_results_root": f"/tmp/results/{project_name}", + "project_test_results_updated_at": project_name, + "project_action_reports": { + project_name: {"test": {"status": "success"}}, + "stale-project": {"test": {"status": "failed"}}, + }, + "dashboard_project_configured_services": { + project_name: [service_type], + "stale-project": ["worker"], + }, + "dashboard_configured_service_types": [service_type, "worker"], + "dashboard_stopped_services": [ + { + "name": f"{project_name} stopped", + "project": project_name, + "type": service_type, + }, + { + "name": "stale stopped", + "project": "stale-project", + "type": "worker", + }, + ], + "startup_identity": { + "mode": "trees", + "projects": [ + {"name": project_name, "root": f"/tmp/{project_name}"}, + {"name": "stale-project", "root": "/tmp/stale"}, + ], + "fingerprint": project_name, + }, + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + merged = repo.load_latest(mode="trees", strict_mode_match=True) + self.assertIsNotNone(merged) + assert merged is not None + for key in ( + "project_roots", + "project_pr_links", + "project_test_summaries", + "project_action_reports", + "dashboard_project_configured_services", + ): + self.assertEqual(set(merged.metadata[key]), {"feature-a", "feature-b"}) self.assertEqual( - repo.tree_diffs_dir_path("run-1", "review"), - runtime_root / "runs" / "run-1" / "tree-diffs" / "review", + {item["project"] for item in merged.metadata["dashboard_stopped_services"]}, + {"feature-a", "feature-b"}, + ) + self.assertEqual( + merged.metadata["dashboard_configured_service_types"], + ["backend", "frontend"], + ) + self.assertEqual( + {project["name"] for project in merged.metadata["startup_identity"]["projects"]}, + {"feature-a", "feature-b"}, ) + self.assertNotIn("project_test_results_root", merged.metadata) + self.assertNotIn("project_test_results_updated_at", merged.metadata) + + selected = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + self.assertIsNotNone(selected) + assert selected is not None + for key in ( + "project_roots", + "project_pr_links", + "project_test_summaries", + "project_action_reports", + "dashboard_project_configured_services", + ): + self.assertEqual(set(selected.metadata[key]), {"feature-a"}) + self.assertEqual( + {item["project"] for item in selected.metadata["dashboard_stopped_services"]}, + {"feature-a"}, + ) + self.assertEqual(selected.metadata["dashboard_configured_service_types"], ["backend"]) + + merged.services = { + name: service for name, service in merged.services.items() if service.project == "feature-a" + } + repo.save_resume_state( + state=merged, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + saved_subset = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + self.assertIsNotNone(saved_subset) + assert saved_subset is not None + for key in ( + "project_roots", + "project_pr_links", + "project_test_summaries", + "project_action_reports", + "dashboard_project_configured_services", + ): + self.assertEqual(set(saved_subset.metadata[key]), {"feature-a"}) + self.assertEqual( + {item["project"] for item in saved_subset.metadata["dashboard_stopped_services"]}, + {"feature-a"}, + ) + self.assertIsNone( + repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-b"], + ) + ) + + def test_deactivate_run_preserves_other_active_run_and_historical_artifacts(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + for project_name in ("feature-a", "feature-b"): + repo.save_resume_state( + state=RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=f"/tmp/{project_name}/backend", + project=project_name, + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, + ) + + self.assertTrue(repo.deactivate_run("run-feature-b")) + self.assertFalse(repo.deactivate_run("run-feature-b")) + self.assertIsNone(repo.load_latest(mode="trees", strict_mode_match=True, project_names=["feature-b"])) + remaining = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + self.assertIsNotNone(remaining) + assert remaining is not None + self.assertEqual(remaining.run_id, "run-feature-a") + self.assertEqual(load_state(str(runtime_root / "run_state.json")).run_id, "run-feature-a") + self.assertTrue((runtime_root / "runs" / "run-feature-b" / "run_state.json").is_file()) + + self.assertTrue(repo.deactivate_run("run-feature-a")) + self.assertEqual(repo.load_all(mode="trees"), []) + self.assertFalse((runtime_root / "run_state.json").exists()) + self.assertTrue((runtime_root / "runs" / "run-feature-a" / "run_state.json").is_file()) + + def test_concurrent_saves_publish_coherent_current_snapshot_and_keep_both_runs(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + first_builder_entered = Event() + release_first_builder = Event() + second_builder_entered = Event() + + def save(project_name: str) -> None: + state = RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=f"/tmp/{project_name}/backend", + project=project_name, + ) + }, + ) + + def runtime_map_builder(_state: RunState) -> dict[str, object]: + if project_name == "feature-a": + first_builder_entered.set() + self.assertTrue(release_first_builder.wait(timeout=2)) + else: + second_builder_entered.set() + return {"run_id": state.run_id} + + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=runtime_map_builder, + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + first = executor.submit(save, "feature-a") + self.assertTrue(first_builder_entered.wait(timeout=2)) + second = executor.submit(save, "feature-b") + self.assertTrue(second_builder_entered.wait(timeout=0.1)) + release_first_builder.set() + first.result(timeout=2) + second.result(timeout=2) + + current_state = load_state(str(runtime_root / "run_state.json")) + current_map = json.loads((runtime_root / "runtime_map.json").read_text(encoding="utf-8")) + self.assertEqual(current_state.run_id, current_map["run_id"]) + self.assertEqual( + {state.run_id for state in repo.load_all(mode="trees")}, {"run-feature-a", "run-feature-b"} + ) + self.assertEqual(list(runtime_root.rglob("*.tmp")), []) + + def test_purge_deactivates_index_without_deleting_history_unless_aggressive(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_resume_state( + state=RunState(run_id="run-a", mode="main"), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + historical_state = runtime_root / "runs" / "run-a" / "run_state.json" + + repo.purge(aggressive=False) + + self.assertIsNone(repo.load_latest()) + index_payload = json.loads((runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(index_payload["entries"], []) + self.assertTrue(historical_state.is_file()) + + repo.purge(aggressive=True) + self.assertFalse(historical_state.exists()) + + def test_invalid_run_ids_and_artifact_components_cannot_escape_runtime_root(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + for run_id in (".", "..", "../escaped", "nested/run", "nested\\run", "/absolute"): + with self.subTest(run_id=run_id): + runtime_root = runtime_dir / f"scope-{abs(hash(run_id))}" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id=f"scope-{abs(hash(run_id))}", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + with self.assertRaisesRegex(ValueError, "path component"): + repo.save_resume_state( + state=RunState(run_id=run_id, mode="trees"), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + self.assertFalse(runtime_root.exists()) + + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "safe-scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="safe-scope", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + for invalid_component in ("..", "../escaped", "/absolute", "nested/name"): + with self.subTest(component=invalid_component): + with self.assertRaisesRegex(ValueError, "path component"): + repo.test_results_dir_path("run-a", invalid_component) + with self.assertRaisesRegex(ValueError, "path component"): + repo.tree_diffs_dir_path("run-a", invalid_component) + + def test_missing_or_corrupt_index_is_rebuilt_from_scoped_run_history(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + for project_name in ("feature-a", "feature-b"): + state_path = runtime_root / "runs" / f"run-{project_name}" / "run_state.json" + state_path.parent.mkdir(parents=True, exist_ok=True) + dump_state( + RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=f"/tmp/{project_name}/backend", + project=project_name, + ) + }, + ), + str(state_path), + ) + + recovered_a = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + self.assertIsNotNone(recovered_a) + assert recovered_a is not None + self.assertEqual(recovered_a.run_id, "run-feature-a") + self.assertEqual({state.run_id for state in repo.load_all()}, {"run-feature-a", "run-feature-b"}) + + (runtime_root / "run_index.json").write_text('{"entries":', encoding="utf-8") + recovered_b = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-b"], + ) + self.assertIsNotNone(recovered_b) + assert recovered_b is not None + self.assertEqual(recovered_b.run_id, "run-feature-b") + rebuilt_payload = json.loads((runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(len(rebuilt_payload["entries"]), 2) + + def test_resume_orchestrator_does_not_write_legacy_state_files_directly(self) -> None: + resume_path = REPO_ROOT / "python/envctl_engine/startup/resume_orchestrator.py" + raw = resume_path.read_text(encoding="utf-8") + self.assertNotIn('runtime_legacy_root / "run_state.json"', raw) + self.assertNotIn('runtime_legacy_root / "runtime_map.json"', raw) + + def test_lifecycle_cleanup_orchestrator_does_not_write_legacy_state_files_directly(self) -> None: + cleanup_path = REPO_ROOT / "python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py" + raw = cleanup_path.read_text(encoding="utf-8") + self.assertNotIn('runtime_legacy_root / "run_state.json"', raw) + self.assertNotIn('runtime_legacy_root / "runtime_map.json"', raw) + + def test_engine_runtime_legacy_compat_helper_is_removed(self) -> None: + runtime_path = REPO_ROOT / "python/envctl_engine/runtime/engine_runtime.py" + raw = runtime_path.read_text(encoding="utf-8") + self.assertNotIn("def _write_legacy_runtime_compat_files(", raw) + + def test_aggressive_purge_removes_runtime_scoped_test_results_with_run(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + runtime_root.mkdir(parents=True, exist_ok=True) + legacy_root.mkdir(parents=True, exist_ok=True) + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + run_state = RunState(run_id="run-1", mode="main", metadata={"repo_scope_id": "repo-123"}) + repo.save_resume_state( + state=run_state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + test_summary = ( + repo.test_results_dir_path("run-1", "run_20260309_100000") / "Main" / "failed_tests_summary.txt" + ) + test_summary.parent.mkdir(parents=True, exist_ok=True) + test_summary.write_text("No failed tests.\n", encoding="utf-8") + + repo.purge(aggressive=False) + self.assertTrue(test_summary.is_file()) + + repo.purge(aggressive=True) + self.assertFalse(repo.run_dir_path("run-1").exists()) + self.assertFalse(test_summary.exists()) + + def test_tree_diffs_dir_path_scopes_to_run_and_runtime_root(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + runtime_root.mkdir(parents=True, exist_ok=True) + legacy_root.mkdir(parents=True, exist_ok=True) + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + self.assertEqual(repo.tree_diffs_dir_path(), (runtime_root / "tree-diffs").resolve()) + self.assertEqual( + repo.tree_diffs_dir_path("run-1", "review"), + (runtime_root / "runs" / "run-1" / "tree-diffs" / "review").resolve(), + ) + + def test_runtime_artifact_writer_updates_active_revision_and_derived_aliases(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + state = RunState(run_id="run-1", mode="main") + repo.save_run( + state=state, + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + accepted = repo.write_runtime_artifact( + run_id="run-1", + artifact_name="runtime_readiness_report.json", + text='{"passed": true}', + ) + + self.assertTrue(accepted) + candidate = repo.run_index.candidates(_state_repository.StateSelector(mode=None, project_names=()))[0] + artifact_paths = ( + candidate.state_path.parent / "runtime_readiness_report.json", + repo.run_dir_path("run-1") / "runtime_readiness_report.json", + runtime_root / "runtime_readiness_report.json", + legacy_root / "runtime_readiness_report.json", + ) + for path in artifact_paths: + with self.subTest(path=path): + self.assertEqual(path.read_text(encoding="utf-8"), '{"passed": true}') + + def test_retired_runtime_artifact_writer_cannot_recreate_files_after_deactivation(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + repo.save_run( + state=RunState(run_id="run-retired", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + self.assertTrue(repo.deactivate_run("run-retired")) + + accepted = repo.write_runtime_artifact( + run_id="run-retired", + artifact_name="runtime_readiness_report.json", + text='{"stale": true}', + ) + + self.assertFalse(accepted) + self.assertFalse((repo.run_dir_path("run-retired") / "runtime_readiness_report.json").exists()) + self.assertFalse((runtime_root / "runtime_readiness_report.json").exists()) + self.assertFalse((legacy_root / "runtime_readiness_report.json").exists()) + + def test_stale_runtime_artifact_writer_cannot_recreate_files_after_aggressive_purge(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + repo.save_run( + state=RunState(run_id="run-purged", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + repo.purge(aggressive=True) + + readiness_accepted = repo.write_runtime_artifact( + run_id="run-purged", + artifact_name="runtime_readiness_report.json", + text='{"stale": true}', + ) + events_accepted = repo.write_runtime_artifact( + run_id="run-purged", + artifact_name="events.jsonl", + text='{"event": "stale"}\n', + ) + + self.assertFalse(readiness_accepted) + self.assertFalse(events_accepted) + self.assertFalse(repo.run_dir_path("run-purged").exists()) + for root in (runtime_root, legacy_root): + with self.subTest(root=root): + self.assertFalse((root / "runtime_readiness_report.json").exists()) + self.assertFalse((root / "events.jsonl").exists()) + + def test_purge_waits_for_inflight_runtime_artifact_write_and_removes_its_output(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_run( + state=RunState(run_id="run-racing", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + artifact_write_started = Event() + release_artifact_write = Event() + original_write_text = repo._write_text + + def blocking_write_text(path: Path, text: str) -> None: + if ( + path.name == "runtime_readiness_report.json" + and "revisions" in path.parts + and not artifact_write_started.is_set() + ): + artifact_write_started.set() + self.assertTrue(release_artifact_write.wait(timeout=2)) + original_write_text(path, text) + + repo._write_text = blocking_write_text # type: ignore[method-assign] + with ThreadPoolExecutor(max_workers=2) as executor: + writer = executor.submit( + repo.write_runtime_artifact, + run_id="run-racing", + artifact_name="runtime_readiness_report.json", + text='{"passed": true}', + ) + self.assertTrue(artifact_write_started.wait(timeout=2)) + purge = executor.submit(repo.purge, aggressive=True) + release_artifact_write.set() + self.assertTrue(writer.result(timeout=2)) + purge.result(timeout=2) + + self.assertFalse(repo.run_dir_path("run-racing").exists()) + self.assertFalse((runtime_root / "runtime_readiness_report.json").exists()) + + def test_unbound_runtime_artifact_is_scoped_and_artifact_name_is_allowlisted(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + accepted = repo.write_runtime_artifact( + run_id=None, + artifact_name="events.jsonl", + text='{"event": "doctor"}\n', + ) + + self.assertTrue(accepted) + self.assertEqual( + (runtime_root / "events.jsonl").read_text(encoding="utf-8"), + '{"event": "doctor"}\n', + ) + self.assertFalse((legacy_root / "events.jsonl").exists()) + with self.assertRaisesRegex(ValueError, "unsupported mutable runtime artifact"): + repo.write_runtime_artifact( + run_id=None, + artifact_name="run_state.json", + text="{}", + ) + with self.assertRaisesRegex(ValueError, "path component"): + repo.write_runtime_artifact( + run_id=None, + artifact_name="../events.jsonl", + text="", + ) + + def test_unbound_runtime_artifact_never_attaches_to_an_active_run(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_run( + state=RunState(run_id="run-active", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[{"event": "run-bound"}], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + candidate = repo.run_index.candidates(_state_repository.StateSelector(mode=None, project_names=()))[0] + + accepted = repo.write_runtime_artifact( + run_id=None, + artifact_name="events.jsonl", + text='{"event": "scope-diagnostic"}\n', + ) + + self.assertTrue(accepted) + self.assertEqual( + (runtime_root / "events.jsonl").read_text(encoding="utf-8"), + json.dumps({"event": "run-bound"}, sort_keys=True) + "\n", + ) + self.assertEqual( + (runtime_root / "diagnostics" / "events.jsonl").read_text(encoding="utf-8"), + '{"event": "scope-diagnostic"}\n', + ) + expected_run_text = json.dumps({"event": "run-bound"}, sort_keys=True) + "\n" + self.assertEqual( + (repo.run_dir_path("run-active") / "events.jsonl").read_text(encoding="utf-8"), + expected_run_text, + ) + self.assertEqual( + (candidate.state_path.parent / "events.jsonl").read_text(encoding="utf-8"), + expected_run_text, + ) + + def test_retired_overlapping_run_artifact_writer_is_fenced_from_current_alias(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + for run_id in ("run-old", "run-new"): + repo.save_run( + state=RunState(run_id=run_id, mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[{"event": run_id}], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + accepted = repo.write_runtime_artifact( + run_id="run-old", + artifact_name="events.jsonl", + text='{"event": "late-old-writer"}\n', + ) + + self.assertFalse(accepted) + self.assertEqual( + (repo.run_dir_path("run-old") / "events.jsonl").read_text(encoding="utf-8"), + json.dumps({"event": "run-old"}, sort_keys=True) + "\n", + ) + expected_current = json.dumps({"event": "run-new"}, sort_keys=True) + "\n" + self.assertEqual((runtime_root / "events.jsonl").read_text(encoding="utf-8"), expected_current) + self.assertEqual((legacy_root / "events.jsonl").read_text(encoding="utf-8"), expected_current) + + def test_corrupt_authoritative_revision_does_not_reactivate_retired_project_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + for run_id, pid in (("run-old", 101), ("run-new", 202)): + repo.save_resume_state( + state=RunState( + run_id=run_id, + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/tmp/feature-a/backend", + project="feature-a", + pid=pid, + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, + ) + + candidates = repo.run_index.candidates(_state_repository.StateSelector(mode="trees", project_names=())) + new_candidate = next(candidate for candidate in candidates if candidate.run_id == "run-new") + new_candidate.state_path.write_text('{"broken":', encoding="utf-8") + + loaded = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + + self.assertIsNone(loaded) + self.assertFalse((runtime_root / "run_state.json").exists()) + self.assertFalse((runtime_root / ".last_state").exists()) + + def test_generic_lookup_skips_corrupt_newest_mode_and_returns_newest_valid_mode(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_resume_state( + state=RunState( + run_id="run-main", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + repo.save_resume_state( + state=RunState( + run_id="run-trees", + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/tmp/feature-a/backend", + project="feature-a", + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + corrupt_candidate = repo.run_index.candidates( + _state_repository.StateSelector(mode="trees", project_names=()) + )[0] + corrupt_candidate.state_path.write_text('{"broken":', encoding="utf-8") + + loaded = repo.load_latest() + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual((loaded.run_id, loaded.mode), ("run-main", "main")) + self.assertEqual(load_state(str(runtime_root / "run_state.json")).run_id, "run-main") + + def test_resuming_older_mode_reactivates_it_without_changing_project_ownership_precedence(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + main_state = RunState( + run_id="run-main", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + ) + }, + ) + tree_state = RunState( + run_id="run-tree", + mode="trees", + services={ + "FeatureA Backend": ServiceRecord( + name="FeatureA Backend", + type="backend", + cwd="/tmp/FeatureA/backend", + project="FeatureA", + ) + }, + ) + for state in (main_state, tree_state): + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + self.assertEqual(repo.load_latest().run_id, "run-tree") # type: ignore[union-attr] + + repo.save_resume_state( + state=main_state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + current = repo.load_latest() + self.assertIsNotNone(current) + assert current is not None + self.assertEqual((current.run_id, current.mode), ("run-main", "main")) + tree = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["FeatureA"], + ) + self.assertIsNotNone(tree) + assert tree is not None + self.assertEqual(tree.run_id, "run-tree") + + def test_corrupt_primary_and_backup_after_deactivation_fails_closed_without_resurrection(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_resume_state( + state=RunState( + run_id="retired-run", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/backend", + project="Main", + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + self.assertTrue(repo.deactivate_run("retired-run")) + historical_state = repo.run_dir_path("retired-run") / "run_state.json" + self.assertTrue(historical_state.is_file()) + repo.run_index.index_path.write_text('{"broken":', encoding="utf-8") + repo.run_index.backup_path.write_text('{"broken":', encoding="utf-8") + + with self.assertRaisesRegex(RuntimeError, "both the primary and backup"): + repo.load_latest() + + self.assertTrue(historical_state.is_file()) + self.assertFalse((runtime_root / "run_state.json").exists()) + + def test_failed_legacy_registry_publish_cleans_staged_revision_directories(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_state = runtime_root / "runs" / "legacy-run" / "run_state.json" + legacy_state.parent.mkdir(parents=True, exist_ok=True) + dump_state( + RunState( + run_id="legacy-run", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/backend", + project="Main", + ) + }, + ), + str(legacy_state), + ) + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + with patch.object(repo.run_index, "replace_all", side_effect=OSError("registry unavailable")): + with self.assertRaisesRegex(OSError, "registry unavailable"): + repo.load_latest() + + revisions = legacy_state.parent / "revisions" + self.assertEqual(list(revisions.iterdir()) if revisions.is_dir() else [], []) + self.assertTrue(legacy_state.is_file()) + self.assertFalse((runtime_root / "run_registry.json").exists()) + + def test_read_only_and_scoped_purges_never_mutate_legacy_runtime(self) -> None: + for compat_mode in ( + RuntimeStateRepository.COMPAT_READ_ONLY, + RuntimeStateRepository.SCOPED_ONLY, + ): + with self.subTest(compat_mode=compat_mode), tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + legacy_run = legacy_root / "runs" / "legacy-run" + legacy_run.mkdir(parents=True, exist_ok=True) + sentinels = { + legacy_root / "run_state.json": "legacy-state", + legacy_root / "events.jsonl": "legacy-event", + legacy_root / ".last_state": "legacy-pointer", + legacy_run / "run_state.json": "legacy-history", + legacy_root / ".events.jsonl.stale.tmp": "legacy-temp", + } + for path, text in sentinels.items(): + path.write_text(text, encoding="utf-8") + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=compat_mode, + ) + + repo.purge(aggressive=True) + + for path, expected in sentinels.items(): + self.assertEqual(path.read_text(encoding="utf-8"), expected) + + def test_first_operation_read_only_purge_cannot_reimport_legacy_after_index_loss(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + legacy_root.mkdir(parents=True) + legacy_state = legacy_root / "run_state.json" + dump_state(RunState(run_id="legacy-run", mode="main"), str(legacy_state)) + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_ONLY, + ) + + repo.purge(aggressive=False) + self.assertTrue(legacy_state.is_file()) + self.assertTrue((runtime_root / "run_registry.json").is_file()) + repo.run_index.index_path.unlink() + repo.run_index.backup_path.unlink() + + with self.assertRaisesRegex(RuntimeError, "both the primary and backup"): + repo.load_latest() + + self.assertTrue(legacy_state.is_file()) + self.assertFalse((runtime_root / "run_state.json").exists()) + + def test_runtime_artifact_directories_reject_symlink_redirection(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + external = Path(tmpdir) / "external" + external.mkdir() + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + runs_root = repo.run_dir_path(None) + runs_root.mkdir(parents=True) + run_link = runs_root / "run-a" + run_link.symlink_to(external, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "symlink"): + repo.run_dir_path("run-a") + run_link.unlink() + + run_dir = repo.run_dir_path("run-a") + run_dir.mkdir() + for name, resolver in ( + ("test-results", lambda: repo.test_results_dir_path("run-a")), + ("tree-diffs", lambda: repo.tree_diffs_dir_path("run-a")), + ): + with self.subTest(name=name): + link = run_dir / name + link.symlink_to(external, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "symlink"): + resolver() + link.unlink() + + root_tree_diffs = runtime_root / "tree-diffs" + root_tree_diffs.symlink_to(external, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "symlink"): + repo.tree_diffs_dir_path() + self.assertEqual(list(external.iterdir()), []) + + def test_runtime_root_retarget_is_rejected_before_lock_or_purge_writes(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + runtime_root.mkdir(parents=True) + legacy_root.mkdir() + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + external = Path(tmpdir) / "external" + external.mkdir() + sentinel = external / "run_state.json" + sentinel.write_text("outside-sentinel", encoding="utf-8") + runtime_root.rmdir() + runtime_root.symlink_to(external, target_is_directory=True) + + with self.assertRaisesRegex(RuntimeError, "symlink"): + repo.purge(aggressive=True) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "outside-sentinel") + self.assertEqual(list(external.iterdir()), [sentinel]) + + def test_legacy_root_retarget_is_rejected_before_compatibility_writes(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + runtime_root.mkdir(parents=True) + legacy_root.mkdir() + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + external = Path(tmpdir) / "external" + external.mkdir() + sentinel = external / "run_state.json" + sentinel.write_text("outside-sentinel", encoding="utf-8") + legacy_root.rmdir() + legacy_root.symlink_to(external, target_is_directory=True) + + with self.assertRaisesRegex(RuntimeError, "symlink"): + repo.save_resume_state( + state=RunState(run_id="run-a", mode="main"), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "outside-sentinel") + self.assertEqual(list(external.iterdir()), [sentinel]) + + def test_runtime_update_carries_forward_ports_errors_events_and_readiness(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + state = RunState( + run_id="run-a", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/backend", + project="Main", + pid=123, + ) + }, + ) + repo.save_run( + state=state, + contexts=[self._context(Path(tmpdir))], + errors=["diagnostic"], + events=[{"event": "initial"}], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + self.assertTrue( + repo.write_runtime_artifact( + run_id="run-a", + artifact_name="runtime_readiness_report.json", + text='{"passed": true}', + ) + ) + artifact_names = ( + "ports_manifest.json", + "error_report.json", + "events.jsonl", + "runtime_readiness_report.json", + ) + expected = {name: (runtime_root / name).read_text(encoding="utf-8") for name in artifact_names} + + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {"Main": {}}}, + ) + + candidate = repo.run_index.candidates(_state_repository.StateSelector(mode="main", project_names=()))[0] + for artifact_name, expected_text in expected.items(): + for root in (runtime_root, legacy_root, candidate.state_path.parent): + with self.subTest(artifact_name=artifact_name, root=root): + actual_text = (root / artifact_name).read_text(encoding="utf-8") + if artifact_name == "error_report.json": + payload = json.loads(actual_text) + self.assertEqual(payload["errors"], ["diagnostic"]) + self.assertEqual(payload["source_run_ids"], ["run-a"]) + else: + self.assertEqual(actual_text, expected_text) + + def test_partial_tree_stop_rebuilds_pointer_inventory_without_resurrecting_removed_project(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + state = RunState( + run_id="run-ab", + mode="trees", + services={ + project: ServiceRecord( + name=project, + type="backend", + cwd=f"/tmp/{project}/backend", + project=project, + ) + for project in ("FeatureA", "FeatureB") + }, + ) + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + pointer_a_name = repo._tree_pointer_name("FeatureA") + pointer_b_name = repo._tree_pointer_name("FeatureB") + assert pointer_a_name is not None and pointer_b_name is not None + old_target = Path((runtime_root / pointer_a_name).read_text(encoding="utf-8").strip()) + self.assertTrue(old_target.is_file()) + + state.services.pop("FeatureA") + repo.save_selected_stop_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + for root in (runtime_root, legacy_root): + self.assertFalse((root / pointer_a_name).exists()) + self.assertTrue((root / pointer_b_name).is_file()) + self.assertTrue(old_target.is_file()) + self.assertIsNone( + repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["FeatureA"], + ) + ) + remaining = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["FeatureB"], + ) + self.assertIsNotNone(remaining) + assert remaining is not None + self.assertEqual(set(remaining.services), {"FeatureB"}) + ports_payload = json.loads((runtime_root / "ports_manifest.json").read_text(encoding="utf-8")) + self.assertEqual( + [project["project"] for project in ports_payload["projects"]], + ["FeatureB"], + ) + + def test_selected_stop_can_preserve_authoritative_metadata_only_tree_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + state = RunState( + run_id="run-metadata-ab", + mode="trees", + metadata={ + "project_names": ["FeatureA", "FeatureB"], + "project_roots": { + "FeatureA": "/tmp/FeatureA", + "FeatureB": "/tmp/FeatureB", + }, + }, + ) + repo.save_selected_stop_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + authoritative_project_names=["FeatureA", "FeatureB"], + ) + + state.metadata["project_names"] = ["FeatureB"] + state.metadata["project_roots"] = {"FeatureB": "/tmp/FeatureB"} + repo.save_selected_stop_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + authoritative_project_names=["FeatureB"], + ) + + candidates = repo.run_index.candidates(StateSelector(mode="trees", project_names=())) + self.assertEqual( + [(candidate.run_id, candidate.project_names) for candidate in candidates], + [("run-metadata-ab", ("featureb",))], + ) + self.assertIsNone( + repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["FeatureA"], + ) + ) + remaining = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["FeatureB"], + ) + self.assertIsNotNone(remaining) + assert remaining is not None + self.assertEqual(remaining.services, {}) + self.assertEqual(remaining.requirements, {}) + self.assertEqual(remaining.metadata["project_names"], ["FeatureB"]) + self.assertEqual(remaining.metadata["project_roots"], {"FeatureB": "/tmp/FeatureB"}) + current = json.loads(repo.run_state_path().read_text(encoding="utf-8")) + self.assertEqual(current["metadata"]["project_names"], ["FeatureB"]) + pointer_a = repo._tree_pointer_name("FeatureA") + pointer_b = repo._tree_pointer_name("FeatureB") + assert pointer_a is not None and pointer_b is not None + self.assertFalse((runtime_root / pointer_a).exists()) + self.assertTrue((runtime_root / pointer_b).exists()) + + def test_lossy_tree_names_and_mixed_case_get_distinct_stable_pointers(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + projects = ("Feature/API v2", "Feature_API_v2") + for index, project in enumerate(projects, start=1): + repo.save_run( + state=RunState(run_id=f"run-{index}", mode="trees"), + contexts=[SimpleNamespace(name=project, root=Path(tmpdir) / f"tree-{index}", ports={})], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + pointer_names = [repo._tree_pointer_name(project) for project in projects] + self.assertTrue(all(pointer_names)) + self.assertEqual(len(set(pointer_names)), 2) + pointed_runs = { + load_state((runtime_root / str(pointer_name)).read_text(encoding="utf-8").strip()).run_id + for pointer_name in pointer_names + } + self.assertEqual(pointed_runs, {"run-1", "run-2"}) + self.assertEqual( + {pointer.name for pointer in runtime_root.glob(".last_state.trees.*")}, + set(pointer_names), + ) + + def test_post_commit_per_run_alias_failure_is_repaired_on_next_read(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + run_alias = repo.run_dir_path("run-a") / "run_state.json" + original_write_text = repo._write_text + failed = False + + def fail_first_run_alias(path: Path, text: str) -> None: + nonlocal failed + if path == run_alias and not failed: + failed = True + raise OSError("alias unavailable") + original_write_text(path, text) + + repo._write_text = fail_first_run_alias # type: ignore[method-assign] + repo.save_run( + state=RunState(run_id="run-a", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + self.assertTrue(failed) + self.assertFalse(run_alias.exists()) + repo._write_text = original_write_text # type: ignore[method-assign] + + loaded = repo.load_latest() + + self.assertIsNotNone(loaded) + self.assertTrue(run_alias.is_file()) + self.assertEqual(load_state(str(run_alias)).run_id, "run-a") if __name__ == "__main__": diff --git a/tests/python/state/test_state_repository_hardening.py b/tests/python/state/test_state_repository_hardening.py new file mode 100644 index 000000000..7a84f305a --- /dev/null +++ b/tests/python/state/test_state_repository_hardening.py @@ -0,0 +1,638 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from envctl_engine.state import dump_state, load_state +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord +from envctl_engine.state.persistence import fsync_directory +from envctl_engine.state.repository import RuntimeStateRepository +from envctl_engine.state.run_index import StateSelector + + +class StateRepositoryHardeningTests(unittest.TestCase): + @staticmethod + def _repository( + root: Path, + *, + scope: str = "scope-a", + compat_mode: str = RuntimeStateRepository.SCOPED_ONLY, + ) -> RuntimeStateRepository: + runtime_dir = root / "runtime" + return RuntimeStateRepository( + runtime_root=runtime_dir / scope, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id=scope, + compat_mode=compat_mode, + ) + + @staticmethod + def _state( + run_id: str, + project: str, + *, + port: int = 8000, + metadata: dict[str, object] | None = None, + ) -> RunState: + return RunState( + run_id=run_id, + mode="main" if project.casefold() == "main" else "trees", + services={ + f"{project} Backend": ServiceRecord( + name=f"{project} Backend", + type="backend", + cwd=f"/tmp/{project}/backend", + project=project, + requested_port=port, + actual_port=port, + status="running", + ) + }, + metadata=dict(metadata or {}), + ) + + @staticmethod + def _context(root: Path, project: str, port: int) -> object: + return SimpleNamespace( + name=project, + root=root / project, + ports={ + "backend": SimpleNamespace( + requested=port, + assigned=port, + final=port, + source="requested", + retries=0, + ) + }, + ) + + @staticmethod + def _save_resume(repo: RuntimeStateRepository, state: RunState) -> None: + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda saved: {"run_id": saved.run_id}, + ) + + @staticmethod + def _save_run( + repo: RuntimeStateRepository, + state: RunState, + *, + context: object, + errors: list[str] | None = None, + ) -> None: + repo.save_run( + state=state, + contexts=[context], + errors=list(errors or []), + events=[{"event": state.run_id}], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda saved: {"run_id": saved.run_id}, + ) + + def test_clean_save_scavenges_crash_temps_for_rewritten_alias_targets(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + state = self._state("run-a", "Main") + self._save_resume(repo, state) + run_dir = repo.run_dir_path("run-a") + stale_root_alias = repo.runtime_root / ".run_state.json.abcdefgh.tmp" + stale_run_alias = run_dir / ".run_state.json.ijklmnop.tmp" + stale_index = repo.runtime_root / ".run_index.json.qrstuvwx.tmp" + for path in (stale_root_alias, stale_run_alias, stale_index): + path.write_text("crash debris", encoding="utf-8") + + self._save_resume(repo, state) + + self.assertFalse(stale_root_alias.exists()) + self.assertFalse(stale_run_alias.exists()) + self.assertFalse(stale_index.exists()) + + def test_revision_retention_keeps_indexed_revision_and_one_grace_revision(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + revisions_dir = repo.run_dir_path("run-a") / "revisions" + external = root / "external" + external.mkdir() + + with patch( + "envctl_engine.state.repository.fsync_directory", + wraps=fsync_directory, + ) as synced_directory: + for port in range(8100, 8106): + self._save_resume(repo, self._state("run-a", "feature-a", port=port)) + real_revisions = [ + path for path in revisions_dir.iterdir() if path.is_dir() and not path.is_symlink() + ] + self.assertLessEqual(len(real_revisions), 2) + + candidate = repo.run_index.candidates(StateSelector(mode="trees", project_names=()))[0] + real_revisions = [path for path in revisions_dir.iterdir() if path.is_dir() and not path.is_symlink()] + self.assertEqual(len(real_revisions), 2) + self.assertIn(candidate.state_path.parent, real_revisions) + self.assertTrue(any(call.args == (revisions_dir,) for call in synced_directory.call_args_list)) + + external_sentinel = external / "sentinel" + external_sentinel.write_text("safe", encoding="utf-8") + revision_link = revisions_dir / "foreign-link" + revision_link.symlink_to(external, target_is_directory=True) + self._save_resume(repo, self._state("run-a", "feature-a", port=9000)) + self.assertTrue(revision_link.is_symlink()) + self.assertEqual(external_sentinel.read_text(encoding="utf-8"), "safe") + + def test_revision_pruning_is_deferred_until_failed_promotion_is_repaired(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + for port in (8100, 8101): + self._save_resume(repo, self._state("run-a", "feature-a", port=port)) + revisions_dir = repo.run_dir_path("run-a") / "revisions" + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 2) + + with patch.object( + repo, + "_promote_latest_active_state", + side_effect=OSError("alias promotion unavailable"), + ): + self._save_resume(repo, self._state("run-a", "feature-a", port=8102)) + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 3) + + loaded = repo.load_latest(mode="trees", strict_mode_match=True) + self.assertIsNotNone(loaded) + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 2) + + def test_supersession_prunes_failed_promotion_backlog_for_retired_run(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + for port in (8100, 8101): + self._save_resume(repo, self._state("run-old", "feature-a", port=port)) + revisions_dir = repo.run_dir_path("run-old") / "revisions" + with patch.object( + repo, + "_promote_latest_active_state", + side_effect=OSError("alias promotion unavailable"), + ): + for port in (8102, 8103, 8104): + self._save_resume(repo, self._state("run-old", "feature-a", port=port)) + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 5) + + replacement = self._state( + "run-new", + "feature-a", + port=8200, + metadata={"state_source_run_ids": ["run-old"]}, + ) + self._save_resume(repo, replacement) + + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 2) + payload = json.loads(repo.run_index.index_path.read_text(encoding="utf-8")) + self.assertEqual([entry["run_id"] for entry in payload["entries"]], ["run-new"]) + self.assertIn("run-old", payload["retired_run_ids"]) + + def test_revision_cleanup_io_failures_do_not_lose_state_and_are_retried(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + for port in (8100, 8101): + self._save_resume(repo, self._state("run-old", "feature-a", port=port)) + revisions_dir = repo.run_dir_path("run-old") / "revisions" + with patch.object( + repo, + "_promote_latest_active_state", + side_effect=OSError("alias promotion unavailable"), + ): + for port in (8102, 8103, 8104): + self._save_resume(repo, self._state("run-old", "feature-a", port=port)) + + replacement = self._state( + "run-new", + "feature-a", + port=8200, + metadata={"state_source_run_ids": ["run-old"]}, + ) + with patch( + "envctl_engine.state.repository.shutil.rmtree", + side_effect=OSError("cleanup unavailable"), + ): + self._save_resume(repo, replacement) + + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 5) + cleanup_dir = repo.runtime_root / repo._REVISION_CLEANUP_DIR_NAME + self.assertEqual(len(list(cleanup_dir.glob("*.json"))), 1) + for index in range(100): + clean_history = repo.run_dir_path(f"inactive-{index}") / "revisions" / "only-revision" + clean_history.mkdir(parents=True) + original_prune = repo._prune_run_revisions_unlocked + inspected_run_ids: list[str] = [] + + def recording_prune(candidate: object) -> None: + inspected_run_ids.append(str(getattr(candidate, "run_id", ""))) + original_prune(candidate) # type: ignore[arg-type] + + with patch.object( + repo, + "_prune_run_revisions_unlocked", + side_effect=recording_prune, + ): + loaded = repo.load_latest(mode="trees", strict_mode_match=True) + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual(loaded.run_id, "run-new") + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 2) + self.assertEqual(set(inspected_run_ids), {"run-new", "run-old"}) + self.assertFalse(cleanup_dir.exists()) + + with patch( + "envctl_engine.state.repository.fsync_directory", + side_effect=OSError("directory fsync failed"), + ): + for port in (8201, 8202): + self._save_resume(repo, self._state("run-new", "feature-a", port=port)) + loaded = repo.load_latest(mode="trees", strict_mode_match=True) + self.assertIsNotNone(loaded) + self.assertLessEqual( + len([path for path in (repo.run_dir_path("run-new") / "revisions").iterdir() if path.is_dir()]), + 2, + ) + + def test_superseded_cleanup_is_queued_when_alias_promotion_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + with patch.object( + repo, + "_promote_latest_active_state", + side_effect=OSError("alias promotion unavailable"), + ): + for port in range(8100, 8105): + self._save_resume(repo, self._state("run-old", "feature-a", port=port)) + self._save_resume( + repo, + self._state( + "run-new", + "feature-a", + port=8200, + metadata={"state_source_run_ids": ["run-old"]}, + ), + ) + + old_revisions = repo.run_dir_path("run-old") / "revisions" + cleanup_dir = repo.runtime_root / repo._REVISION_CLEANUP_DIR_NAME + self.assertEqual( + len([path for path in old_revisions.iterdir() if path.is_dir()]), + 5, + ) + self.assertEqual(len(list(cleanup_dir.glob("*.json"))), 1) + + loaded = repo.load_latest(mode="trees", strict_mode_match=True) + + self.assertIsNotNone(loaded) + self.assertLessEqual( + len([path for path in old_revisions.iterdir() if path.is_dir()]), + 2, + ) + self.assertFalse(cleanup_dir.exists()) + + def test_current_ancillary_alias_signatures_detect_tampering_and_source_updates(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository( + root, + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + self._save_run( + repo, + self._state("run-a", "Main"), + context=self._context(root, "Main", 8000), + errors=["expected error"], + ) + self.assertTrue( + repo.write_runtime_artifact( + run_id="run-a", + artifact_name="runtime_readiness_report.json", + text='{"passed": true}', + ) + ) + candidate = repo.run_index.candidates(StateSelector(mode="main", project_names=()))[0] + scoped_ports = repo.ports_manifest_path() + expected_ports = scoped_ports.read_text(encoding="utf-8") + scoped_ports.write_text("x" * len(expected_ports), encoding="utf-8") + repo.load_latest() + self.assertEqual(scoped_ports.read_text(encoding="utf-8"), expected_ports) + + legacy_error = repo.runtime_legacy_root / "error_report.json" + expected_error = legacy_error.read_text(encoding="utf-8") + legacy_error.write_text("x" * len(expected_error), encoding="utf-8") + repo.load_latest() + repaired_error = json.loads(legacy_error.read_text(encoding="utf-8")) + self.assertEqual(repaired_error["errors"], ["expected error"]) + self.assertEqual( + legacy_error.read_text(encoding="utf-8"), + repo.error_report_path().read_text(encoding="utf-8"), + ) + + source_events = candidate.state_path.parent / "events.jsonl" + updated_events = '{"event": "late-source-update"}\n' + source_events.write_text(updated_events, encoding="utf-8") + repo.load_latest() + self.assertEqual( + (repo.runtime_root / "events.jsonl").read_text(encoding="utf-8"), + updated_events, + ) + self.assertEqual( + (repo.runtime_legacy_root / "events.jsonl").read_text(encoding="utf-8"), + updated_events, + ) + + def test_per_run_alias_manifest_uses_signatures_and_repairs_tampering(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + self._save_run( + repo, + self._state("run-a", "Main"), + context=self._context(root, "Main", 8000), + ) + self.assertTrue( + repo.write_runtime_artifact( + run_id="run-a", + artifact_name="runtime_readiness_report.json", + text='{"passed": true}', + ) + ) + repo.load_latest() + candidate = repo.run_index.candidates(StateSelector(mode="main", project_names=()))[0] + + observed_reads: list[Path] = [] + original_read_text = Path.read_text + + def recording_read_text(path: Path, *args: object, **kwargs: object) -> str: + observed_reads.append(path) + return original_read_text(path, *args, **kwargs) + + indexed = repo._load_valid_index_states( + [candidate], + allowed_root=str(repo.runtime_dir), + ) + observed_reads.clear() + with patch.object(Path, "read_text", recording_read_text): + repo._reconcile_run_aliases_unlocked(indexed) + + source_dir = candidate.state_path.parent + run_dir = repo.run_dir_path("run-a") + compared_artifacts = {source_dir / artifact_name for artifact_name in repo._RUN_ALIAS_ARTIFACT_NAMES} | { + run_dir / artifact_name for artifact_name in repo._RUN_ALIAS_ARTIFACT_NAMES + } + self.assertTrue(compared_artifacts.isdisjoint(observed_reads)) + + tampered_alias = run_dir / "events.jsonl" + expected_events = (source_dir / "events.jsonl").read_text(encoding="utf-8") + tampered_alias.write_text("x" * len(expected_events), encoding="utf-8") + repo.load_latest() + self.assertEqual(tampered_alias.read_text(encoding="utf-8"), expected_events) + + def test_deactivate_does_not_retire_unknown_run_ids(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + repo = self._repository(Path(tmpdir)) + self._save_resume(repo, self._state("run-active", "Main")) + + self.assertFalse(repo.deactivate_run("never-active")) + self.assertTrue(repo.deactivate_runs(["run-active", "also-never-active"])) + + payload = json.loads(repo.run_index.index_path.read_text(encoding="utf-8")) + self.assertEqual(payload["retired_run_ids"], ["run-active"]) + + def test_legacy_migration_preserves_failed_health_state_and_all_artifacts(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime_dir = root / "runtime" + legacy_root = runtime_dir / "python-engine" + legacy_root.mkdir(parents=True) + failed_state = RunState( + run_id="legacy-failed", + mode="main", + requirements={ + "Main": RequirementsResult( + project="Main", + health="degraded", + failures=["redis unavailable"], + components={ + "redis": { + "enabled": True, + "success": False, + "runtime_status": "unreachable", + } + }, + ) + }, + metadata={"failed": True}, + ) + dump_state(failed_state, str(legacy_root / "run_state.json")) + artifacts = { + "runtime_map.json": '{"legacy": true}', + "ports_manifest.json": '{"projects": [{"project": "Main", "ports": {}}]}', + "error_report.json": '{"errors": ["redis unavailable"]}', + "events.jsonl": '{"event": "failed"}\n', + "runtime_readiness_report.json": '{"passed": false}', + } + for artifact_name, text in artifacts.items(): + (legacy_root / artifact_name).write_text(text, encoding="utf-8") + + repo = self._repository( + root, + compat_mode=RuntimeStateRepository.COMPAT_READ_ONLY, + ) + loaded = repo.load_latest(mode="main", strict_mode_match=True) + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertTrue(loaded.metadata["failed"]) + self.assertEqual(loaded.requirements["Main"].health, "degraded") + self.assertEqual( + loaded.requirements["Main"].component("redis")["runtime_status"], + "unreachable", + ) + candidate = repo.run_index.candidates(StateSelector(mode="main", project_names=()))[0] + for artifact_name, expected in artifacts.items(): + self.assertEqual( + (candidate.state_path.parent / artifact_name).read_text(encoding="utf-8"), + expected, + ) + + def test_unbound_diagnostics_do_not_replace_active_aliases_and_purge_cleans_them(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository( + root, + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + runtime_root = repo.runtime_root + legacy_root = repo.runtime_legacy_root + no_run_event = '{"event": "no-run"}\n' + self.assertTrue( + repo.write_runtime_artifact( + run_id=None, + artifact_name="events.jsonl", + text=no_run_event, + ) + ) + self.assertEqual((runtime_root / "events.jsonl").read_text(encoding="utf-8"), no_run_event) + self.assertEqual((legacy_root / "events.jsonl").read_text(encoding="utf-8"), no_run_event) + + self._save_run( + repo, + self._state("run-a", "Main"), + context=self._context(root, "Main", 8000), + ) + active_event = (runtime_root / "events.jsonl").read_text(encoding="utf-8") + diagnostic_event = '{"event": "doctor"}\n' + self.assertTrue( + repo.write_runtime_artifact( + run_id=None, + artifact_name="events.jsonl", + text=diagnostic_event, + ) + ) + self.assertEqual((runtime_root / "events.jsonl").read_text(encoding="utf-8"), active_event) + self.assertEqual((legacy_root / "events.jsonl").read_text(encoding="utf-8"), active_event) + self.assertEqual( + (runtime_root / "diagnostics" / "events.jsonl").read_text(encoding="utf-8"), + diagnostic_event, + ) + + repo.purge(aggressive=False) + self.assertFalse((runtime_root / "diagnostics").exists()) + + def test_shared_legacy_pointers_are_only_removed_by_their_own_scope(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo_a = self._repository( + root, + scope="scope-a", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + repo_b = self._repository( + root, + scope="scope-b", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + self._save_run( + repo_a, + self._state("run-a", "feature-a"), + context=self._context(root, "feature-a", 8100), + ) + self._save_run( + repo_b, + self._state("run-b", "feature-b"), + context=self._context(root, "feature-b", 8200), + ) + legacy_root = repo_a.runtime_legacy_root + pointer_b_name = repo_b._tree_pointer_name("feature-b") + assert pointer_b_name is not None + pointer_b = legacy_root / pointer_b_name + generic_pointer = legacy_root / ".last_state" + self.assertIn("scope-b", Path(pointer_b.read_text(encoding="utf-8")).parts) + self.assertIn("scope-b", Path(generic_pointer.read_text(encoding="utf-8")).parts) + + self.assertTrue(repo_a.deactivate_run("run-a")) + + self.assertTrue(pointer_b.is_file()) + self.assertTrue(generic_pointer.is_file()) + self.assertIn("scope-b", Path(pointer_b.read_text(encoding="utf-8")).parts) + self.assertIn("scope-b", Path(generic_pointer.read_text(encoding="utf-8")).parts) + + def test_partial_restart_ports_merge_current_context_with_preserved_project(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + initial = RunState( + run_id="run-ab", + mode="trees", + services={ + **self._state("unused", "feature-a", port=8100).services, + **self._state("unused", "feature-b", port=8200).services, + }, + ) + repo.save_run( + state=initial, + contexts=[ + self._context(root, "feature-a", 8100), + self._context(root, "feature-b", 8200), + ], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda saved: {"run_id": saved.run_id}, + ) + restarted = RunState( + run_id="run-restart-a", + mode="trees", + services={ + **self._state("unused", "feature-a", port=8111).services, + **self._state("unused", "feature-b", port=8200).services, + }, + metadata={"state_source_run_ids": ["run-ab"]}, + ) + self._save_run( + repo, + restarted, + context=self._context(root, "feature-a", 8111), + ) + + payload = json.loads(repo.ports_manifest_path().read_text(encoding="utf-8")) + ports_by_project = { + project["project"]: project["ports"]["backend"]["final"] for project in payload["projects"] + } + self.assertEqual(ports_by_project, {"feature-a": 8111, "feature-b": 8200}) + + def test_aggregate_aliases_merge_independent_owner_ports_and_errors(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + self._save_run( + repo, + self._state("run-a", "feature-a", port=8100), + context=self._context(root, "feature-a", 8100), + errors=["feature-a failed"], + ) + self._save_run( + repo, + self._state("run-b", "feature-b", port=8200), + context=self._context(root, "feature-b", 8200), + errors=["feature-b failed"], + ) + + current = load_state(str(repo.run_state_path())) + self.assertEqual(set(current.services), {"feature-a Backend", "feature-b Backend"}) + ports = json.loads(repo.ports_manifest_path().read_text(encoding="utf-8")) + errors = json.loads(repo.error_report_path().read_text(encoding="utf-8")) + self.assertEqual( + {project["project"] for project in ports["projects"]}, + {"feature-a", "feature-b"}, + ) + self.assertEqual(set(errors["errors"]), {"feature-a failed", "feature-b failed"}) + + self.assertTrue(repo.deactivate_run("run-a")) + remaining = load_state(str(repo.run_state_path())) + ports = json.loads(repo.ports_manifest_path().read_text(encoding="utf-8")) + errors = json.loads(repo.error_report_path().read_text(encoding="utf-8")) + self.assertEqual(set(remaining.services), {"feature-b Backend"}) + self.assertEqual( + {project["project"] for project in ports["projects"]}, + {"feature-b"}, + ) + self.assertEqual(errors["errors"], ["feature-b failed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/state/test_state_roundtrip.py b/tests/python/state/test_state_roundtrip.py index 781aad674..b130670f9 100644 --- a/tests/python/state/test_state_roundtrip.py +++ b/tests/python/state/test_state_roundtrip.py @@ -6,7 +6,7 @@ REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" -from envctl_engine.state.models import RunState, ServiceRecord +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord from envctl_engine.state import dump_state, load_legacy_shell_state, load_state, merge_states @@ -25,11 +25,22 @@ def test_json_state_roundtrip_preserves_services(self) -> None: pid=12345, requested_port=8000, actual_port=8001, + port_lock_session="planner-session-1", listener_pids=[12345, 12346], status="running", started_at=1234.5, ) }, + requirements={ + "Tree Alpha": RequirementsResult( + project="Tree Alpha", + db={ + "enabled": True, + "final": 5432, + "port_lock_session": "planner-session-1", + }, + ) + }, metadata={"source": "python"}, ) @@ -38,8 +49,16 @@ def test_json_state_roundtrip_preserves_services(self) -> None: self.assertEqual(loaded.run_id, "run-123") self.assertEqual(loaded.services["Tree Alpha Backend"].actual_port, 8001) + self.assertEqual( + loaded.services["Tree Alpha Backend"].port_lock_session, + "planner-session-1", + ) self.assertEqual(loaded.services["Tree Alpha Backend"].listener_pids, [12345, 12346]) self.assertEqual(loaded.services["Tree Alpha Backend"].started_at, 1234.5) + self.assertEqual( + loaded.requirements["Tree Alpha"].db["port_lock_session"], + "planner-session-1", + ) self.assertEqual(loaded.metadata["source"], "python") def test_legacy_shell_state_is_loaded_without_source_execution(self) -> None: From 03e91463f8f1ee65184956c14b93937ae73ae78c Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:26:49 +0300 Subject: [PATCH 09/26] Restore legacy pointer migration into scoped state registry --- python/envctl_engine/state/repository.py | 30 +++++++++++++++++-- .../state/test_state_shell_compatibility.py | 15 ++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/python/envctl_engine/state/repository.py b/python/envctl_engine/state/repository.py index 5e33259b9..c1ba28df2 100644 --- a/python/envctl_engine/state/repository.py +++ b/python/envctl_engine/state/repository.py @@ -13,7 +13,7 @@ from envctl_engine.dashboard_metadata import DASHBOARD_STOPPED_SERVICES_KEY from envctl_engine.shared.artifact_names import safe_artifact_stem -from envctl_engine.state import load_legacy_shell_state, load_state, state_to_dict +from envctl_engine.state import load_legacy_shell_state, load_state, load_state_from_pointer, state_to_dict from envctl_engine.state.fingerprints import file_fingerprint, state_fingerprint, text_fingerprint from envctl_engine.state.models import RunState, ServiceRecord from envctl_engine.state.persistence import ( @@ -1111,10 +1111,14 @@ def _ensure_active_index_unlocked(self) -> None: continue recovered_run_ids = {state.run_id for _, _, state, _ in recovered} for root in self._legacy_migration_roots(): - for candidate_path, legacy_loader in ( + legacy_candidates = [ (root / "run_state.json", self._load_state_file_candidate), (root / "run_state.state", self._load_legacy_shell_candidate), - ): + ] + legacy_candidates.extend( + (pointer, self._load_pointer_candidate) for pointer in self._legacy_pointer_paths(root) + ) + for candidate_path, legacy_loader in legacy_candidates: if candidate_path.is_symlink() or not candidate_path.is_file(): continue try: @@ -1129,6 +1133,10 @@ def _ensure_active_index_unlocked(self) -> None: project_names = self._project_names_from_ports_manifest(root / "ports_manifest.json") if not project_names: project_names = self._project_names_from_pointers(root, candidate_path) + if not project_names and candidate_path.name.startswith(".last_state.trees."): + project_name = candidate_path.name.removeprefix(".last_state.trees.").strip() + if project_name: + project_names = [project_name] state.metadata["repo_scope_id"] = self.runtime_scope_id state.metadata["project_names"] = sorted(set(project_names), key=str.casefold) recovered.append((candidate_path.stat().st_mtime_ns, candidate_path, state, project_names)) @@ -1173,6 +1181,22 @@ def _legacy_migration_roots(self) -> list[Path]: roots.append(self.runtime_legacy_root) return roots + @staticmethod + def _legacy_pointer_paths(root: Path) -> list[Path]: + candidates = [ + root / ".last_state.main", + *sorted(root.glob(".last_state.trees.*")), + root / ".last_state", + ] + return [pointer for pointer in candidates if not pointer.is_symlink() and pointer.is_file()] + + @staticmethod + def _load_pointer_candidate(path: Path, *, allowed_root: str) -> RunState | None: + try: + return load_state_from_pointer(str(path), allowed_root=allowed_root) + except Exception: + return None + def _stage_legacy_revision(self, *, state: RunState, source_path: Path) -> Path: run_dir = self.run_dir_path(state.run_id) revisions_dir = self._contained_child(run_dir, "revisions", label="revisions directory") diff --git a/tests/python/state/test_state_shell_compatibility.py b/tests/python/state/test_state_shell_compatibility.py index f293f9a3a..a2ef3bcac 100644 --- a/tests/python/state/test_state_shell_compatibility.py +++ b/tests/python/state/test_state_shell_compatibility.py @@ -134,6 +134,21 @@ def test_engine_runtime_resolves_state_from_pointer_files(self) -> None: assert loaded is not None self.assertEqual(loaded.mode, "main") self.assertIn("Main Backend", loaded.services) + primary_index = runtime.runtime_root / "run_index.json" + backup_index = runtime.runtime_root / "run_index.backup.json" + self.assertEqual( + primary_index.read_text(encoding="utf-8"), + backup_index.read_text(encoding="utf-8"), + ) + migrated_pointer = Path((runtime.runtime_root / ".last_state.main").read_text(encoding="utf-8").strip()) + self.assertTrue(migrated_pointer.is_file()) + self.assertTrue(migrated_pointer.is_relative_to(runtime.runtime_root / "runs")) + self.assertTrue(state_path.is_file()) + + loaded_again = runtime._try_load_existing_state() + self.assertIsNotNone(loaded_again) + assert loaded_again is not None + self.assertEqual(loaded_again.run_id, loaded.run_id) if __name__ == "__main__": From ce242f3272bebabf520a3caad22474437d2eeed8 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:37:51 +0300 Subject: [PATCH 10/26] Make resume fallback tests PID-independent --- .../test_engine_runtime_plan_resume_startup.py | 5 ++++- .../runtime/test_engine_runtime_resume_startup.py | 12 ++++++++++-- .../startup/test_startup_orchestrator_flow_reuse.py | 2 ++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/python/runtime/test_engine_runtime_plan_resume_startup.py b/tests/python/runtime/test_engine_runtime_plan_resume_startup.py index 1c7b37174..1a239fd61 100644 --- a/tests/python/runtime/test_engine_runtime_plan_resume_startup.py +++ b/tests/python/runtime/test_engine_runtime_plan_resume_startup.py @@ -100,12 +100,14 @@ def fake_start_project_context(*, context, mode, route, run_id): # noqa: ANN001 with ( patch.object(engine, "_try_load_existing_state", return_value=state), patch.object(engine, "_resume", return_value=1) as resume_mock, + patch.object(engine, "_terminate_services_from_state", return_value=set()) as terminate_mock, patch.object(engine, "_start_project_context", side_effect=fake_start_project_context), ): code = engine.dispatch(route) self.assertEqual(code, 0) self.assertEqual(resume_mock.call_count, 1) + self.assertEqual(terminate_mock.call_count, 1) self.assertEqual(started_projects, ["feature-a-1"]) def test_plan_auto_resumes_existing_run_when_selected_projects_are_subset_of_state(self) -> None: @@ -282,6 +284,7 @@ def test_plan_no_resume_flag_disables_auto_resume(self) -> None: with ( patch.object(engine, "_try_load_existing_state", return_value=state), patch.object(engine, "_resume", return_value=0) as resume_mock, + patch.object(engine, "_terminate_services_from_state", return_value=set()) as terminate_mock, patch.object(engine, "_start_project_context", side_effect=RuntimeError("expected startup path")), ): out = StringIO() @@ -290,6 +293,7 @@ def test_plan_no_resume_flag_disables_auto_resume(self) -> None: self.assertEqual(code, 1) self.assertEqual(resume_mock.call_count, 0) + self.assertEqual(terminate_mock.call_count, 1) self.assertIn("expected startup path", out.getvalue()) def test_exact_tree_plan_match_reuses_prior_run_id(self) -> None: @@ -339,4 +343,3 @@ def fake_result(context: ProjectContext) -> ProjectStartupResult: assert second_state is not None self.assertEqual(second_state.run_id, first_state.run_id) self.assertNotEqual(first_engine._current_session_id(), second_session_id) - diff --git a/tests/python/runtime/test_engine_runtime_resume_startup.py b/tests/python/runtime/test_engine_runtime_resume_startup.py index 30bc9014b..8bc3e98c9 100644 --- a/tests/python/runtime/test_engine_runtime_resume_startup.py +++ b/tests/python/runtime/test_engine_runtime_resume_startup.py @@ -225,12 +225,20 @@ def fake_result(context: ProjectContext) -> ProjectStartupResult: second_engine = PythonEngineRuntime(self._config(repo, runtime, extra={"BACKEND_DIR": "api"}), env={}) second_engine._reconcile_state_truth = lambda _state: [] # type: ignore[assignment] - with patch.object( - second_engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"]) + with ( + patch.object( + second_engine, + "_terminate_services_from_state", + return_value=set(), + ) as terminate_mock, + patch.object( + second_engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"]) + ), ): second_code = second_engine.dispatch(parse_route(["--plan", "feature-a", "--batch"], env={})) self.assertEqual(second_code, 0) + self.assertEqual(terminate_mock.call_count, 1) second_state = second_engine.state_repository.load_latest(mode="trees", strict_mode_match=True) self.assertIsNotNone(second_state) assert second_state is not None diff --git a/tests/python/startup/test_startup_orchestrator_flow_reuse.py b/tests/python/startup/test_startup_orchestrator_flow_reuse.py index d9c159aa3..63d80557e 100644 --- a/tests/python/startup/test_startup_orchestrator_flow_reuse.py +++ b/tests/python/startup/test_startup_orchestrator_flow_reuse.py @@ -441,6 +441,7 @@ def test_resume_reuse_failure_falls_back_to_fresh_run_id_before_failure_write(se patch.object(engine, "_discover_projects", return_value=[context]), patch.object(engine, "_try_load_existing_state", return_value=existing_state), patch.object(engine, "_resume", return_value=1), + patch.object(engine, "_terminate_services_from_state", return_value=set()) as terminate_mock, patch.object(engine, "_new_run_id", return_value="run-fresh-after-resume-failure"), patch.object(engine, "_start_project_context", side_effect=RuntimeError("fresh startup failed")), patch.object( @@ -455,6 +456,7 @@ def test_resume_reuse_failure_falls_back_to_fresh_run_id_before_failure_write(se code = engine.dispatch(parse_route(["start", "--batch"], env={"ENVCTL_DEFAULT_MODE": "main"})) self.assertEqual(code, 1) + self.assertEqual(terminate_mock.call_count, 1) written_state = cast(RunState, captured["state"]) self.assertEqual(written_state.run_id, "run-fresh-after-resume-failure") self.assertNotEqual(written_state.run_id, existing_state.run_id) From 72cec8a8f570ad3fb5e4b5f24b6751f52ea5e3d8 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:08:53 +0300 Subject: [PATCH 11/26] Prevent missing-interpreter startup failures --- docs/reference/configuration.md | 7 ++ .../runtime/engine_runtime_commands.py | 24 ++-- .../runtime/engine_runtime_service_truth.py | 2 + .../runtime/service_listener_truth.py | 16 +++ .../runtime/service_truth_diagnostics.py | 32 ++++- .../startup/service_attach_execution.py | 39 +++++-- .../runtime/test_engine_runtime_commands.py | 37 ++++++ .../test_engine_runtime_real_startup.py | 3 +- .../test_engine_runtime_service_truth.py | 39 ++++++- .../startup/test_service_attach_execution.py | 109 +++++++++++++++++- 10 files changed, 280 insertions(+), 28 deletions(-) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 227ffd302..8625ae8a0 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -139,6 +139,13 @@ Only simple `${VAR}` placeholders are supported. Shell-style defaults, command s Frontend launch-env sections may use `ENVCTL_SOURCE_SUPABASE_URL` and `ENVCTL_SOURCE_SUPABASE_ANON_KEY`. They cannot reference `ENVCTL_SOURCE_SUPABASE_SERVICE_ROLE_KEY`; envctl rejects that mapping before launching the frontend process. +Every launched service also receives `ENVCTL_PYTHON_EXECUTABLE`, the absolute +interpreter path running envctl. Repository-owned shell launchers can use this +as a bootstrap interpreter on minimal hosts where a `python` command alias is +not installed. Applications should still create and execute their own project +environment rather than importing application dependencies from envctl's +environment. + ## Managed Supabase Ports Managed Supabase exposes two distinct local resources: diff --git a/python/envctl_engine/runtime/engine_runtime_commands.py b/python/envctl_engine/runtime/engine_runtime_commands.py index 49fbddf58..a50437637 100644 --- a/python/envctl_engine/runtime/engine_runtime_commands.py +++ b/python/envctl_engine/runtime/engine_runtime_commands.py @@ -4,6 +4,7 @@ from pathlib import Path import shlex import shutil +import sys from typing import Any, Mapping from envctl_engine.runtime.command_resolution import resolve_requirement_start_command, resolve_service_start_command @@ -150,14 +151,14 @@ def split_command( def _command_exists_for_cwd(runtime: Any, executable: str, *, cwd: str | Path | None) -> bool: - if runtime._command_exists(executable): - return True - if cwd is None or ("/" not in executable and "\\" not in executable): - return False + if "/" not in executable and "\\" not in executable: + return bool(runtime._command_exists(executable)) candidate = Path(executable).expanduser() - if candidate.is_absolute(): - return candidate.exists() - return (Path(cwd) / candidate).exists() + if not candidate.is_absolute(): + if cwd is None: + return False + candidate = Path(cwd) / candidate + return candidate.is_file() and os.access(candidate, os.X_OK) def command_env(runtime: Any, *, port: int, extra: Mapping[str, str] | None = None) -> dict[str, str]: @@ -166,6 +167,15 @@ def command_env(runtime: Any, *, port: int, extra: Mapping[str, str] | None = No env["PORT"] = str(port) if extra: env.update(extra) + # Every envctl process necessarily has one known-good Python executable, + # even on minimal hosts that intentionally provide no ``python`` shim. + # Repository-owned launchers can use this explicit path to bootstrap their + # own isolated environments without guessing which command name exists. + python_executable = str(sys.executable or "").strip() + if python_executable: + env["ENVCTL_PYTHON_EXECUTABLE"] = python_executable + else: + env.pop("ENVCTL_PYTHON_EXECUTABLE", None) for key in SERVICE_LAUNCH_ENV_REMOVALS: env.pop(key, None) return env diff --git a/python/envctl_engine/runtime/engine_runtime_service_truth.py b/python/envctl_engine/runtime/engine_runtime_service_truth.py index 1ada806a5..0df85d2d1 100644 --- a/python/envctl_engine/runtime/engine_runtime_service_truth.py +++ b/python/envctl_engine/runtime/engine_runtime_service_truth.py @@ -28,6 +28,7 @@ STARTUP_FAILURE_TOKENS, STARTUP_PROGRESS_TOKENS, command_result_error_text, + service_listener_failure_class, service_listener_failure_detail, tail_log_error_line, tail_log_startup_progress_line, @@ -50,6 +51,7 @@ "rebind_stale_service_pid", "refresh_service_listener_pids", "service_display_name", + "service_listener_failure_class", "service_listener_failure_detail", "service_startup_progress_timeout", "service_truth_discovery", diff --git a/python/envctl_engine/runtime/service_listener_truth.py b/python/envctl_engine/runtime/service_listener_truth.py index f6f6c1377..8cdc4390d 100644 --- a/python/envctl_engine/runtime/service_listener_truth.py +++ b/python/envctl_engine/runtime/service_listener_truth.py @@ -35,6 +35,12 @@ def wait_for_service_listener( return True except Exception: # noqa: BLE001 pass + # A generic port probe is only a safe fallback while the process envctl + # launched is still alive. Once that PID has exited, waiting on the port can + # hide the real startup error for minutes and can adopt an unrelated process + # that happens to bind the same port later. + if _launch_pid_definitively_exited(runtime, pid): + return False if ( debug_listener_group in {"", "port_fallback"} and service_truth_fallback_enabled(runtime) @@ -51,6 +57,16 @@ def wait_for_service_listener( return False +def _launch_pid_definitively_exited(runtime: Any, pid: int) -> bool: + checker = getattr(runtime.process_runner, "is_pid_running", None) + if not callable(checker): + return False + try: + return not bool(checker(pid)) + except Exception: # noqa: BLE001 + return False + + def process_tree_probe_supported(runtime: Any) -> bool: try: return bool(runtime.process_runner.supports_process_tree_probe()) diff --git a/python/envctl_engine/runtime/service_truth_diagnostics.py b/python/envctl_engine/runtime/service_truth_diagnostics.py index 094fb23ff..7fcf7b4b5 100644 --- a/python/envctl_engine/runtime/service_truth_diagnostics.py +++ b/python/envctl_engine/runtime/service_truth_diagnostics.py @@ -1,8 +1,11 @@ from __future__ import annotations from pathlib import Path +import re from typing import Any +from envctl_engine.shared.reason_codes import ServiceFailureReason + STARTUP_PROGRESS_TOKENS = ( "Waiting for application startup", @@ -28,6 +31,9 @@ "No module named", "Address already in use", "address already in use", + "command not found", + "exec:", + "No such file or directory", ) @@ -64,6 +70,25 @@ def service_listener_failure_detail(runtime: Any, *, log_path: str | None, pid: return "; ".join(parts) +def service_listener_failure_class(detail: str | None) -> str: + """Classify listener failures using process and log evidence.""" + + text = str(detail or "") + normalized = text.casefold() + missing_executable = bool( + "command not found" in normalized + or "modulenotfounderror" in normalized + or "no module named" in normalized + or re.search(r"\bexec:\s+[^:;\s]+:\s+not found\b", normalized) + or ("/usr/bin/env:" in normalized and "no such file or directory" in normalized) + ) + if missing_executable: + return ServiceFailureReason.DEPENDENCY_MISSING.value + if re.search(r"\bprocess(?:\s+\d+)?\s+exited\b", normalized): + return ServiceFailureReason.PROCESS_EXITED.value + return "listener_not_detected" + + def tail_log_error_line(log_path: str | None, *, max_chars: int = 800) -> str | None: lines = _tail_candidate_lines(log_path) if not lines: @@ -77,6 +102,9 @@ def tail_log_error_line(log_path: str | None, *, max_chars: int = 800) -> str | "Error", "Exception", "No module named", + "command not found", + "exec:", + "No such file or directory", ) selected = lines[-1] selected_index = len(lines) - 1 @@ -115,9 +143,7 @@ def _tail_candidate_lines(log_path: str | None) -> list[str]: return [] try: return [ - line.strip() - for line in path.read_text(encoding="utf-8", errors="replace").splitlines() - if line.strip() + line.strip() for line in path.read_text(encoding="utf-8", errors="replace").splitlines() if line.strip() ] except OSError: return [] diff --git a/python/envctl_engine/startup/service_attach_execution.py b/python/envctl_engine/startup/service_attach_execution.py index e15a202c8..c0e51fc24 100644 --- a/python/envctl_engine/startup/service_attach_execution.py +++ b/python/envctl_engine/startup/service_attach_execution.py @@ -7,6 +7,8 @@ from typing import Any, cast from envctl_engine.runtime.service_manager import ServiceCleanupError, ServiceStartDescriptor +from envctl_engine.runtime.service_truth_diagnostics import service_listener_failure_class +from envctl_engine.shared.reason_codes import ServiceFailureReason from envctl_engine.startup.service_execution_policy import ordered_service_layers from envctl_engine.startup.service_execution_records import PreparedServiceLaunch from envctl_engine.startup.session import unconfirmed_service_names @@ -293,12 +295,24 @@ def _launch_prepared_process( env_extra: dict[str, str], ) -> tuple[bool, str | None, int | None]: launch_started = time.monotonic() - process = self._start_process( - command, - cwd=str(launch.cwd), - env=self.command_env_builder(port=launch_port, extra=env_extra), - log_path=launch.log_path, - ) + try: + process = self._start_process( + command, + cwd=str(launch.cwd), + env=self.command_env_builder(port=launch_port, extra=env_extra), + log_path=launch.log_path, + ) + except OSError as exc: + detail = f"process spawn failed for {command[0]}: {exc}" + self.runtime._emit( + "service.failure", + project=self.project_name, + service=service_name, + failure_class=ServiceFailureReason.PROCESS_SPAWN_FAILED.value, + requested_port=event_port, + detail=detail, + ) + return False, detail, None pid = getattr(process, "pid", None) try: self._emit_attach_phase(service_name, "process_launch", launch_started) @@ -533,17 +547,22 @@ def _resolve_detected_actual( if self.runtime._listener_truth_enforced(): detail = self.runtime._service_listener_failure_detail(log_path=log_path, pid=pid) error_suffix = f" ({detail})" if detail else "" + failure_class = service_listener_failure_class(detail) self.runtime._emit( "service.failure", project=self.project_name, service=service_name, - failure_class="listener_not_detected", + failure_class=failure_class, requested_port=requested, detail=detail, ) - raise RuntimeError( - f"{service_name} listener not detected for {self.project_name} on port {failure_port}{error_suffix}" - ) + if failure_class == ServiceFailureReason.DEPENDENCY_MISSING.value: + failure_summary = f"{service_name} is missing a required executable or module" + elif failure_class == ServiceFailureReason.PROCESS_EXITED.value: + failure_summary = f"{service_name} process exited before opening its listener" + else: + failure_summary = f"{service_name} listener not detected" + raise RuntimeError(f"{failure_summary} for {self.project_name} on port {failure_port}{error_suffix}") self.runtime._emit( "service.failure", project=self.project_name, diff --git a/tests/python/runtime/test_engine_runtime_commands.py b/tests/python/runtime/test_engine_runtime_commands.py index 9b1af759b..81cdd8cf5 100644 --- a/tests/python/runtime/test_engine_runtime_commands.py +++ b/tests/python/runtime/test_engine_runtime_commands.py @@ -2,6 +2,7 @@ import os from pathlib import Path +import sys import tempfile import unittest from types import SimpleNamespace @@ -49,6 +50,22 @@ def test_command_env_sets_port_and_overrides(self) -> None: self.assertEqual(env["PORT"], "9000") self.assertEqual(env["A"], "1") self.assertEqual(env["B"], "2") + self.assertEqual(env["ENVCTL_PYTHON_EXECUTABLE"], sys.executable) + + def test_command_env_keeps_running_interpreter_authoritative(self) -> None: + runtime = SimpleNamespace(env={"ENVCTL_PYTHON_EXECUTABLE": "/stale/runtime/python"}) + + with patch( + "envctl_engine.runtime.engine_runtime_commands.sys.executable", + "/opt/envctl/bin/python", + ): + env = command_env( + runtime, + port=9000, + extra={"ENVCTL_PYTHON_EXECUTABLE": "/spoofed/project/python"}, + ) + + self.assertEqual(env["ENVCTL_PYTHON_EXECUTABLE"], "/opt/envctl/bin/python") def test_command_env_strips_github_actions_cleanup_variables(self) -> None: runtime = SimpleNamespace(env={"A": "1"}) @@ -113,12 +130,32 @@ def test_split_command_accepts_relative_executable_from_cwd(self) -> None: script = root / "scripts" / "start-service.sh" script.parent.mkdir(parents=True, exist_ok=True) script.write_text("#!/usr/bin/env sh\n", encoding="utf-8") + script.chmod(0o755) runtime = SimpleNamespace(_command_exists=lambda executable: False) parsed = split_command(runtime, "scripts/start-service.sh {port}", port=8123, cwd=root) self.assertEqual(parsed, ["scripts/start-service.sh", "8123"]) + def test_split_command_rejects_non_executable_file_and_directory(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + file_path = root / "start-service.sh" + file_path.write_text("#!/usr/bin/env sh\n", encoding="utf-8") + directory_path = root / "service-dir" + directory_path.mkdir() + runtime = SimpleNamespace(_command_exists=lambda _executable: True) + + for command in (str(file_path), str(directory_path)): + with ( + self.subTest(command=command), + self.assertRaisesRegex( + RuntimeError, + "Resolved command executable not found", + ), + ): + split_command(runtime, command, cwd=root) + def test_default_python_executable_falls_back_to_python3(self) -> None: runtime = SimpleNamespace(env={}, _command_exists=lambda executable: False) diff --git a/tests/python/runtime/test_engine_runtime_real_startup.py b/tests/python/runtime/test_engine_runtime_real_startup.py index e65fcd7a8..00310d1a9 100644 --- a/tests/python/runtime/test_engine_runtime_real_startup.py +++ b/tests/python/runtime/test_engine_runtime_real_startup.py @@ -253,10 +253,11 @@ def capture_emit(event: str, **payload: object) -> None: rendered = out.getvalue() self.assertIn("\x1b]8;;file://", rendered) visible = strip_ansi(rendered) - self.assertIn("backend listener not detected", visible) + self.assertIn("backend is missing a required executable or module", visible) self.assertIn("log_path:", visible) failure_events = [payload for event, payload in captured_events if event == "service.failure"] self.assertTrue(failure_events) + self.assertEqual(failure_events[0]["failure_class"], "dependency_missing") detail = str(failure_events[0]["detail"]) self.assertIn("log_path:", detail) self.assertNotIn("\x1b]8;;", detail) diff --git a/tests/python/runtime/test_engine_runtime_service_truth.py b/tests/python/runtime/test_engine_runtime_service_truth.py index a02e39b04..24d04dcfd 100644 --- a/tests/python/runtime/test_engine_runtime_service_truth.py +++ b/tests/python/runtime/test_engine_runtime_service_truth.py @@ -16,6 +16,7 @@ listener_pids_for_port, rebind_stale_service_pid, refresh_service_listener_pids, + service_listener_failure_class, service_listener_failure_detail, service_truth_fallback_enabled, service_truth_status, @@ -34,6 +35,7 @@ def __init__(self) -> None: self.listener_pids_result: list[int] = [] self.process_tree_listener_pids_result: list[int] = [] self.running_result = True + self.wait_for_port_calls: list[tuple[int, float]] = [] def wait_for_pid_port( self, @@ -49,6 +51,7 @@ def wait_for_pid_port( def wait_for_port(self, port: int, *, host: str = "127.0.0.1", timeout: float = 30.0) -> bool: # noqa: ARG002 _ = host + self.wait_for_port_calls.append((port, timeout)) return self.wait_for_port_result def find_pid_listener_port(self, pid: int, requested_port: int, *, max_delta: int) -> int | None: # noqa: ARG002 @@ -120,6 +123,16 @@ def test_service_listener_failure_detail_reports_startup_progress_without_callin self.assertIn("startup still in progress: INFO: Waiting for application startup.", detail) self.assertNotIn("log: INFO: Waiting for application startup.", detail) + def test_service_listener_failure_class_recognizes_missing_nested_executable(self) -> None: + detail = ( + "process 2304354 exited; log_path: /tmp/voice-runtime.txt; " + "log: scripts/envctl/start-voice-runtime.sh: line 20: exec: python: not found" + ) + + self.assertEqual(service_listener_failure_class(detail), "dependency_missing") + self.assertEqual(service_listener_failure_class("process 42 exited"), "process_exited") + self.assertEqual(service_listener_failure_class("listener timed out"), "listener_not_detected") + def test_wait_for_service_listener_uses_port_probe_fallback(self) -> None: events: list[tuple[str, dict[str, object]]] = [] runner = _RunnerStub() @@ -138,6 +151,25 @@ def test_wait_for_service_listener_uses_port_probe_fallback(self) -> None: self.assertTrue(ok) self.assertEqual(events[0][0], "service.bind.port_fallback") self.assertEqual(events[0][1]["service"], "Main Backend") + self.assertEqual(runner.wait_for_port_calls, [(8000, 2.0)]) + + def test_wait_for_service_listener_never_falls_back_after_launch_pid_exits(self) -> None: + runner = _RunnerStub() + runner.running_result = False + runner.wait_for_port_result = True + runner.process_tree_probe_supported = False + runtime = SimpleNamespace( + process_runner=runner, + config=SimpleNamespace(runtime_truth_mode="auto"), + _listener_probe_supported=True, + _service_listener_timeout=lambda: 300.0, + _emit=lambda *_args, **_kwargs: None, + ) + + ok = wait_for_service_listener(runtime, 2304354, 8251, service_name="voice-runtime") + + self.assertFalse(ok) + self.assertEqual(runner.wait_for_port_calls, []) def test_wait_for_service_listener_rejects_generic_port_when_process_tree_probe_is_available(self) -> None: events: list[tuple[str, dict[str, object]]] = [] @@ -236,8 +268,7 @@ def wait_for_pid_port( self.wait_calls += 1 if self.wait_calls == 2: self.log_path.write_text( - "INFO: Waiting for application startup.\n" - "INFO: Application startup complete.\n", + "INFO: Waiting for application startup.\nINFO: Application startup complete.\n", encoding="utf-8", ) return True @@ -309,9 +340,7 @@ def wait_for_pid_port( ) self.assertEqual(detected, 8000) - self.assertTrue( - any(event == "service.startup.progress.listener_accepted" for event, _payload in events) - ) + self.assertTrue(any(event == "service.startup.progress.listener_accepted" for event, _payload in events)) self.assertFalse(any(event == "service.startup.progress.timeout" for event, _payload in events)) self.assertFalse(any(event == "service.bind.actual.discovered" for event, _payload in events)) diff --git a/tests/python/startup/test_service_attach_execution.py b/tests/python/startup/test_service_attach_execution.py index 6d5743117..c737c976e 100644 --- a/tests/python/startup/test_service_attach_execution.py +++ b/tests/python/startup/test_service_attach_execution.py @@ -218,6 +218,56 @@ def emit(event: str, **payload: object) -> None: self.assertIsNone(error) self.assertEqual(pid, 62001) + def test_process_spawn_error_is_structured_without_listener_wait(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + runtime = SimpleNamespace( + _conflict_remaining={}, + _emit=lambda event, **payload: events.append((event, payload)), + _service_start_command_resolved=lambda **_kwargs: (["missing-backend"], "configured"), + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace( + start_background=lambda *_args, **_kwargs: (_ for _ in ()).throw( + FileNotFoundError("missing interpreter") + ) + ), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="Main", + project_root=Path("/repo"), + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=Path("/repo/backend"), + frontend_cwd=Path("/repo/frontend"), + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=Path("/repo/backend"), + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + success, error, pid = runner.start_backend(8000) + + self.assertFalse(success) + self.assertIn("process spawn failed for missing-backend", error) + self.assertIsNone(pid) + failure_events = [payload for event, payload in events if event == "service.failure"] + self.assertEqual(failure_events[0]["failure_class"], "process_spawn_failed") + def test_runner_builds_layered_descriptors_and_preserves_additional_urls(self) -> None: project_root = Path("/tmp/envctl-project") events: list[tuple[str, dict[str, object]]] = [] @@ -440,7 +490,7 @@ def test_additional_listener_truth_failure_emits_failure_event(self) -> None: with self.assertRaisesRegex( RuntimeError, - r"voice-runtime listener not detected for Main on port 8010 \(process exited\)", + r"voice-runtime process exited before opening its listener for Main on port 8010 \(process exited\)", ): runner.detect_additional_actual("voice-runtime", pid=4321, requested=8010) @@ -450,7 +500,7 @@ def test_additional_listener_truth_failure_emits_failure_event(self) -> None: { "project": "Main", "service": "voice-runtime", - "failure_class": "listener_not_detected", + "failure_class": "process_exited", "requested_port": 8010, "detail": "process exited", }, @@ -458,6 +508,61 @@ def test_additional_listener_truth_failure_emits_failure_event(self) -> None: events, ) + def test_missing_nested_executable_is_reported_as_dependency_failure(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + detail = ( + "process 2304354 exited; log_path: /tmp/voice-runtime.txt; " + "log: scripts/envctl/start-voice-runtime.sh: line 20: exec: python: not found" + ) + runtime = SimpleNamespace( + services=SimpleNamespace(), + _conflict_remaining={}, + _emit=lambda event, **payload: events.append((event, payload)), + _detect_service_actual_port=lambda **_kwargs: None, + _listener_truth_enforced=lambda: True, + _service_listener_failure_detail=lambda **_kwargs: detail, + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="refactoring-elevenlabs-agent-runtime-automation-and-pipecat-strangler-3", + project_root=Path("/repo"), + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=Path("/repo/backend"), + frontend_cwd=Path("/repo/frontend"), + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "voice-runtime": PreparedServiceLaunch( + service_name="voice-runtime", + cwd=Path("/repo/voice-runtime"), + log_path="/tmp/voice-runtime.txt", + requested_port=8251, + env={"PORT": "8251"}, + command_source="configured", + ) + }, + selected_service_types={"voice-runtime"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + with self.assertRaisesRegex( + RuntimeError, + "voice-runtime is missing a required executable or module", + ): + runner.detect_additional_actual("voice-runtime", pid=2304354, requested=8251) + + failure_events = [payload for event, payload in events if event == "service.failure"] + self.assertEqual(failure_events[0]["failure_class"], "dependency_missing") + self.assertEqual(failure_events[0]["detail"], detail) + def test_core_only_runner_uses_legacy_service_manager_when_generic_attach_is_missing(self) -> None: attach_kwargs: dict[str, object] = {} runtime = SimpleNamespace( From cd20552eed248e5ce1be7039ab800a25cd294ce9 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:26:49 +0300 Subject: [PATCH 12/26] Reconcile same-head previews and honor readiness --- docs/reference/commands.md | 5 + python/envctl_engine/pr_preview_controller.py | 41 ++++++- .../runtime/test_pr_preview_controller.py | 110 ++++++++++++++++++ 3 files changed, 154 insertions(+), 2 deletions(-) diff --git a/docs/reference/commands.md b/docs/reference/commands.md index bd311a630..76b3cf18c 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -253,11 +253,16 @@ envctl pr-preview-controller --command start --pr-number 287 Behavior: - starts or reconciles a preview when the configured label is added +- treats an explicit `--command start` as a forced reconciliation: an existing + same-head runtime is stopped before the import/start path runs again, so + changed CI environment or secret inputs cannot be hidden by Git identity - redeploys labeled PRs when GitHub sends a new-commit event - loads a tracked `.envctl` from the exact PR head when present; otherwise it falls back to the controller-generated configuration - publishes explicitly listed `ENVCTL_PREVIEW_PUBLIC_SERVICES` on deterministic TLS hosts and projects matching HTTP/WebSocket source URLs into launch env +- rejects a routed service whose JSON health response explicitly reports + `"ready": false`, even when the endpoint itself returns HTTP 2xx - creates `ENVCTL_PREVIEW_GENERATED_SECRETS` ephemerally for each preview start, so shared service credentials do not need to be stored in GitHub - also redeploys during scheduled reconciliation if the saved preview head is stale diff --git a/python/envctl_engine/pr_preview_controller.py b/python/envctl_engine/pr_preview_controller.py index 95df9c72e..e86fe7146 100644 --- a/python/envctl_engine/pr_preview_controller.py +++ b/python/envctl_engine/pr_preview_controller.py @@ -99,6 +99,7 @@ PUBLIC_ROUTE_HEALTH_DEADLINE_SECONDS = 30.0 PUBLIC_ROUTE_HEALTH_INTERVAL_SECONDS = 2.0 PUBLIC_ROUTE_REQUEST_TIMEOUT_SECONDS = 5.0 +PUBLIC_ROUTE_RESPONSE_BODY_LIMIT_BYTES = 64 * 1024 PLAN_AGENT_BOOLEAN_CONFIG_KEYS = { "ENVCTL_PLAN_AGENT_TERMINALS_ENABLE", "CMUX", @@ -652,6 +653,8 @@ def probe_public_route_url(url: str) -> str | None: timeout=PUBLIC_ROUTE_REQUEST_TIMEOUT_SECONDS, ) as response: status = int(response.status) + read = getattr(response, "read", None) + body = read(PUBLIC_ROUTE_RESPONSE_BODY_LIMIT_BYTES) if callable(read) else b"" except urllib.error.HTTPError as exc: return f"{url} returned HTTP {exc.code}" except urllib.error.URLError as exc: @@ -661,10 +664,35 @@ def probe_public_route_url(url: str) -> str | None: except OSError as exc: return f"{url} failed: {exc}" if 200 <= status < 400: + readiness_failure = semantic_readiness_failure(url, body) + if readiness_failure: + return readiness_failure return None return f"{url} returned HTTP {status}" +def semantic_readiness_failure(url: str, body: object) -> str | None: + if not isinstance(body, (bytes, bytearray)) or not body: + return None + try: + payload = json.loads(bytes(body).decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return None + if not isinstance(payload, dict) or "ready" not in payload: + return None + if payload["ready"] is True: + return None + + reason = payload.get("reason") + safe_reason = "" + if isinstance(reason, str): + safe_reason = re.sub(r"[^A-Za-z0-9_.:-]+", "_", reason).strip("_")[:80] + suffix = f" (reason={safe_reason})" if safe_reason else "" + if payload["ready"] is False: + return f"{url} reported ready=false{suffix}" + return f"{url} returned a non-boolean readiness value{suffix}" + + def public_route_health(endpoints: dict[str, Any]) -> PublicRouteHealth: routed_services = tuple( name @@ -1623,7 +1651,12 @@ def run_command(self, command: str, pr_number: int | None) -> int: raise SystemExit(f"command {command!r} requires --pr-number") pr = self.get_pr(pr_number) if command == "start": - return self.start(pr, reason="manual workflow dispatch", refresh_ttl=True) + return self.start( + pr, + reason="manual workflow dispatch", + refresh_ttl=True, + force_restart=True, + ) if command == "stop": return self.stop(pr, reason="manual workflow dispatch", comment=True) if command == "delete": @@ -1742,13 +1775,17 @@ def start( *, reason: str, refresh_ttl: bool = False, + force_restart: bool = False, ) -> int: self.validate_same_repo(pr) label_added_at = self.label_active_since(pr.number) or utc_now() if refresh_ttl: label_added_at = utc_now() existing_state = self.load_state(pr.number) - if self.current_head_preview_is_running(pr, existing_state): + if not force_restart and self.current_head_preview_is_running( + pr, + existing_state, + ): assert existing_state is not None updated_state = self.refresh_running_preview_state( existing_state, diff --git a/tests/python/runtime/test_pr_preview_controller.py b/tests/python/runtime/test_pr_preview_controller.py index b11a61682..80cba7916 100644 --- a/tests/python/runtime/test_pr_preview_controller.py +++ b/tests/python/runtime/test_pr_preview_controller.py @@ -11,6 +11,9 @@ import envctl_engine.pr_preview_controller as preview_controller +REAL_PROBE_PUBLIC_ROUTE_URL = preview_controller.probe_public_route_url + + def load_controller(): return preview_controller @@ -1308,6 +1311,113 @@ def test_manual_start_refreshes_preview_ttl_when_label_is_old(tmp_path): assert "2000-01-01" not in runner.comments[-1] +def test_manual_start_reconciles_same_head_instead_of_reusing_stale_runtime( + tmp_path, + monkeypatch, +): + controller = load_controller() + root = tmp_path / "control" / "trees" / "imported" / "feature-demo" + root.mkdir(parents=True) + runner = FakeRunner( + controller, + projects={"projects": [{"name": "feature/demo", "root": str(root), "running": True}]}, + endpoints={"frontend": {"port": 9000}, "backend": {"port": 8000}}, + root_to_branch={str(root): "feature/demo"}, + ) + instance = controller.PreviewController(make_config(controller, tmp_path), runner) + pr = controller.pr_from_event( + pr_payload( + action="labeled", + labels=["deploy-app"], + event_label="deploy-app", + ) + ) + instance.save_state( + controller.PreviewState( + pr_number=pr.number, + label="deploy-app", + project="feature/demo", + root=str(root), + head_ref=pr.head_ref, + head_sha=pr.head_sha, + status="running", + label_added_at="2026-06-14T00:00:00Z", + started_at="2026-06-14T00:00:00Z", + expires_at="2026-06-14T00:45:00Z", + updated_at="2026-06-14T00:00:00Z", + endpoints={"frontend": {"public_url": "https://preview.example"}}, + deployment_id="12345", + ) + ) + monkeypatch.setattr(instance, "get_pr", lambda _number: pr) + + exit_code = instance.run_command("start", pr.number) + + assert exit_code == 0 + sequence = [ + call["argv"][:2] + for call in runner.calls + if call["argv"][:2] in (["envctl", "stop"], ["envctl", "import"], ["envctl", "start"]) + ] + assert sequence == [ + ["envctl", "stop"], + ["envctl", "import"], + ["envctl", "stop"], + ["envctl", "start"], + ] + state = instance.load_state(pr.number) + assert state is not None + assert state.status == "running" + assert state.head_sha == pr.head_sha + assert state.started_at != "2026-06-14T00:00:00Z" + assert "already running for this head" not in runner.comments[-1] + + +@pytest.mark.parametrize( + ("payload", "expected"), + [ + ({"ready": True}, None), + ( + {"ready": False, "reason": "missing_provider_credentials"}, + "reported ready=false (reason=missing_provider_credentials)", + ), + ( + {"ready": "true", "reason": "invalid/value with spaces"}, + "returned a non-boolean readiness value (reason=invalid_value_with_spaces)", + ), + ], +) +def test_public_route_probe_honors_json_readiness_semantics( + monkeypatch, + payload, + expected, +): + controller = load_controller() + + class Response: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, limit): + body = json.dumps(payload).encode("utf-8") + assert len(body) <= limit + return body + + monkeypatch.setattr(controller.urllib.request, "urlopen", lambda *_args, **_kwargs: Response()) + + result = REAL_PROBE_PUBLIC_ROUTE_URL("https://preview.example/readyz") + + if expected is None: + assert result is None + else: + assert expected in str(result) + + def test_push_event_refreshes_preview_ttl_for_labeled_branch(tmp_path): controller = load_controller() root = tmp_path / "control" / "trees" / "imported" / "feature-demo" From 4071b18694a2c5942d2f13d4feb5a9e8ce8c5f8e Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 22:27:38 +0300 Subject: [PATCH 13/26] fix review findings in runtime lifecycle hardening --- .../runtime/engine_runtime_startup_support.py | 18 ++++-- .../runtime/lifecycle_cleanup_orchestrator.py | 8 ++- .../shared/process_termination.py | 2 +- python/envctl_engine/state/repository.py | 17 +++++- .../test_engine_runtime_startup_support.py | 39 +++++++++++++ ...t_lifecycle_cleanup_spinner_integration.py | 22 ++++++++ .../python/shared/test_process_termination.py | 4 +- .../state/test_state_repository_contract.py | 55 +++++++++++++++++++ 8 files changed, 156 insertions(+), 9 deletions(-) diff --git a/python/envctl_engine/runtime/engine_runtime_startup_support.py b/python/envctl_engine/runtime/engine_runtime_startup_support.py index e118ce4aa..0cddda46b 100644 --- a/python/envctl_engine/runtime/engine_runtime_startup_support.py +++ b/python/envctl_engine/runtime/engine_runtime_startup_support.py @@ -25,7 +25,12 @@ def effective_start_mode(runtime: Any, route: Route) -> str: - if route.mode == "main" and runtime._setup_worktree_requested(route): + setup_worktree_requested = getattr(runtime, "_setup_worktree_requested", None) + if ( + route.mode == "main" + and callable(setup_worktree_requested) + and setup_worktree_requested(route) + ): return "trees" return route.mode @@ -289,6 +294,7 @@ def _port_reservation_enabled( ) -> bool: if route is None: return True + runtime_mode = effective_start_mode(runtime, route) dependency_ids = { definition.id @@ -301,7 +307,10 @@ def _port_reservation_enabled( enabled = getattr(runtime, "_requirement_enabled", None) if not callable(enabled): return True - return any(bool(enabled(dependency_id, mode=route.mode, route=route)) for dependency_id in dependency_ids) + return any( + bool(enabled(dependency_id, mode=runtime_mode, route=route)) + for dependency_id in dependency_ids + ) return selected_app_services is None or service_name in selected_app_services @@ -314,10 +323,11 @@ def _selected_app_services_for_port_reservation( ) -> set[str] | None: if route is None: return None + runtime_mode = effective_start_mode(runtime, route) try: configured_service_types = configured_service_types_for_mode( runtime, - route.mode, + runtime_mode, Path(context.root), ) except AttributeError: @@ -330,7 +340,7 @@ def _selected_app_services_for_port_reservation( for service in getattr(getattr(runtime, "config", None), "additional_services", ()) if additional_service_enabled_for_context( service, - mode=route.mode, + mode=runtime_mode, project_root=Path(context.root), ) ) diff --git a/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py b/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py index 17cdac4a6..122df92c2 100644 --- a/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py +++ b/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py @@ -441,7 +441,13 @@ def clear_runtime_state(self, *, command: str, aggressive: bool = False, route: rt._emit("cleanup.stop", aggressive=aggressive) # type: ignore[attr-defined] state_repository = self._state_repository(rt) load_all = getattr(state_repository, "load_all", None) - mode_filter = route.mode if command == "stop-all" and route is not None else None + mode_filter = None + if ( + command == "stop-all" + and route is not None + and rt._state_lookup_strict_mode_match(route) # type: ignore[attr-defined] + ): + mode_filter = route.mode if callable(load_all): states = [state for state in load_all(mode=mode_filter) if isinstance(state, RunState)] else: diff --git a/python/envctl_engine/shared/process_termination.py b/python/envctl_engine/shared/process_termination.py index 646e2d561..3ff0d9bbd 100644 --- a/python/envctl_engine/shared/process_termination.py +++ b/python/envctl_engine/shared/process_termination.py @@ -41,7 +41,7 @@ def terminate_pid( if callable(getattr(process_runner, "_process_group_is_running", None)): return False if _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity): - return False + return True else: if not terminated: # Killing only the recorded leader after group cleanup failed diff --git a/python/envctl_engine/state/repository.py b/python/envctl_engine/state/repository.py index c1ba28df2..408e863a1 100644 --- a/python/envctl_engine/state/repository.py +++ b/python/envctl_engine/state/repository.py @@ -401,13 +401,28 @@ def load_for_mode(expected_mode: str | None) -> RunState | None: candidates, allowed_root=allowed_root, ) + selected_project_keys = normalized_project_names(selected_projects) + if selected_project_keys: + valid_candidates = [ + indexed + for indexed in valid_candidates + if selected_project_keys.intersection( + normalized_project_names( + self._project_names_from_state(indexed.state) + ) + ) + ] if not valid_candidates: return None newest_mode = max( valid_candidates, key=lambda indexed: indexed.candidate.activation_sequence, ).candidate.mode - candidates = [candidate for candidate in candidates if candidate.mode == newest_mode] + candidates = [ + indexed.candidate + for indexed in valid_candidates + if indexed.candidate.mode == newest_mode + ] indexed_candidate = self._load_index_candidates( candidates, allowed_root=allowed_root, diff --git a/tests/python/runtime/test_engine_runtime_startup_support.py b/tests/python/runtime/test_engine_runtime_startup_support.py index 988d3618a..3c808f71a 100644 --- a/tests/python/runtime/test_engine_runtime_startup_support.py +++ b/tests/python/runtime/test_engine_runtime_startup_support.py @@ -697,6 +697,45 @@ def test_disabled_requirements_do_not_reserve_dependency_ports(self) -> None: self.assertEqual(reserve_calls, [(8000, "Main:backend")]) + def test_setup_worktree_reserves_dependencies_for_effective_trees_mode(self) -> None: + reserve_calls: list[tuple[int, str]] = [] + enabled_modes: list[str] = [] + runtime = SimpleNamespace( + port_planner=SimpleNamespace( + reserve_next=lambda assigned, owner: reserve_calls.append( + (assigned, owner) + ) + or assigned, + max_port=65000, + availability_mode="lock", + update_final_port=lambda plan, reserved, source: None, + ), + _emit=lambda *_args, **_kwargs: None, + _lock_inventory=lambda: [], + _setup_worktree_requested=lambda _route: True, + _requirement_enabled=lambda _dependency, *, mode, route: ( + enabled_modes.append(mode) or mode == "trees" + ), + ) + route = Route( + command="start", + mode="main", + flags={"setup_worktree": True, "runtime_scope": "entire-system"}, + ) + + reserve_project_ports( + runtime, + ProjectContext( + name="FeatureA", + root=Path("/repo/trees/FeatureA"), + ports={"db": PortPlan("FeatureA", 5432, 5432, 5432, "fixed")}, + ), + route=route, + ) + + self.assertEqual(enabled_modes, ["trees"]) + self.assertEqual(reserve_calls, [(5432, "FeatureA:db")]) + def test_port_reservations_match_the_selected_runtime_surface(self) -> None: voice = SimpleNamespace( name="voice", diff --git a/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py b/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py index cff6c5e24..5e840ce43 100644 --- a/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py +++ b/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py @@ -257,6 +257,25 @@ def test_stop_all_terminates_every_independent_active_run_once(self) -> None: self.assertEqual(terminated, ["run-a", "run-b"]) self.assertEqual(runtime.state_repository.purge_calls, [False]) + def test_plain_stop_all_terminates_active_runs_across_modes(self) -> None: + runtime = _RuntimeStub() + runtime.state_repository.active_states = [ + RunState(run_id="run-main", mode="main"), + RunState(run_id="run-tree", mode="trees"), + ] + terminated: list[str] = [] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: terminated.append(state.run_id) or set() + ) + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route(command="stop-all", mode="main") + ) + + self.assertEqual(code, 0) + self.assertEqual(terminated, ["run-main", "run-tree"]) + self.assertEqual(runtime.state_repository.active_states, []) + def test_stop_all_trees_preserves_main_registry_alias_and_port_lock(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -303,6 +322,7 @@ def test_stop_all_trees_preserves_main_registry_alias_and_port_lock(self) -> Non main_port = planner.reserve_next(8111, owner="Main:backend") runtime = _RuntimeStub() runtime.state_repository = repository # type: ignore[assignment] + runtime._state_lookup_strict_mode_match = lambda _route: True # type: ignore[method-assign] runtime.port_planner = planner terminated: list[str] = [] runtime._terminate_services_from_state = ( # type: ignore[method-assign] @@ -363,6 +383,7 @@ def test_mode_scoped_stop_all_ignores_tampered_other_mode_and_preserves_it(self) runtime = _RuntimeStub() runtime.state_repository = repository # type: ignore[assignment] + runtime._state_lookup_strict_mode_match = lambda _route: True # type: ignore[method-assign] termination_calls: list[str] = [] runtime._terminate_services_from_state = ( # type: ignore[method-assign] lambda state, **_kwargs: termination_calls.append(state.run_id) or set() @@ -715,6 +736,7 @@ def test_stop_all_failure_preserves_failed_run_and_never_purges_registry(self) - lambda state, **_kwargs: {"Main Backend"} if state.run_id == "run-failed" else set() ) + runtime._state_lookup_strict_mode_match = lambda _route: True # type: ignore[method-assign] code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) self.assertEqual(code, 1) diff --git a/tests/python/shared/test_process_termination.py b/tests/python/shared/test_process_termination.py index dcd09238a..190c4a477 100644 --- a/tests/python/shared/test_process_termination.py +++ b/tests/python/shared/test_process_termination.py @@ -105,7 +105,7 @@ def test_identity_replacement_after_failed_group_cleanup_is_not_reported_as_succ self.assertFalse(result) self.assertEqual(sent, []) - def test_group_cleanup_exception_is_not_reported_as_success_after_leader_replacement(self) -> None: + def test_group_cleanup_exception_is_success_after_leader_replacement(self) -> None: identities = iter(("original", "replacement")) sent: list[signal.Signals] = [] runner = SimpleNamespace( @@ -122,7 +122,7 @@ def test_group_cleanup_exception_is_not_reported_as_success_after_leader_replace ): result = terminate_pid(987658, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) - self.assertFalse(result) + self.assertTrue(result) self.assertEqual(sent, []) diff --git a/tests/python/state/test_state_repository_contract.py b/tests/python/state/test_state_repository_contract.py index 01dc3ca66..396d70dbf 100644 --- a/tests/python/state/test_state_repository_contract.py +++ b/tests/python/state/test_state_repository_contract.py @@ -1724,6 +1724,61 @@ def test_generic_lookup_skips_corrupt_newest_mode_and_returns_newest_valid_mode( self.assertEqual((loaded.run_id, loaded.mode), ("run-main", "main")) self.assertEqual(load_state(str(runtime_root / "run_state.json")).run_id, "run-main") + def test_non_strict_project_lookup_filters_projects_before_newest_mode(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_resume_state( + state=RunState( + run_id="run-tree", + mode="trees", + services={ + "FeatureA Backend": ServiceRecord( + name="FeatureA Backend", + type="backend", + cwd="/tmp/FeatureA/backend", + project="FeatureA", + ) + }, + metadata={"project_names": ["FeatureA"]}, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + repo.save_resume_state( + state=RunState( + run_id="run-main", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + ) + }, + metadata={"project_names": ["Main"]}, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + loaded = repo.load_latest( + mode="main", + strict_mode_match=False, + project_names=["FeatureA"], + ) + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual((loaded.run_id, loaded.mode), ("run-tree", "trees")) + def test_resuming_older_mode_reactivates_it_without_changing_project_ownership_precedence(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: runtime_dir = Path(tmpdir) / "runtime" From 6913f7d3f8a4fef44e79e051281f0a708ec6aef6 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 22:29:35 +0300 Subject: [PATCH 14/26] fix config-only service target resolution --- .../actions/action_target_support.py | 20 +++++++++++++++++-- ...est_action_command_orchestrator_targets.py | 9 +++++++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/python/envctl_engine/actions/action_target_support.py b/python/envctl_engine/actions/action_target_support.py index 010649384..033660ba0 100644 --- a/python/envctl_engine/actions/action_target_support.py +++ b/python/envctl_engine/actions/action_target_support.py @@ -163,6 +163,16 @@ def _resolve_projects_for_services( return _deduplicate_project_names(resolved), tuple(unresolved) +def _is_configured_test_service_selector(runtime: Any, route: Route, selector: str) -> bool: + if route.command not in {"test", "test-focused"}: + return False + if selector in {"backend", "frontend"}: + return True + config = getattr(runtime, "config", None) + app_service_by_name = getattr(config, "app_service_by_name", None) + return callable(app_service_by_name) and app_service_by_name(selector) is not None + + def _discover_action_candidates(runtime: Any, route: Route, *, trees_only: bool) -> list[object]: mode = "trees" if trees_only else route.mode candidates = list(runtime.discover_projects(mode=mode)) @@ -244,13 +254,17 @@ def _request_from_route(self) -> _ActionTargetRequest: project_selectors = {name.lower() for name in self.route.projects} project_selectors.update(self.runtime.selectors_from_passthrough(self.route.passthrough_args)) services = self.route.flags.get("services") - has_service_selectors = isinstance(services, list) and bool(services) unresolved_service_selectors: tuple[str, ...] = () if isinstance(services, list): service_projects, unresolved_service_selectors = _resolve_projects_for_services( self.runtime, services, ) + unresolved_service_selectors = tuple( + selector + for selector in unresolved_service_selectors + if not _is_configured_test_service_selector(self.runtime, self.route, selector) + ) project_selectors.update(project.lower() for project in service_projects) run_all = bool(self.route.flags.get("all")) untested_selected = bool(self.route.flags.get("untested")) @@ -258,7 +272,9 @@ def _request_from_route(self) -> _ActionTargetRequest: run_all=run_all, untested_selected=untested_selected, project_selectors=frozenset(project_selectors), - explicit_targeting=bool(project_selectors or has_service_selectors or run_all or untested_selected), + explicit_targeting=bool( + project_selectors or unresolved_service_selectors or run_all or untested_selected + ), unresolved_service_selectors=unresolved_service_selectors, ) diff --git a/tests/python/actions/test_action_command_orchestrator_targets.py b/tests/python/actions/test_action_command_orchestrator_targets.py index 0eef8bb8e..176173d5c 100644 --- a/tests/python/actions/test_action_command_orchestrator_targets.py +++ b/tests/python/actions/test_action_command_orchestrator_targets.py @@ -333,7 +333,7 @@ def test_test_summary_artifact_methods_delegate_to_summary_owner(self) -> None: summary_metadata={"Main": {"status": "passed"}}, ) - def test_build_test_execution_specs_uses_additional_service_test_command(self) -> None: + def test_config_only_service_selector_resolves_current_target_and_builds_test_spec(self) -> None: runtime = _RuntimeStub() runtime.config = SimpleNamespace( base_dir=Path("/tmp/repo"), @@ -351,10 +351,13 @@ def test_build_test_execution_specs_uses_additional_service_test_command(self) - orchestrator = ActionCommandOrchestrator(runtime) route = Route(command="test", mode="main", flags={"services": ["voice-runtime"]}) target = SimpleNamespace(name="Main", root=Path("/tmp/repo")) + orchestrator._resolve_current_worktree_target = lambda **_kwargs: target + + targets, error = orchestrator.resolve_targets(route, trees_only=False) specs = orchestrator._build_test_execution_specs( route=route, - targets=[target], + targets=targets, target_contexts=[TargetContext(project_name="Main", project_root=Path("/tmp/repo"), target_obj=target)], include_backend=False, include_frontend=False, @@ -362,6 +365,8 @@ def test_build_test_execution_specs_uses_additional_service_test_command(self) - untested=False, ) + self.assertIsNone(error) + self.assertEqual(targets, [target]) self.assertEqual(len(specs), 1) self.assertEqual(specs[0].spec.command, ["python", "-m", "pytest", "voice-runtime/tests"]) self.assertEqual(specs[0].spec.cwd, Path("/tmp/repo/voice-runtime")) From 7d807874c59a0e2c67bfea21b4527d1cea6e5fe6 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 22:32:11 +0300 Subject: [PATCH 15/26] fix Docker runtime review findings --- .../actions/action_git_state_support.py | 20 ++++-- .../runtime/docker_service_runtime.py | 39 ++++++++++- .../runtime/engine_runtime_commands.py | 6 +- .../runtime/lifecycle_dependency_stop.py | 5 +- .../runtime/lifecycle_service_termination.py | 5 +- .../startup/resume_restore_execution.py | 9 +++ .../actions/test_action_git_state_support.py | 14 +++- .../runtime/test_docker_service_runtime.py | 70 ++++++++++++++++++- .../runtime/test_engine_runtime_commands.py | 11 ++- .../runtime/test_lifecycle_dependency_stop.py | 34 +++++++++ .../startup/test_resume_restore_policy.py | 35 +++++++++- 11 files changed, 228 insertions(+), 20 deletions(-) diff --git a/python/envctl_engine/actions/action_git_state_support.py b/python/envctl_engine/actions/action_git_state_support.py index 2e189b67a..83fbe4738 100644 --- a/python/envctl_engine/actions/action_git_state_support.py +++ b/python/envctl_engine/actions/action_git_state_support.py @@ -71,15 +71,27 @@ def classify_dirty_porcelain(status_output: str) -> tuple[bool, bool, bool]: def detect_default_branch(git_root: Path, *, git_output: Callable[[Path, list[str]], str]) -> str: - ref = git_output(git_root, ["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]).strip() - if ref.startswith("origin/"): - return ref.split("origin/", 1)[1] + origin_default = _origin_default_branch(git_root, git_output=git_output) + if origin_default: + return origin_default for candidate in ("main", "master"): if git_output(git_root, ["rev-parse", "--verify", candidate]).strip(): return candidate return "main" +def _origin_default_branch(git_root: Path, *, git_output: Callable[[Path, list[str]], str]) -> str: + ref = git_output(git_root, ["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]).strip() + if ref.startswith("origin/"): + return ref.split("origin/", 1)[1] + remote_head = git_output(git_root, ["ls-remote", "--symref", "origin", "HEAD"]) + for line in remote_head.splitlines(): + prefix = "ref: refs/heads/" + if line.startswith(prefix) and line.rstrip().endswith("\tHEAD"): + return line[len(prefix) :].split("\t", 1)[0].strip() + return "" + + def detect_pr_base_branch(git_root: Path, *, git_output: Callable[[Path, list[str]], str]) -> str: remote_heads = git_output( git_root, @@ -103,7 +115,7 @@ def detect_pr_base_branch(git_root: Path, *, git_output: Callable[[Path, list[st for ref in (f"origin/{candidate}", candidate): if git_output(git_root, ["rev-parse", "--verify", ref]).strip(): return candidate - return "master" + return _origin_default_branch(git_root, git_output=git_output) or "master" def existing_pr_url( diff --git a/python/envctl_engine/runtime/docker_service_runtime.py b/python/envctl_engine/runtime/docker_service_runtime.py index bc14c3542..781182b66 100644 --- a/python/envctl_engine/runtime/docker_service_runtime.py +++ b/python/envctl_engine/runtime/docker_service_runtime.py @@ -256,13 +256,37 @@ def follow_logs(self, container: str, log_path: str) -> int | None: pid = getattr(process, "pid", None) return pid if isinstance(pid, int) and pid > 0 else None - def stop(self, container: str, *, remove: bool = True) -> bool: + def stop( + self, + container: str, + *, + remove: bool = True, + verify_ownership: bool = False, + ) -> bool: if not container: return True + if verify_ownership and not self._container_owned_by_runtime_scope(container): + return False command = ["docker", "rm", "--force", container] if remove else ["docker", "stop", container] result = self.process_runtime.run(command, timeout=30.0) return result.returncode == 0 or "No such container" in str(result.stderr or "") + def _container_owned_by_runtime_scope(self, container: str) -> bool: + inspected = self.process_runtime.run( + [ + "docker", + "container", + "inspect", + "--format", + f"{{{{index .Config.Labels {json.dumps(_RUNTIME_SCOPE_LABEL)}}}}}", + container, + ], + timeout=10.0, + ) + if inspected.returncode != 0: + return "No such" in str(inspected.stderr or "") + return str(inspected.stdout or "").strip() == self._runtime_scope_identity() + def container_running(self, container: str) -> bool: if not container: return False @@ -548,6 +572,7 @@ def _setting(self, service_name: str, suffix: str) -> str: def _container_port(self, service_name: str, host_port: int, *, image: str) -> int: value = self._setting(service_name, "DOCKER_PORT") if not value: + self._ensure_image_available(image) exposed = self._single_exposed_port(image) return exposed if exposed is not None else host_port try: @@ -558,6 +583,18 @@ def _container_port(self, service_name: str, host_port: int, *, image: str) -> i raise RuntimeError(f"Invalid Docker container port for {service_name}: {value!r}") return port + def _ensure_image_available(self, image: str) -> None: + if self._image_exists(image): + return + self.runtime._emit("service.container.pull", image=image) + pulled = self.process_runtime.run( + ["docker", "pull", image], + timeout=self._timeout("ENVCTL_DOCKER_PULL_TIMEOUT_SECONDS", 300.0), + ) + if pulled.returncode != 0: + detail = str(pulled.stderr or pulled.stdout or f"failed to pull Docker image {image}").strip() + raise RuntimeError(detail) + def _single_exposed_port(self, image: str) -> int | None: result = self.process_runtime.run( ["docker", "image", "inspect", "--format", "{{json .Config.ExposedPorts}}", image], diff --git a/python/envctl_engine/runtime/engine_runtime_commands.py b/python/envctl_engine/runtime/engine_runtime_commands.py index 39c5a5c95..22843e289 100644 --- a/python/envctl_engine/runtime/engine_runtime_commands.py +++ b/python/envctl_engine/runtime/engine_runtime_commands.py @@ -106,17 +106,15 @@ def service_start_command_resolved( project_root: Path | None = None, port: int = 0, ) -> tuple[list[str], str]: - command_exists = runtime._command_exists if docker_service_uses_image_command(runtime, service_name): - def command_exists(_executable: str) -> bool: - return True + return [], "docker_image" result = resolve_service_start_command( service_name=service_name, project_root=(project_root or runtime.config.base_dir), port=port, env=runtime.env, config_raw=runtime.config.raw, - command_exists=command_exists, + command_exists=runtime._command_exists, ) return result.command, result.source diff --git a/python/envctl_engine/runtime/lifecycle_dependency_stop.py b/python/envctl_engine/runtime/lifecycle_dependency_stop.py index c18c7ba02..4f7f5ac06 100644 --- a/python/envctl_engine/runtime/lifecycle_dependency_stop.py +++ b/python/envctl_engine/runtime/lifecycle_dependency_stop.py @@ -118,7 +118,10 @@ def stop_requirement_component_containers(runtime: Any, component: Mapping[str, if removed.returncode != 0: detail = removed.stderr or removed.stdout or "failed stopping Supabase containers" raise RuntimeError(str(detail).strip()) - return + return + # Native Supabase DB startup records the concrete container name and + # does not attach a Compose project label. Fall through to direct + # removal when the saved component is not a Compose stack. removed = process_runner.run(["docker", "rm", "--force", container_name], timeout=30.0) if removed.returncode != 0 and "No such container" not in str(removed.stderr or ""): raise RuntimeError(str(removed.stderr or removed.stdout or f"failed stopping {container_name}").strip()) diff --git a/python/envctl_engine/runtime/lifecycle_service_termination.py b/python/envctl_engine/runtime/lifecycle_service_termination.py index 88ca18af7..e468e6cbc 100644 --- a/python/envctl_engine/runtime/lifecycle_service_termination.py +++ b/python/envctl_engine/runtime/lifecycle_service_termination.py @@ -51,7 +51,10 @@ def terminate_service_record(runtime: Any, service: object, *, aggressive: bool, container = str( getattr(service, "container_id", "") or getattr(service, "container_name", "") or "" ).strip() - stopped = DockerServiceRuntime(runtime, runtime.process_runner).stop(container) + stopped = DockerServiceRuntime(runtime, runtime.process_runner).stop( + container, + verify_ownership=verify_ownership, + ) pid = getattr(service, "pid", None) if isinstance(pid, int) and pid > 0: try: diff --git a/python/envctl_engine/startup/resume_restore_execution.py b/python/envctl_engine/startup/resume_restore_execution.py index 6367c0661..d5be89152 100644 --- a/python/envctl_engine/startup/resume_restore_execution.py +++ b/python/envctl_engine/startup/resume_restore_execution.py @@ -7,6 +7,7 @@ from typing import Callable, Mapping from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.docker_service_runtime import state_uses_docker_services from envctl_engine.state.models import RunState from envctl_engine.startup.resume_restore_project import ResumeProjectRestoreRunner, _round_ms from envctl_engine.startup.resume_restore_results import apply_restore_result @@ -113,6 +114,7 @@ def _execute_restore_missing( use_single_spinner = use_spinner and not use_project_spinner_group suppress_timing_output = use_project_spinner_group or use_single_spinner route_for_startup = _route_for_resume_restore(route, state=state, suppress_timing_output=suppress_timing_output) + _apply_restore_runtime_mode(rt, route_for_startup) if use_single_spinner: rt._emit( # type: ignore[attr-defined] "ui.spinner.lifecycle", @@ -240,6 +242,8 @@ def _route_for_resume_restore( **route_for_startup.flags, "_resume_restore": True, } + if state_uses_docker_services(state): + flags["docker"] = True if suppress_timing_output: flags["_suppress_timing_print"] = True return Route( @@ -252,6 +256,11 @@ def _route_for_resume_restore( ) +def _apply_restore_runtime_mode(runtime: object, route: Route) -> None: + if bool(route.flags.get("docker")): + getattr(runtime, "env")["DOCKER_MODE"] = "true" + + def _run_restore_jobs( *, sorted_projects: list[str], diff --git a/tests/python/actions/test_action_git_state_support.py b/tests/python/actions/test_action_git_state_support.py index be6a7c674..3afca2459 100644 --- a/tests/python/actions/test_action_git_state_support.py +++ b/tests/python/actions/test_action_git_state_support.py @@ -43,12 +43,22 @@ def local_main(_git_root: Path, args: list[str]) -> str: self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=local_main), "main") def test_detect_pr_base_branch_prefers_remote_or_local_dev_then_main_then_master(self) -> None: - def resolve(existing_refs: set[str], remote_heads: set[str] | None = None) -> str: + def resolve( + existing_refs: set[str], + remote_heads: set[str] | None = None, + *, + origin_default: str = "", + symbolic_default: bool = True, + ) -> str: def fake_git_output(_git_root: Path, args: list[str]) -> str: if args[:2] == ["ls-remote", "--heads"]: return "".join( f"abc123\trefs/heads/{branch}\n" for branch in sorted(remote_heads or set()) ) + if args == ["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]: + return f"origin/{origin_default}\n" if origin_default and symbolic_default else "" + if args == ["ls-remote", "--symref", "origin", "HEAD"]: + return f"ref: refs/heads/{origin_default}\tHEAD\nabc123\tHEAD\n" if origin_default else "" self.assertEqual(args[:2], ["rev-parse", "--verify"]) return "abc123\n" if args[2] in existing_refs else "" @@ -63,6 +73,8 @@ def fake_git_output(_git_root: Path, args: list[str]) -> str: self.assertEqual(resolve(set()), "master") self.assertEqual(resolve({"origin/main"}, {"dev", "main"}), "dev") self.assertEqual(resolve({"dev"}, {"main"}), "dev") + self.assertEqual(resolve(set(), origin_default="develop"), "develop") + self.assertEqual(resolve(set(), origin_default="trunk", symbolic_default=False), "trunk") def test_existing_pr_url_uses_gh_pr_list_for_open_branch(self) -> None: calls: list[tuple[list[str], str]] = [] diff --git a/tests/python/runtime/test_docker_service_runtime.py b/tests/python/runtime/test_docker_service_runtime.py index 0e01dea3d..0328c8fdd 100644 --- a/tests/python/runtime/test_docker_service_runtime.py +++ b/tests/python/runtime/test_docker_service_runtime.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from pathlib import Path from types import SimpleNamespace @@ -23,12 +24,15 @@ def __init__(self) -> None: self.terminated: list[int] = [] self.build_env: dict[str, str] = {} self.existing_container: tuple[bool, str] | None = None + self.local_images: set[str] = set() + self.remote_exposed_ports: dict[str, dict[str, object]] = {} def run(self, command, **_kwargs): argv = [str(part) for part in command] self.calls.append(argv) if argv[:2] == ["docker", "build"]: self.build_env = dict(_kwargs.get("env", {})) + self.local_images.add(argv[argv.index("--tag") + 1]) if argv[:2] == ["docker", "run"]: env_file = Path(argv[argv.index("--env-file") + 1]) self.env_file_payload = env_file.read_text(encoding="utf-8") @@ -37,11 +41,23 @@ def run(self, command, **_kwargs): if self.existing_container is None: return SimpleNamespace(returncode=1, stdout="", stderr="No such container") running, owner = self.existing_container + format_value = argv[argv.index("--format") + 1] + if ".State.Running" not in format_value: + return SimpleNamespace(returncode=0, stdout=f"{owner}\n", stderr="") return SimpleNamespace(returncode=0, stdout=f"{str(running).lower()}|{owner}\n", stderr="") if argv[:3] == ["docker", "inspect", "--format"]: return SimpleNamespace(returncode=0 if self.running else 1, stdout="true\n" if self.running else "", stderr="") if argv[:3] == ["docker", "image", "inspect"]: - return SimpleNamespace(returncode=1, stdout="", stderr="missing") + image = argv[-1] + if image not in self.local_images: + return SimpleNamespace(returncode=1, stdout="", stderr="missing") + if "--format" in argv: + payload = self.remote_exposed_ports.get(image, {}) + return SimpleNamespace(returncode=0, stdout=json.dumps(payload) + "\n", stderr="") + return SimpleNamespace(returncode=0, stdout="[]\n", stderr="") + if argv[:2] == ["docker", "pull"]: + self.local_images.add(argv[-1]) + return SimpleNamespace(returncode=0, stdout="pulled\n", stderr="") if argv[:3] == ["docker", "rm", "--force"]: return SimpleNamespace(returncode=0, stdout="", stderr="") if argv[:2] == ["docker", "logs"]: @@ -256,6 +272,58 @@ def test_configured_image_runs_without_build_and_preserves_browser_public_url(tm assert launch.image == "example/frontend:dev" +def test_remote_image_is_pulled_before_exposed_port_detection(tmp_path: Path) -> None: + runner = _Runner() + image = "example/frontend:remote" + runner.remote_exposed_ports[image] = {"80/tcp": {}} + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_FRONTEND_DOCKER_IMAGE": image}) + + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="frontend", + cwd=tmp_path, + command=[], + env={}, + host_port=9100, + container_port=9100, + listener_expected=True, + log_path=str(tmp_path / "frontend.log"), + ) + + pull_index = runner.calls.index(["docker", "pull", image]) + exposed_inspect_index = next( + index + for index, call in enumerate(runner.calls) + if call[:3] == ["docker", "image", "inspect"] and "--format" in call + ) + run_index = next(index for index, call in enumerate(runner.calls) if call[:2] == ["docker", "run"]) + assert pull_index < exposed_inspect_index < run_index + assert "127.0.0.1:9100:80" in runner.calls[run_index] + + +def test_verified_stop_refuses_container_from_another_runtime_scope(tmp_path: Path) -> None: + runner = _Runner() + runner.existing_container = (True, "another-runtime") + runtime, _events = _runtime(tmp_path, runner) + + stopped = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd=str(tmp_path), + runtime_kind="docker", + container_id="reused-container-id", + ), + aggressive=False, + verify_ownership=True, + ) + + assert not stopped + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + + def test_container_state_drives_health_and_stop(tmp_path: Path) -> None: runner = _Runner() runtime, events = _runtime(tmp_path, runner) diff --git a/tests/python/runtime/test_engine_runtime_commands.py b/tests/python/runtime/test_engine_runtime_commands.py index 6730825d6..a8732d123 100644 --- a/tests/python/runtime/test_engine_runtime_commands.py +++ b/tests/python/runtime/test_engine_runtime_commands.py @@ -20,20 +20,19 @@ class EngineRuntimeCommandsTests(unittest.TestCase): - def test_docker_image_command_does_not_require_container_executable_on_host(self) -> None: + def test_docker_image_command_skips_host_command_resolution(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) - (root / "frontend").mkdir() runtime = SimpleNamespace( env={ "DOCKER_MODE": "true", - "ENVCTL_FRONTEND_START_CMD": "container-only-server --port {port}", + "ENVCTL_FRONTEND_DOCKER_IMAGE": "example/frontend:dev", }, config=SimpleNamespace( base_dir=root, raw={"FRONTEND_DIR": "frontend"}, ), - _command_exists=lambda _executable: False, + _command_exists=lambda _executable: self.fail("host command lookup must be skipped"), ) command, source = service_start_command_resolved( @@ -43,8 +42,8 @@ def test_docker_image_command_does_not_require_container_executable_on_host(self port=8010, ) - self.assertEqual(command, ["container-only-server", "--port", "8010"]) - self.assertEqual(source, "configured") + self.assertEqual(command, []) + self.assertEqual(source, "docker_image") def test_docker_service_command_still_requires_executable_on_host(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: diff --git a/tests/python/runtime/test_lifecycle_dependency_stop.py b/tests/python/runtime/test_lifecycle_dependency_stop.py index 9f0ab968d..fc5e7b4b3 100644 --- a/tests/python/runtime/test_lifecycle_dependency_stop.py +++ b/tests/python/runtime/test_lifecycle_dependency_stop.py @@ -55,6 +55,40 @@ def run(command, **_kwargs): # noqa: ANN001 ) self.assertIn(["docker", "rm", "--force", "one", "two"], calls) + def test_stop_native_supabase_falls_back_to_saved_container_name(self) -> None: + calls: list[list[str]] = [] + + def run(command, **_kwargs): # noqa: ANN001 + argv = [str(part) for part in command] + calls.append(argv) + if argv[:3] == ["docker", "ps", "--all"]: + return SimpleNamespace(returncode=0, stdout="", stderr="") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + stop_requirement_component_containers( + SimpleNamespace(process_runner=SimpleNamespace(run=run)), + { + "id": "supabase", + "enabled": True, + "container_name": "envctl-feature-supabase-db-1", + }, + ) + + self.assertEqual( + calls, + [ + [ + "docker", + "ps", + "--all", + "--quiet", + "--filter", + "label=com.docker.compose.project=envctl-feature-supabase-db-1", + ], + ["docker", "rm", "--force", "envctl-feature-supabase-db-1"], + ], + ) + def test_select_dependency_components_filters_unknown_projects_dependencies_and_disabled_components(self) -> None: state = RunState( run_id="run-1", diff --git a/tests/python/startup/test_resume_restore_policy.py b/tests/python/startup/test_resume_restore_policy.py index a5fa2b510..26c49ee52 100644 --- a/tests/python/startup/test_resume_restore_policy.py +++ b/tests/python/startup/test_resume_restore_policy.py @@ -6,6 +6,10 @@ from types import SimpleNamespace from envctl_engine.runtime.command_router import Route, parse_route +from envctl_engine.startup.resume_restore_execution import ( + _apply_restore_runtime_mode, + _route_for_resume_restore, +) from envctl_engine.startup.resume_restore_policy import ( _configured_restore_service_types, _requirements_reuse_decision, @@ -16,10 +20,39 @@ apply_ports_to_context, project_root, ) -from envctl_engine.state.models import RequirementsResult, ServiceRecord +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord class ResumeRestorePolicyTests(unittest.TestCase): + def test_docker_state_forces_docker_mode_for_direct_resume_restore(self) -> None: + state = RunState( + run_id="run-docker", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + runtime_kind="docker", + container_id="stale-container", + ) + }, + ) + route = Route(command="resume", mode="main", flags={"batch": True}) + + restored_route = _route_for_resume_restore( + route, + state=state, + suppress_timing_output=False, + ) + + self.assertTrue(restored_route.flags["docker"]) + self.assertTrue(restored_route.flags["_resume_restore"]) + self.assertTrue(restored_route.flags["batch"]) + runtime = SimpleNamespace(env={}) + _apply_restore_runtime_mode(runtime, restored_route) + self.assertEqual(runtime.env["DOCKER_MODE"], "true") + def test_plan_resume_defaults_to_parallel_tree_restore(self) -> None: captured_routes: list[Route] = [] From cd08020ee9c1839047afcb5dccc325124afe0265 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 22:51:34 +0300 Subject: [PATCH 16/26] fix linked worktree action targeting --- .../actions/action_target_support.py | 18 ++++- .../action_worktree_target_resolution.py | 58 +++++++++++++- .../actions/test_action_target_support.py | 63 +++++++++++++++ .../actions/test_action_worktree_runner.py | 60 +++++++++++++++ .../python/actions/test_actions_git_parity.py | 76 +++++++++++++++++++ 5 files changed, 272 insertions(+), 3 deletions(-) diff --git a/python/envctl_engine/actions/action_target_support.py b/python/envctl_engine/actions/action_target_support.py index 033660ba0..3945a27b8 100644 --- a/python/envctl_engine/actions/action_target_support.py +++ b/python/envctl_engine/actions/action_target_support.py @@ -248,7 +248,23 @@ def _current_worktree_result( require_configured_main_root=self.route.mode == "main", require_configured_root_match=True, ) - return ([current], None) if current is not None else None + if current is not None: + return [current], None + if self._execution_root_differs_from_configured_root(): + return [], self.no_target_selected_message(self.route) + return None + + def _execution_root_differs_from_configured_root(self) -> bool: + raw_runtime = getattr(self.runtime, "raw_runtime", self.runtime) + config = getattr(raw_runtime, "config", None) + base_dir = getattr(config, "base_dir", None) + execution_root = getattr(config, "execution_root", None) + if base_dir is None or execution_root is None: + return False + try: + return Path(str(base_dir)).expanduser().resolve() != Path(str(execution_root)).expanduser().resolve() + except OSError: + return True def _request_from_route(self) -> _ActionTargetRequest: project_selectors = {name.lower() for name in self.route.projects} diff --git a/python/envctl_engine/actions/action_worktree_target_resolution.py b/python/envctl_engine/actions/action_worktree_target_resolution.py index 543309386..3a0fd6385 100644 --- a/python/envctl_engine/actions/action_worktree_target_resolution.py +++ b/python/envctl_engine/actions/action_worktree_target_resolution.py @@ -33,9 +33,16 @@ def resolve(self) -> object | None: return None matches = self._matching_candidates(cwd, repo_root=repo_root, trees_dir_name=trees_dir_name) - if len(matches) != 1: + if len(matches) == 1: + return matches[0] + if matches: return None - return matches[0] + return self._linked_worktree_candidate( + cwd, + repo_root=repo_root, + configured_root_path=configured_root_path, + raw_runtime=raw_runtime, + ) def _current_working_directory(self) -> Path: invocation_cwd = str(getattr(self.runtime, "env", {}).get("ENVCTL_INVOCATION_CWD") or "").strip() @@ -57,6 +64,9 @@ def _repo_root(self, cwd: Path, *, configured_root_path: Path | None, trees_dir_ repo_root = repo_root_from_worktree_layout(cwd, trees_dir_name) if repo_root is None: repo_root = self.main_repo_root_for_linked_worktree(cwd) + if repo_root is None: + resolver = self.git_main_repo_root_for_worktree or main_repo_root_for_worktree + repo_root = resolver(worktree_root=cwd, runtime=self.runtime, trees_dir_name=trees_dir_name) if repo_root is None or repo_root.resolve() != configured_root_path: return None return repo_root @@ -84,6 +94,35 @@ def _matching_candidates(self, cwd: Path, *, repo_root: Path, trees_dir_name: st if _path_is_at_or_under(cwd, Path(str(getattr(candidate, "root"))).resolve()) ] + @staticmethod + def _linked_worktree_candidate( + cwd: Path, + *, + repo_root: Path, + configured_root_path: Path | None, + raw_runtime: Any, + ) -> object | None: + """Resolve a Git-linked checkout that lives outside the configured trees directory.""" + + top_level_raw = _git_output(raw_runtime, cwd, ["rev-parse", "--show-toplevel"]) + if not top_level_raw: + return None + try: + top_level = Path(top_level_raw).expanduser().resolve() + except OSError: + return None + if not _path_is_at_or_under(cwd, top_level): + return None + + normalized_repo_root = repo_root.resolve() + if top_level == normalized_repo_root: + return None + if configured_root_path is not None and top_level == configured_root_path: + return None + + branch = _git_output(raw_runtime, top_level, ["branch", "--show-current"]) + return SimpleNamespace(name=branch or top_level.name, root=top_level) + def resolve_current_worktree_target( *, @@ -179,3 +218,18 @@ def repo_root_from_worktree_layout(worktree_root: Path, trees_dir_name: str) -> def _path_is_at_or_under(path: Path, root: Path) -> bool: return path == root or root in path.parents + + +def _git_output(runtime: object, cwd: Path, args: list[str]) -> str: + try: + process_runtime = resolve_process_runtime(runtime) + completed = process_runtime.run( + ["git", *args], + cwd=cwd, + timeout=10.0, + ) + except Exception: # noqa: BLE001 - target inference must fail closed + return "" + if getattr(completed, "returncode", 1) != 0: + return "" + return str(getattr(completed, "stdout", "") or "").strip() diff --git a/tests/python/actions/test_action_target_support.py b/tests/python/actions/test_action_target_support.py index e7f6c63f7..627676982 100644 --- a/tests/python/actions/test_action_target_support.py +++ b/tests/python/actions/test_action_target_support.py @@ -145,6 +145,69 @@ def test_resolve_action_targets_prefers_current_worktree_for_tree_mode_action_co self.assertIsNone(error) self.assertEqual(calls, [{"require_configured_main_root": False, "require_configured_root_match": True}]) + def test_unresolved_external_checkout_never_falls_back_to_main_candidate(self) -> None: + main = _Target(name="Main", root="/tmp/repo") + runtime = SimpleNamespace( + config=SimpleNamespace( + base_dir=Path("/tmp/repo"), + execution_root=Path("/tmp/envctl-external-worktree"), + ), + discover_projects=lambda mode: [main] if mode == "main" else [], + selectors_from_passthrough=lambda _args: [], + ) + route = parse_route(["ship"], env={"ENVCTL_DEFAULT_MODE": "main"}) + + selected, error = resolve_action_targets( + runtime=runtime, + route=route, + trees_only=False, + resolve_current_worktree_target=lambda **_kwargs: None, + interactive_selection_allowed=lambda _route: False, + no_target_selected_message=lambda _route: "no ship target", + ) + + self.assertEqual(selected, []) + self.assertEqual(error, "no ship target") + + def test_all_current_checkout_actions_prefer_external_linked_target_in_both_modes(self) -> None: + external = _Target(name="feature/external", root="/tmp/envctl-external-worktree") + main = _Target(name="Main", root="/tmp/repo") + runtime = SimpleNamespace( + config=SimpleNamespace( + base_dir=Path("/tmp/repo"), + execution_root=Path(external.root), + ), + discover_projects=lambda mode: [main] if mode == "main" else [], + selectors_from_passthrough=lambda _args: [], + ) + + for mode in ("trees", "main"): + for command in ("ship", "pr", "commit", "test", "test-focused"): + with self.subTest(mode=mode, command=command): + calls: list[dict[str, object]] = [] + route = parse_route([command], env={"ENVCTL_DEFAULT_MODE": mode}) + + selected, error = resolve_action_targets( + runtime=runtime, + route=route, + trees_only=False, + resolve_current_worktree_target=lambda **kwargs: calls.append(kwargs) or external, + interactive_selection_allowed=lambda _route: False, + no_target_selected_message=lambda _route: "no target", + ) + + self.assertEqual(selected, [external]) + self.assertIsNone(error) + self.assertEqual( + calls, + [ + { + "require_configured_main_root": mode == "main", + "require_configured_root_match": True, + } + ], + ) + def test_resolve_action_targets_explicit_project_overrides_current_worktree(self) -> None: current = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") requested = _Target(name="feature-b-1", root="/tmp/repo/trees/feature-b/1") diff --git a/tests/python/actions/test_action_worktree_runner.py b/tests/python/actions/test_action_worktree_runner.py index 39e25d99c..200743dbb 100644 --- a/tests/python/actions/test_action_worktree_runner.py +++ b/tests/python/actions/test_action_worktree_runner.py @@ -117,6 +117,66 @@ def test_resolve_current_worktree_target_discovers_matching_current_tree(self) - self.assertEqual(getattr(target, "name"), "feature-a-1") self.assertEqual(getattr(target, "root"), tree_root) + def test_resolve_current_worktree_target_accepts_external_linked_checkout_in_both_modes(self) -> None: + repo = Path("/tmp/repo").resolve() + linked_root = Path("/tmp/envctl-deep-code-cleanup").resolve() + invocation_cwd = linked_root / "python" / "envctl_engine" + + def run(command, *, cwd, timeout): # noqa: ANN001 + self.assertEqual(timeout, 10.0) + if command == ["git", "rev-parse", "--show-toplevel"]: + return SimpleNamespace(returncode=0, stdout=f"{linked_root}\n") + if command == ["git", "branch", "--show-current"]: + self.assertEqual(Path(cwd), linked_root) + return SimpleNamespace(returncode=0, stdout="agent/deep-code-cleanup\n") + return SimpleNamespace(returncode=1, stdout="") + + runtime = SimpleNamespace( + env={"ENVCTL_INVOCATION_CWD": str(invocation_cwd)}, + raw_runtime=SimpleNamespace( + config=SimpleNamespace(base_dir=repo, trees_dir_name="trees"), + process_runner=SimpleNamespace(run=run), + ), + ) + + for require_configured_main_root in (False, True): + with self.subTest(require_configured_main_root=require_configured_main_root): + target = resolve_current_worktree_target( + runtime=runtime, + require_configured_main_root=require_configured_main_root, + require_configured_root_match=True, + current_cwd=lambda: Path("/should/not/be/used"), + discover_tree_projects_fn=lambda _repo_root, _trees_dir_name: [], + main_repo_root_for_linked_worktree_fn=lambda _worktree_root: None, + git_main_repo_root_for_worktree_fn=lambda **_kwargs: repo, + ) + + self.assertIsNotNone(target) + self.assertEqual(getattr(target, "name"), "agent/deep-code-cleanup") + self.assertEqual(getattr(target, "root"), linked_root) + + def test_resolve_current_worktree_target_does_not_synthesize_main_checkout(self) -> None: + repo = Path("/tmp/repo").resolve() + runtime = SimpleNamespace( + env={"ENVCTL_INVOCATION_CWD": str(repo / "python")}, + raw_runtime=SimpleNamespace( + config=SimpleNamespace(base_dir=repo, trees_dir_name="trees"), + process_runner=SimpleNamespace( + run=lambda *_args, **_kwargs: SimpleNamespace(returncode=0, stdout=f"{repo}\n") + ), + ), + ) + + target = resolve_current_worktree_target( + runtime=runtime, + require_configured_main_root=True, + require_configured_root_match=True, + discover_tree_projects_fn=lambda _repo_root, _trees_dir_name: [], + git_main_repo_root_for_worktree_fn=lambda **_kwargs: repo, + ) + + self.assertIsNone(target) + def test_run_delete_worktree_action_runs_cleanup_and_delete(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo_root = Path(tmpdir) / "repo" diff --git a/tests/python/actions/test_actions_git_parity.py b/tests/python/actions/test_actions_git_parity.py index d4eab762e..9bd724713 100644 --- a/tests/python/actions/test_actions_git_parity.py +++ b/tests/python/actions/test_actions_git_parity.py @@ -1,5 +1,8 @@ from __future__ import annotations +import os +import shlex +import subprocess import sys import tempfile from contextlib import redirect_stdout @@ -19,6 +22,9 @@ ) +REPO_ROOT = Path(__file__).resolve().parents[3] + + class ActionsGitParityTests(_ActionsParityTestCase): def test_headless_review_auto_selects_single_main_target(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -225,6 +231,76 @@ def test_ship_without_project_infers_current_worktree_in_trees_mode(self) -> Non self.assertEqual(fake_runner.run_envs[0]["ENVCTL_ACTION_PROJECT"], "feature-a-1") self.assertEqual(fake_runner.run_envs[0]["ENVCTL_ACTION_PROJECT_ROOT"], str(target.resolve())) + def test_repo_wrapper_ship_targets_external_linked_worktree_in_main_and_trees_modes(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = root / "repo" + linked = root / "external-linked-worktree" + repo.mkdir() + + def git(*args: str, cwd: Path = repo) -> None: + subprocess.run( + ["git", *args], + cwd=cwd, + check=True, + capture_output=True, + text=True, + ) + + git("init", "-q", "-b", "main") + git("config", "user.email", "envctl-test@example.test") + git("config", "user.name", "Envctl Test") + (repo / ".envctl").write_text( + "ENVCTL_DEFAULT_MODE=trees\nMAIN_STARTUP_ENABLE=false\nTREES_STARTUP_ENABLE=false\n", + encoding="utf-8", + ) + (repo / "README.md").write_text("# linked worktree ship test\n", encoding="utf-8") + git("add", "-f", ".envctl", "README.md") + git("commit", "-q", "-m", "initial") + git("worktree", "add", "-q", "-b", "feature/external", str(linked)) + + invocation_cwd = linked / "nested" / "source" + invocation_cwd.mkdir(parents=True) + assertion_script = ( + "import os,sys; from pathlib import Path; " + "expected=Path(sys.argv[1]).resolve(); " + "ok=(Path.cwd().resolve()==expected " + "and os.environ.get('ENVCTL_ACTION_PROJECT')=='feature/external' " + "and Path(os.environ.get('ENVCTL_ACTION_PROJECT_ROOT','')).resolve()==expected); " + "raise SystemExit(0 if ok else 9)" + ) + harmless_ship_command = shlex.join( + [sys.executable, "-c", assertion_script, str(linked.resolve())] + ) + + for mode in ("trees", "main"): + with self.subTest(mode=mode): + env = dict(os.environ) + for key in ("RUN_REPO_ROOT", "ENVCTL_EXECUTION_ROOT", "ENVCTL_INVOCATION_CWD"): + env.pop(key, None) + env.update( + { + "ENVCTL_ACTION_SHIP_CMD": harmless_ship_command, + "ENVCTL_DEFAULT_MODE": mode, + "ENVCTL_UI_BACKEND": "non_interactive", + "ENVCTL_USE_REPO_WRAPPER": "1", + "RUN_SH_RUNTIME_DIR": str(root / f"runtime-{mode}"), + } + ) + completed = subprocess.run( + [str(REPO_ROOT / "bin" / "envctl"), "ship", "--human"], + cwd=invocation_cwd, + env=env, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + self.assertEqual(completed.returncode, 0, msg=completed.stdout + completed.stderr) + self.assertIn("ship handoff status for feature/external: success", completed.stdout) + self.assertNotIn("ship handoff status for Main", completed.stdout) + def test_git_actions_use_python_native_defaults_when_available(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo = Path(tmpdir) / "repo" From 728cb44e866a4a1e850702d2ecd8bc03f190e5e8 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:12:58 +0300 Subject: [PATCH 17/26] preserve aggregate untested test scope --- .../actions/action_test_execution_support.py | 2 + .../actions/action_test_runner.py | 6 +++ ...est_action_command_orchestrator_targets.py | 48 +++++++++++++++++++ .../test_action_test_command_support.py | 11 +++++ .../python/actions/test_action_test_runner.py | 30 ++++++++++++ 5 files changed, 97 insertions(+) diff --git a/python/envctl_engine/actions/action_test_execution_support.py b/python/envctl_engine/actions/action_test_execution_support.py index 13603e2df..dc4814dd7 100644 --- a/python/envctl_engine/actions/action_test_execution_support.py +++ b/python/envctl_engine/actions/action_test_execution_support.py @@ -113,6 +113,8 @@ def _selected_services(self) -> tuple[bool, bool]: ) def _target_contexts(self) -> list[TestTargetContext]: + if self.untested and not self.targets: + return [] return self.orchestrator._test_target_contexts(self.targets) def _execution_specs( diff --git a/python/envctl_engine/actions/action_test_runner.py b/python/envctl_engine/actions/action_test_runner.py index 5540c15cd..5a06530a2 100644 --- a/python/envctl_engine/actions/action_test_runner.py +++ b/python/envctl_engine/actions/action_test_runner.py @@ -82,6 +82,12 @@ def run_test_action( execution_specs = plan.execution_specs interactive_command = plan.interactive_command if not execution_specs: + if plan.untested and not plan.target_contexts: + print( + "No test command supports the all-untested scope. " + "Configure a test-all-trees.sh action command or select projects explicitly." + ) + return 1 print("No test command configured. Set Backend test command or Frontend test command in envctl config.") return 1 diff --git a/tests/python/actions/test_action_command_orchestrator_targets.py b/tests/python/actions/test_action_command_orchestrator_targets.py index 176173d5c..9e29fd510 100644 --- a/tests/python/actions/test_action_command_orchestrator_targets.py +++ b/tests/python/actions/test_action_command_orchestrator_targets.py @@ -10,6 +10,7 @@ REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" from envctl_engine.actions.action_command_orchestrator import ActionCommandOrchestrator +from envctl_engine.actions.action_test_execution_support import build_test_action_execution_plan import envctl_engine.actions.action_test_summary_support as action_test_summary_support_module from envctl_engine.actions.action_test_summary_support import ( write_failed_tests_summary_for_orchestrator, @@ -183,6 +184,53 @@ def test_resolve_targets_does_not_flush_for_non_interactive_command(self) -> Non self.assertEqual(len(targets), 2) flush_pending.assert_not_called() + def test_untested_scope_reaches_legacy_matrix_without_main_project_filter(self) -> None: + runtime = _RuntimeStub() + runtime.config = SimpleNamespace( + base_dir=Path("/tmp/repo"), + raw={"ENVCTL_ACTION_TEST_CMD": "bash scripts/test-all-trees.sh"}, + frontend_test_path="", + ) + runtime._discover_projects = lambda mode: [ + SimpleNamespace(name="feature-a-1", root=Path("/tmp/repo/trees/feature-a/1")) + ] + runtime.split_command = lambda raw, *, replacements: raw.split() + orchestrator = ActionCommandOrchestrator(runtime) + route = Route(command="test", mode="trees", flags={"untested": True}) + + targets, error = orchestrator.resolve_targets(route, trees_only=False) + plan = build_test_action_execution_plan(orchestrator, route, targets) + + self.assertIsNone(error) + self.assertEqual(targets, []) + self.assertEqual(plan.target_contexts, []) + self.assertEqual(len(plan.execution_specs), 1) + self.assertEqual(plan.execution_specs[0].project_name, "all-targets") + self.assertEqual(plan.execution_specs[0].args, ["untested=true"]) + self.assertEqual(plan.execution_specs[0].spec.cwd, Path("/tmp/repo")) + + def test_untested_scope_does_not_run_non_matrix_command_against_main(self) -> None: + runtime = _RuntimeStub() + runtime.config = SimpleNamespace( + base_dir=Path("/tmp/repo"), + raw={"ENVCTL_ACTION_TEST_CMD": "python -m pytest"}, + frontend_test_path="", + ) + runtime._discover_projects = lambda mode: [ + SimpleNamespace(name="feature-a-1", root=Path("/tmp/repo/trees/feature-a/1")) + ] + runtime.split_command = lambda raw, *, replacements: raw.split() + orchestrator = ActionCommandOrchestrator(runtime) + route = Route(command="test", mode="trees", flags={"untested": True}) + + targets, error = orchestrator.resolve_targets(route, trees_only=False) + plan = build_test_action_execution_plan(orchestrator, route, targets) + + self.assertIsNone(error) + self.assertEqual(targets, []) + self.assertEqual(plan.target_contexts, []) + self.assertEqual(plan.execution_specs, []) + def test_projects_for_services_matches_additional_service_slug_from_state(self) -> None: runtime = _RuntimeStub() runtime._latest_state = RunState( diff --git a/tests/python/actions/test_action_test_command_support.py b/tests/python/actions/test_action_test_command_support.py index f3a73c222..49e0b5b99 100644 --- a/tests/python/actions/test_action_test_command_support.py +++ b/tests/python/actions/test_action_test_command_support.py @@ -10,6 +10,7 @@ from envctl_engine.actions.action_test_command_support import ActionTestExecutionSpecBuilder from envctl_engine.actions.action_test_command_support import SharedTestCommandPlanner from envctl_engine.actions.action_test_command_support import build_test_execution_specs +from envctl_engine.actions.action_test_command_support import build_test_target_contexts from envctl_engine.actions.action_test_command_support import is_legacy_tree_test_script from envctl_engine.actions.action_test_command_support import normalize_backend_python_test_command from envctl_engine.actions.action_test_support_models import TestTargetContext as TargetContext @@ -58,6 +59,16 @@ def test_shared_legacy_tree_command_collapses_to_one_all_targets_spec(self) -> N self.assertEqual(specs[0].spec.cwd, repo) self.assertEqual(specs[0].args, ["projects=feature-a-1", "untested=true"]) + def test_empty_target_contexts_keep_main_fallback_for_non_aggregate_callers(self) -> None: + repo = Path("/repo") + + contexts = build_test_target_contexts([], repo_root=repo) + + self.assertEqual(len(contexts), 1) + self.assertEqual(contexts[0].project_name, "Main") + self.assertEqual(contexts[0].project_root, repo) + self.assertIsNone(contexts[0].target_obj) + def test_legacy_tree_script_detection_accepts_absolute_bash_path(self) -> None: self.assertTrue(is_legacy_tree_test_script(["bash", "scripts/test-all-trees.sh"])) self.assertTrue(is_legacy_tree_test_script(["/bin/bash", "/repo/scripts/test-all-trees.sh"])) diff --git a/tests/python/actions/test_action_test_runner.py b/tests/python/actions/test_action_test_runner.py index 46f901ef1..eef8319a4 100644 --- a/tests/python/actions/test_action_test_runner.py +++ b/tests/python/actions/test_action_test_runner.py @@ -49,6 +49,36 @@ def _run(orchestrator: _Orchestrator, route, targets: list[object]) -> int: # n class ActionTestRunnerShipOnPassTests(unittest.TestCase): + def test_untested_scope_without_matrix_command_fails_without_running_main(self) -> None: + orchestrator = _Orchestrator() + route = parse_route(["test", "--untested"], env={}) + + with ( + patch.object( + action_test_runner, + "build_test_action_execution_plan", + return_value=SimpleNamespace( + execution_specs=[], + interactive_command=False, + untested=True, + target_contexts=[], + ), + ), + patch("builtins.print") as printer, + patch.object( + action_test_runner, + "execute_test_suites", + side_effect=AssertionError("test suites must not run"), + ), + ): + code = _run(orchestrator, route, []) + + self.assertEqual(code, 1) + printer.assert_called_once_with( + "No test command supports the all-untested scope. " + "Configure a test-all-trees.sh action command or select projects explicitly." + ) + def test_ship_on_pass_rejects_blank_message_before_planning_tests(self) -> None: orchestrator = _Orchestrator() route = parse_route(["test", "--ship-on-pass="], env={}) From 593e35a653ca1602d6ff6594f551385ed4358fba Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:16:15 +0300 Subject: [PATCH 18/26] fix second-round Docker runtime review findings --- .../runtime/docker_service_runtime.py | 25 ++++-- .../runtime/engine_runtime_commands.py | 7 +- .../runtime/lifecycle_cleanup_orchestrator.py | 18 +++- python/envctl_engine/startup/finalization.py | 6 +- python/envctl_engine/startup/lifecycle.py | 27 +++++- .../startup/run_reuse_decision.py | 5 +- .../startup/run_reuse_fresh_start.py | 2 + .../startup/service_attach_execution.py | 33 ++++--- .../startup/service_project_execution.py | 52 +++++++---- .../startup/startup_selection_support.py | 7 ++ .../envctl_engine/state/action_log_support.py | 29 +++++- .../runtime/test_engine_runtime_commands.py | 15 ++++ ...t_lifecycle_cleanup_spinner_integration.py | 79 ++++++++++++++++ .../python/startup/test_run_reuse_support.py | 90 +++++++++++++++++++ .../startup/test_service_attach_execution.py | 22 ++++- .../startup/test_startup_lifecycle_lock.py | 35 +++++++- .../test_startup_orchestrator_profiles.py | 6 +- .../startup/test_support_module_decoupling.py | 89 ++++++++++++++++++ .../state/test_state_action_log_support.py | 24 +++++ 19 files changed, 520 insertions(+), 51 deletions(-) diff --git a/python/envctl_engine/runtime/docker_service_runtime.py b/python/envctl_engine/runtime/docker_service_runtime.py index 781182b66..e52118116 100644 --- a/python/envctl_engine/runtime/docker_service_runtime.py +++ b/python/envctl_engine/runtime/docker_service_runtime.py @@ -63,14 +63,14 @@ def docker_service_mode_enabled(runtime: Any) -> bool: return parse_bool(str(env.get("DOCKER_MODE", "")), False) -def docker_service_uses_image_command( +def docker_service_container_command_source( runtime: Any, service_name: str, *, docker_mode: bool | None = None, -) -> bool: +) -> str | None: if not (docker_service_mode_enabled(runtime) if docker_mode is None else docker_mode): - return False + return None env = getattr(runtime, "env", {}) raw = getattr(getattr(runtime, "config", None), "raw", {}) suffix = re.sub(r"[^A-Z0-9]+", "_", service_name.upper()).strip("_") or "SERVICE" @@ -83,8 +83,23 @@ def setting(name: str) -> str: return "" if setting("DOCKER_COMMAND"): - return False - return (setting("DOCKER_COMMAND_MODE") or "image").lower() in {"image", "default"} + return "docker_command" + if (setting("DOCKER_COMMAND_MODE") or "image").lower() in {"image", "default"}: + return "docker_image" + return None + + +def docker_service_uses_image_command( + runtime: Any, + service_name: str, + *, + docker_mode: bool | None = None, +) -> bool: + return docker_service_container_command_source( + runtime, + service_name, + docker_mode=docker_mode, + ) == "docker_image" def state_uses_docker_services(state: object) -> bool: diff --git a/python/envctl_engine/runtime/engine_runtime_commands.py b/python/envctl_engine/runtime/engine_runtime_commands.py index 22843e289..690c6f427 100644 --- a/python/envctl_engine/runtime/engine_runtime_commands.py +++ b/python/envctl_engine/runtime/engine_runtime_commands.py @@ -7,7 +7,7 @@ from typing import Any, Mapping from envctl_engine.runtime.command_resolution import resolve_requirement_start_command, resolve_service_start_command -from envctl_engine.runtime.docker_service_runtime import docker_service_uses_image_command +from envctl_engine.runtime.docker_service_runtime import docker_service_container_command_source SERVICE_LAUNCH_ENV_REMOVALS = { "ACTIONS_CACHE_URL", @@ -106,8 +106,9 @@ def service_start_command_resolved( project_root: Path | None = None, port: int = 0, ) -> tuple[list[str], str]: - if docker_service_uses_image_command(runtime, service_name): - return [], "docker_image" + container_command_source = docker_service_container_command_source(runtime, service_name) + if container_command_source is not None: + return [], container_command_source result = resolve_service_start_command( service_name=service_name, project_root=(project_root or runtime.config.base_dir), diff --git a/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py b/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py index f145dc768..a2654fb56 100644 --- a/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py +++ b/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py @@ -266,6 +266,19 @@ def _release_selected_dependency_components( def _stop_requirement_component_containers(self, component: Mapping[str, object]) -> None: stop_requirement_component_containers(self.runtime, component) + def _stop_requirement_component_containers_best_effort(self, component: Mapping[str, object]) -> None: + try: + self._stop_requirement_component_containers(component) + except Exception as exc: # noqa: BLE001 + component_id = str(component.get("id") or "dependency").strip() or "dependency" + detail = str(exc).strip() or exc.__class__.__name__ + self.runtime._emit( # type: ignore[attr-defined] + "cleanup.dependency_container.warning", + component=component_id, + detail=detail, + ) + print(f"Warning: could not stop {component_id} container during cleanup: {detail}") + def _release_requirement_component_ports(self, component: Mapping[str, object]) -> None: port_allocator = self._port_allocator(self.runtime) release_requirement_component_ports(component, release_port_fn=port_allocator.release) @@ -327,7 +340,10 @@ def clear_runtime_state(self, *, command: str, aggressive: bool = False, route: for definition in dependency_definitions(): component = requirements.component(definition.id) if bool(component.get("enabled", False)): - self._stop_requirement_component_containers(component) + if command in {"stop-all", "blast-all"}: + self._stop_requirement_component_containers_best_effort(component) + else: + self._stop_requirement_component_containers(component) if aggressive and self.blast_all_ecosystem_enabled(): self.blast_all_ecosystem_cleanup(route=route) diff --git a/python/envctl_engine/startup/finalization.py b/python/envctl_engine/startup/finalization.py index 77ce8611d..59ca72d26 100644 --- a/python/envctl_engine/startup/finalization.py +++ b/python/envctl_engine/startup/finalization.py @@ -96,6 +96,7 @@ def finalize_successful_startup( print_headless_plan_session_summary: Callable[[StartupSession], None], maybe_attach_plan_agent_terminal: Callable[[StartupSession], int | None], finalize_plan_agent_degraded_handoff: Callable[[StartupSession], int], + run_interactive_dashboard_loop: Callable[[RunState], int] | None = None, ) -> int: if session.plan_agent_handoff_degraded: return finalize_plan_agent_degraded_handoff(session) @@ -171,7 +172,8 @@ def finalize_successful_startup( if attach_code is not None: return attach_code if runtime._should_enter_post_start_interactive(session.effective_route): - return runtime._run_interactive_dashboard_loop(run_state) + dashboard_runner = run_interactive_dashboard_loop or runtime._run_interactive_dashboard_loop + return dashboard_runner(run_state) return 0 @@ -181,6 +183,7 @@ def finalize_successful_startup_with_runtime( *, validate_plan_agent_handoff: Callable[..., None], print_fn: Callable[[str], None] = print, + run_interactive_dashboard_loop: Callable[[RunState], int] | None = None, ) -> int: orchestrator_like = type("_StartupFinalizationRuntimeFacade", (), {"runtime": runtime})() return finalize_successful_startup( @@ -228,6 +231,7 @@ def finalize_successful_startup_with_runtime( validate_plan_agent_handoff=validate_plan_agent_handoff, print_fn=print_fn, ), + run_interactive_dashboard_loop=run_interactive_dashboard_loop, ) diff --git a/python/envctl_engine/startup/lifecycle.py b/python/envctl_engine/startup/lifecycle.py index 2eba8eaf0..7da06ef18 100644 --- a/python/envctl_engine/startup/lifecycle.py +++ b/python/envctl_engine/startup/lifecycle.py @@ -2,7 +2,7 @@ from functools import partial from pathlib import Path -from typing import Any +from typing import Any, Callable from envctl_engine.debug.debug_utils import file_lock from envctl_engine.planning.plan_agent.launch import launch_plan_agent_terminals @@ -53,6 +53,7 @@ trees_start_selection_required, ) from envctl_engine.shared.parsing import parse_float +from envctl_engine.state.models import RunState from envctl_engine.ui.debug_snapshot import emit_startup_plan_handoff_snapshot from envctl_engine.ui.spinner import spinner, use_spinner_policy from envctl_engine.ui.spinner_service import emit_spinner_policy, resolve_spinner_policy @@ -65,17 +66,36 @@ def execute_startup_lifecycle(orchestrator: Any, route: Any) -> int: ) timeout = max(parse_float(timeout_raw, 3600.0), 1.0) lock_path = Path(runtime.runtime_root) / "locks" / "startup.lock" + pending_dashboard_state: RunState | None = None + + def defer_interactive_dashboard(state: RunState) -> int: + nonlocal pending_dashboard_state + pending_dashboard_state = state + return 0 + try: with file_lock(lock_path, timeout=timeout): - return _execute_startup_lifecycle_locked(orchestrator, route) + code = _execute_startup_lifecycle_locked( + orchestrator, + route, + run_interactive_dashboard_loop=defer_interactive_dashboard, + ) except TimeoutError as exc: message = f"Startup could not acquire the repository lifecycle lock: {exc}" runtime._emit("startup.lock.timeout", lock_path=str(lock_path), timeout_seconds=timeout) print(message) return 1 + if code == 0 and pending_dashboard_state is not None: + return runtime._run_interactive_dashboard_loop(pending_dashboard_state) + return code -def _execute_startup_lifecycle_locked(orchestrator: Any, route: Any) -> int: +def _execute_startup_lifecycle_locked( + orchestrator: Any, + route: Any, + *, + run_interactive_dashboard_loop: Callable[[RunState], int] | None = None, +) -> int: runtime = orchestrator.runtime session = create_startup_session(runtime, route) finalize_failure = partial( @@ -113,6 +133,7 @@ def _execute_startup_lifecycle_locked(orchestrator: Any, route: Any) -> int: runtime, validate_plan_agent_attach_target, ), + run_interactive_dashboard_loop=run_interactive_dashboard_loop, ) except RuntimeError as exc: return finalize_failure(error=str(exc)) diff --git a/python/envctl_engine/startup/run_reuse_decision.py b/python/envctl_engine/startup/run_reuse_decision.py index 4c0a9228e..a77c851d7 100644 --- a/python/envctl_engine/startup/run_reuse_decision.py +++ b/python/envctl_engine/startup/run_reuse_decision.py @@ -336,7 +336,10 @@ def _candidate_fresh( def mark_run_reused(metadata: Mapping[str, object] | None, *, reason: str) -> dict[str, object]: updated = dict(metadata or {}) raw_count = updated.get("run_reuse_count", 0) - count = int(raw_count) if isinstance(raw_count, (bool, float, int, str)) else 0 + try: + count = int(raw_count) if isinstance(raw_count, (bool, float, int, str)) else 0 + except (OverflowError, TypeError, ValueError): + count = 0 updated["run_reuse_count"] = max(0, count) + 1 updated["last_reopened_at"] = datetime.now(tz=UTC).isoformat() updated["last_reuse_reason"] = reason diff --git a/python/envctl_engine/startup/run_reuse_fresh_start.py b/python/envctl_engine/startup/run_reuse_fresh_start.py index e1b350e21..53efcd16d 100644 --- a/python/envctl_engine/startup/run_reuse_fresh_start.py +++ b/python/envctl_engine/startup/run_reuse_fresh_start.py @@ -33,9 +33,11 @@ def fresh_start_replacement_services( if str(getattr(context, "name", "")).strip() } runtime_scope = str(route.flags.get("runtime_scope") or "").strip().lower() + has_app_launch_selection = any(key in route.flags for key in ("launch_backend", "launch_frontend")) replace_all_for_explicit_fresh_start = ( bool(route.flags.get("no_resume")) and not bool(route.flags.get("services")) + and not has_app_launch_selection and runtime_scope not in {"backend", "frontend", "dependencies", "fullstack"} ) selected: set[str] = set() diff --git a/python/envctl_engine/startup/service_attach_execution.py b/python/envctl_engine/startup/service_attach_execution.py index 686e44664..86aa1897d 100644 --- a/python/envctl_engine/startup/service_attach_execution.py +++ b/python/envctl_engine/startup/service_attach_execution.py @@ -11,8 +11,8 @@ from envctl_engine.runtime.docker_service_runtime import ( DockerServiceLaunch, DockerServiceRuntime, + docker_service_container_command_source, docker_service_mode_enabled, - docker_service_uses_image_command, ) from envctl_engine.startup.service_execution_policy import ordered_service_layers from envctl_engine.startup.service_execution_records import PreparedServiceLaunch @@ -310,11 +310,7 @@ def start_backend(self, port: int) -> tuple[bool, str | None, int | None]: if conflict_result is not None: return conflict_result command_resolve_started = time.monotonic() - command, _resolved_source = self.runtime._service_start_command_resolved( - service_name="backend", - project_root=self.project_root, - port=port, - ) + command = self._core_service_command("backend", port) self._emit_attach_phase("backend", "command_resolution", command_resolve_started) return self._launch_prepared_process( service_name="backend", @@ -337,11 +333,7 @@ def start_frontend(self, port: int) -> tuple[bool, str | None, int | None]: owner=f"{self.project_name}:services:frontend-launch", ) command_resolve_started = time.monotonic() - command, _resolved_source = self.runtime._service_start_command_resolved( - service_name="frontend", - project_root=self.project_root, - port=launch_port, - ) + command = self._core_service_command("frontend", launch_port) self._emit_attach_phase("frontend", "command_resolution", command_resolve_started) return self._launch_prepared_process( service_name="frontend", @@ -353,6 +345,21 @@ def start_frontend(self, port: int) -> tuple[bool, str | None, int | None]: pid_fallback=os.getpid() + 1, ) + def _core_service_command(self, service_name: str, port: int) -> list[str]: + container_command_source = docker_service_container_command_source( + self.runtime, + service_name, + docker_mode=self.docker_mode or docker_service_mode_enabled(self.runtime), + ) + if container_command_source is not None: + return [] + command, _resolved_source = self.runtime._service_start_command_resolved( + service_name=service_name, + project_root=self.project_root, + port=port, + ) + return command + def start_additional_service(self, service_name: str, port: int) -> tuple[bool, str | None, int | None]: conflict_result = self._conflicting_start_result(service_name, port) if conflict_result is not None: @@ -362,11 +369,11 @@ def start_additional_service(self, service_name: str, port: int) -> tuple[bool, if service is None: return False, f"unknown additional service {service_name}", None command_resolve_started = time.monotonic() - if docker_service_uses_image_command( + if docker_service_container_command_source( self.runtime, service_name, docker_mode=self.docker_mode or docker_service_mode_enabled(self.runtime), - ): + ) is not None: command = [] else: command = self.runtime._split_command( diff --git a/python/envctl_engine/startup/service_project_execution.py b/python/envctl_engine/startup/service_project_execution.py index 3a2c04d30..89077ea3f 100644 --- a/python/envctl_engine/startup/service_project_execution.py +++ b/python/envctl_engine/startup/service_project_execution.py @@ -6,7 +6,10 @@ from typing import cast from envctl_engine.runtime.command_resolution import CommandExists, suggest_service_start_command -from envctl_engine.runtime.docker_service_runtime import docker_service_mode_enabled +from envctl_engine.runtime.docker_service_runtime import ( + docker_service_container_command_source, + docker_service_mode_enabled, +) from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.runtime_context import resolve_process_runtime from envctl_engine.startup.startup_selection_support import _restart_service_types_for_project @@ -282,12 +285,18 @@ def project_env_for_service(service_name: str) -> dict[str, str]: reason="all_services_disabled", ) return {} - if route is not None and route.command == "plan" and _no_local_app_system_configured( - rt, - mode=effective_mode, - project_root=context.root, - selected_service_types=selected_service_types, - configured_additional_services=configured_additional_services, + use_docker_services = bool(route is not None and route.flags.get("docker")) or docker_service_mode_enabled(rt) + if ( + route is not None + and route.command == "plan" + and not use_docker_services + and _no_local_app_system_configured( + rt, + mode=effective_mode, + project_root=context.root, + selected_service_types=selected_service_types, + configured_additional_services=configured_additional_services, + ) ): rt._emit( "service.attach.skipped", @@ -303,7 +312,6 @@ def project_env_for_service(service_name: str) -> dict[str, str]: return {} service_started = time.monotonic() - use_docker_services = bool(route is not None and route.flags.get("docker")) or docker_service_mode_enabled(rt) prepare_backend_duration_ms = 0.0 prepare_frontend_duration_ms = 0.0 prepare_backend = "backend" in selected_service_types @@ -387,18 +395,30 @@ def prepare_frontend_runtime() -> float: rebound_delta = parse_int(rt.env.get("ENVCTL_TEST_FRONTEND_REBOUND_DELTA"), 0) backend_command_source = None if "backend" in selected_service_types: - backend_command_source = rt._service_command_source( - service_name="backend", - project_root=context.root, - port=backend_plan.final, + backend_command_source = docker_service_container_command_source( + rt, + "backend", + docker_mode=use_docker_services, ) + if backend_command_source is None: + backend_command_source = rt._service_command_source( + service_name="backend", + project_root=context.root, + port=backend_plan.final, + ) frontend_command_source = None if "frontend" in selected_service_types: - frontend_command_source = rt._service_command_source( - service_name="frontend", - project_root=context.root, - port=(frontend_plan.final + rebound_delta if rebound_delta > 0 else frontend_plan.final), + frontend_command_source = docker_service_container_command_source( + rt, + "frontend", + docker_mode=use_docker_services, ) + if frontend_command_source is None: + frontend_command_source = rt._service_command_source( + service_name="frontend", + project_root=context.root, + port=(frontend_plan.final + rebound_delta if rebound_delta > 0 else frontend_plan.final), + ) prepared_launches: dict[str, PreparedServiceLaunch] = {} command_env_builder = resolve_command_env_builder(rt) diff --git a/python/envctl_engine/startup/startup_selection_support.py b/python/envctl_engine/startup/startup_selection_support.py index 2235e19a6..80d1475b4 100644 --- a/python/envctl_engine/startup/startup_selection_support.py +++ b/python/envctl_engine/startup/startup_selection_support.py @@ -472,6 +472,13 @@ def _restart_service_types_for_project( def _apply_startup_service_launch_flags(service_types: set[str], *, route: Route) -> set[str]: selected = set(service_types) + if "launch_backend" in route.flags or "launch_frontend" in route.flags: + explicitly_selected: set[str] = set() + if route.flags.get("launch_backend") is True: + explicitly_selected.add("backend") + if route.flags.get("launch_frontend") is True: + explicitly_selected.add("frontend") + return selected.intersection(explicitly_selected) if route.flags.get("launch_backend") is False: selected.discard("backend") if route.flags.get("launch_frontend") is False: diff --git a/python/envctl_engine/state/action_log_support.py b/python/envctl_engine/state/action_log_support.py index b73a67e63..64d8f6fc7 100644 --- a/python/envctl_engine/state/action_log_support.py +++ b/python/envctl_engine/state/action_log_support.py @@ -204,7 +204,34 @@ def clear_one(service: ServiceRecord) -> tuple[int, int, int, int, str | None]: if not raw_path: return 0, 0, 1, 0, (None if quiet else f"{service.name}: log=n/a") log_path = Path(raw_path) + is_docker = str(getattr(service, "runtime_kind", "process") or "process").lower() == "docker" if not log_path.is_file(): + if is_docker: + try: + Path(f"{log_path}.docker-since").touch() + line = f"{service.name}: Docker log cursor advanced at {log_path}" + return 1, 0, 0, 0, ( + None + if quiet + else render_paths_in_terminal_text( + line, + paths=[log_path], + env=env, + interactive_tty=interactive_tty, + ) + ) + except OSError as exc: + line = f"{service.name}: failed to clear Docker logs at {log_path} ({exc})" + return 0, 0, 0, 1, ( + None + if quiet + else render_paths_in_terminal_text( + line, + paths=[log_path], + env=env, + interactive_tty=interactive_tty, + ) + ) line = f"{service.name}: log missing at {log_path}" return 0, 1, 0, 0, ( None @@ -219,7 +246,7 @@ def clear_one(service: ServiceRecord) -> tuple[int, int, int, int, str | None]: try: with log_path.open("w", encoding="utf-8"): pass - if str(getattr(service, "runtime_kind", "process") or "process").lower() == "docker": + if is_docker: Path(f"{log_path}.docker-since").touch() line = f"{service.name}: log cleared at {log_path}" return 1, 0, 0, 0, ( diff --git a/tests/python/runtime/test_engine_runtime_commands.py b/tests/python/runtime/test_engine_runtime_commands.py index a8732d123..f0918c21f 100644 --- a/tests/python/runtime/test_engine_runtime_commands.py +++ b/tests/python/runtime/test_engine_runtime_commands.py @@ -45,6 +45,21 @@ def test_docker_image_command_skips_host_command_resolution(self) -> None: self.assertEqual(command, []) self.assertEqual(source, "docker_image") + def test_explicit_docker_command_skips_host_command_resolution(self) -> None: + runtime = SimpleNamespace( + env={ + "DOCKER_MODE": "true", + "ENVCTL_BACKEND_DOCKER_COMMAND": "uvicorn app:api --port 8000", + }, + config=SimpleNamespace(base_dir=Path("/repo"), raw={}), + _command_exists=lambda _executable: self.fail("host command lookup must be skipped"), + ) + + command, source = service_start_command_resolved(runtime, service_name="backend", port=8000) + + self.assertEqual(command, []) + self.assertEqual(source, "docker_command") + def test_docker_service_command_still_requires_executable_on_host(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) diff --git a/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py b/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py index d28f7f304..02ebfe3a6 100644 --- a/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py +++ b/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py @@ -106,6 +106,85 @@ def _terminate_services_from_state(self, *args, **kwargs): # noqa: ANN001, ARG0 class LifecycleCleanupSpinnerIntegrationTests(unittest.TestCase): + def test_clear_runtime_state_purges_after_dependency_container_cleanup_failure(self) -> None: + for command, aggressive in (("stop-all", False), ("blast-all", True)): + with self.subTest(command=command): + runtime = _RuntimeStub() + runtime._try_load_existing_state = lambda *args, **kwargs: RunState( # type: ignore[method-assign] + run_id="run-1", + mode="main", + requirements={ + "Main": RequirementsResult( + project="Main", + redis={ + "id": "redis", + "enabled": True, + "container_name": "envctl-redis-main", + }, + ) + }, + ) + orchestrator = LifecycleCleanupOrchestrator(runtime) + orchestrator.blast_all_ecosystem_enabled = lambda: False # type: ignore[method-assign] + orchestrator.blast_all_purge_legacy_state_artifacts = lambda: None # type: ignore[method-assign] + output = StringIO() + + with ( + patch( + "envctl_engine.runtime.lifecycle_cleanup_orchestrator.stop_requirement_component_containers", + side_effect=RuntimeError("Docker daemon is unavailable"), + ), + redirect_stdout(output), + ): + orchestrator.clear_runtime_state(command=command, aggressive=aggressive) + + self.assertEqual(runtime.state_repository.purge_calls, [aggressive]) + self.assertIn("could not stop redis container", output.getvalue()) + warnings = [ + event + for event in runtime.events + if event.get("event") == "cleanup.dependency_container.warning" + ] + self.assertEqual( + warnings, + [ + { + "event": "cleanup.dependency_container.warning", + "component": "redis", + "detail": "Docker daemon is unavailable", + } + ], + ) + + def test_targeted_runtime_cleanup_remains_fail_closed(self) -> None: + runtime = _RuntimeStub() + runtime._try_load_existing_state = lambda *args, **kwargs: RunState( # type: ignore[method-assign] + run_id="run-1", + mode="main", + requirements={ + "Main": RequirementsResult( + project="Main", + redis={ + "id": "redis", + "enabled": True, + "container_name": "envctl-redis-main", + }, + ) + }, + ) + orchestrator = LifecycleCleanupOrchestrator(runtime) + + with ( + patch( + "envctl_engine.runtime.lifecycle_cleanup_orchestrator.stop_requirement_component_containers", + side_effect=RuntimeError("Docker daemon is unavailable"), + ), + self.assertRaisesRegex(RuntimeError, "Docker daemon is unavailable"), + ): + orchestrator.clear_runtime_state(command="stop", aggressive=False) + + self.assertEqual(runtime.state_repository.purge_calls, []) + def test_stop_all_emits_spinner_policy_and_lifecycle(self) -> None: runtime = _RuntimeStub() orchestrator = LifecycleCleanupOrchestrator(runtime) diff --git a/tests/python/startup/test_run_reuse_support.py b/tests/python/startup/test_run_reuse_support.py index 4f80f1735..21db183f1 100644 --- a/tests/python/startup/test_run_reuse_support.py +++ b/tests/python/startup/test_run_reuse_support.py @@ -7,6 +7,7 @@ from envctl_engine.startup.run_reuse_support import ( dashboard_stopped_service_entries, fresh_start_replacement_services, + mark_run_reused, prepare_dashboard_stopped_service_restore, replace_existing_project_services_for_fresh_start, metadata_without_dashboard_stopped_services, @@ -15,6 +16,12 @@ class RunReuseSupportTests(unittest.TestCase): + def test_mark_run_reused_recovers_from_malformed_counter(self) -> None: + updated = mark_run_reused({"run_reuse_count": "not-a-number"}, reason="exact_match") + + self.assertEqual(updated["run_reuse_count"], 1) + self.assertEqual(updated["last_reuse_reason"], "exact_match") + def test_run_reuse_debug_orch_groups_only_apply_to_plan_commands(self) -> None: runtime = SimpleNamespace(env={"ENVCTL_DEBUG_PLAN_ORCH_GROUP": "alpha+ beta,gamma ,,"}) @@ -219,6 +226,89 @@ def test_unfiltered_explicit_no_resume_replaces_tracked_removed_additional_servi self.assertEqual(selected, {"Main Backend", "Main Old Worker"}) + def test_explicit_no_resume_honors_launch_only_service_selection(self) -> None: + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace(name="Main Backend", project="Main", type="backend"), + "Main Frontend": SimpleNamespace(name="Main Frontend", project="Main", type="frontend"), + "Main Old Worker": SimpleNamespace( + name="Main Old Worker", + project="Main", + type="old-worker", + service_slug="old-worker", + ), + } + ) + + cases = ( + ( + { + "no_resume": True, + "launch_backend": True, + "launch_frontend": False, + "launch_dependencies": False, + }, + {"Main Backend"}, + ), + ( + { + "no_resume": True, + "launch_backend": False, + "launch_frontend": True, + "launch_dependencies": False, + }, + {"Main Frontend"}, + ), + ( + { + "no_resume": True, + "launch_backend": False, + "launch_frontend": False, + "launch_dependencies": False, + }, + set(), + ), + ) + for flags, expected in cases: + with self.subTest(flags=flags): + selected = fresh_start_replacement_services( + route=Route(command="start", mode="main", flags=flags), + selected_contexts=[SimpleNamespace(name="Main")], + candidate_state=state, + configured_service_types={"backend", "frontend"}, + additional_services=(), + project_name_from_service=lambda _name: "Main", + ) + self.assertEqual(selected, expected) + + def test_explicit_no_resume_with_no_dependencies_still_replaces_all_app_services(self) -> None: + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace(name="Main Backend", project="Main", type="backend"), + "Main Old Worker": SimpleNamespace( + name="Main Old Worker", + project="Main", + type="old-worker", + service_slug="old-worker", + ), + } + ) + + selected = fresh_start_replacement_services( + route=Route( + command="start", + mode="main", + flags={"no_resume": True, "launch_dependencies": False}, + ), + selected_contexts=[SimpleNamespace(name="Main")], + candidate_state=state, + configured_service_types={"backend"}, + additional_services=(), + project_name_from_service=lambda _name: "Main", + ) + + self.assertEqual(selected, {"Main Backend", "Main Old Worker"}) + def test_replace_existing_project_services_for_fresh_start_terminates_selected_services_and_orphans(self) -> None: route = Route(command="start", mode="trees", raw_args=["start"], flags={}) session = SimpleNamespace(effective_route=route, runtime_mode="trees") diff --git a/tests/python/startup/test_service_attach_execution.py b/tests/python/startup/test_service_attach_execution.py index c6001af36..d9688aed7 100644 --- a/tests/python/startup/test_service_attach_execution.py +++ b/tests/python/startup/test_service_attach_execution.py @@ -28,9 +28,8 @@ def test_explicit_docker_mode_launches_and_annotates_container_service(self) -> services=ServiceManager(), _conflict_remaining={}, _emit=lambda event, **payload: None, - _service_start_command_resolved=lambda service_name, project_root, port: ( - ["python", "app.py", "--port", str(port)], - "configured", + _service_start_command_resolved=lambda **kwargs: self.fail( + f"host command resolution must be skipped: {kwargs}" ), ) runner = ServiceAttachRunner( @@ -89,6 +88,23 @@ def test_explicit_docker_mode_launches_and_annotates_container_service(self) -> self.assertEqual(record.container_name, "envctl-app-main-backend") self.assertEqual(record.container_image, "example/backend:dev") docker_launch.assert_called_once() + self.assertEqual(docker_launch.call_args.kwargs["command"], []) + + runtime.config.raw["ENVCTL_BACKEND_DOCKER_COMMAND"] = "uvicorn app:api --port 8000" + self.assertEqual(runner._core_service_command("backend", 8001), []) + + runtime.config.raw.pop("ENVCTL_BACKEND_DOCKER_COMMAND") + runtime.config.raw["ENVCTL_BACKEND_DOCKER_COMMAND_MODE"] = "service" + resolved_calls: list[dict[str, object]] = [] + runtime._service_start_command_resolved = lambda **kwargs: ( + resolved_calls.append(kwargs) or ["container-service", "--port", str(kwargs["port"])], + "configured", + ) + self.assertEqual( + runner._core_service_command("backend", 8001), + ["container-service", "--port", "8001"], + ) + self.assertEqual(resolved_calls, [{"service_name": "backend", "project_root": project_root, "port": 8001}]) def test_runner_builds_layered_descriptors_and_preserves_additional_urls(self) -> None: project_root = Path("/tmp/envctl-project") diff --git a/tests/python/startup/test_startup_lifecycle_lock.py b/tests/python/startup/test_startup_lifecycle_lock.py index a7c39f0f5..d89206466 100644 --- a/tests/python/startup/test_startup_lifecycle_lock.py +++ b/tests/python/startup/test_startup_lifecycle_lock.py @@ -26,13 +26,46 @@ def recording_lock(path: Path, *, timeout: float): monkeypatch.setattr( lifecycle, "_execute_startup_lifecycle_locked", - lambda received_orchestrator, route: seen.append((received_orchestrator, route)) or 7, + lambda received_orchestrator, route, **_kwargs: seen.append((received_orchestrator, route)) or 7, ) assert lifecycle.execute_startup_lifecycle(orchestrator, "route") == 7 assert seen == [tmp_path / "locks" / "startup.lock", 3600.0, "entered", (orchestrator, "route"), "exited"] +def test_execute_startup_lifecycle_releases_lock_before_dashboard(monkeypatch, tmp_path: Path) -> None: + seen: list[str] = [] + state = SimpleNamespace(run_id="run-1") + + @contextmanager + def recording_lock(_path: Path, *, timeout: float): + _ = timeout + seen.append("lock-entered") + yield + seen.append("lock-exited") + + def execute_locked(_orchestrator, _route, *, run_interactive_dashboard_loop): # noqa: ANN001 + seen.append("startup-finalized") + return run_interactive_dashboard_loop(state) + + def dashboard(received_state: object) -> int: + assert received_state is state + seen.append("dashboard-entered") + return 0 + + runtime = SimpleNamespace( + runtime_root=tmp_path, + env={}, + config=SimpleNamespace(raw={}), + _run_interactive_dashboard_loop=dashboard, + ) + monkeypatch.setattr(lifecycle, "file_lock", recording_lock) + monkeypatch.setattr(lifecycle, "_execute_startup_lifecycle_locked", execute_locked) + + assert lifecycle.execute_startup_lifecycle(SimpleNamespace(runtime=runtime), "route") == 0 + assert seen == ["lock-entered", "startup-finalized", "lock-exited", "dashboard-entered"] + + def test_execute_startup_lifecycle_reports_lock_timeout(monkeypatch, tmp_path: Path, capsys) -> None: @contextmanager def timing_out_lock(path: Path, *, timeout: float): diff --git a/tests/python/startup/test_startup_orchestrator_profiles.py b/tests/python/startup/test_startup_orchestrator_profiles.py index a69b2e0e0..2fae84505 100644 --- a/tests/python/startup/test_startup_orchestrator_profiles.py +++ b/tests/python/startup/test_startup_orchestrator_profiles.py @@ -365,7 +365,7 @@ def test_startup_only_flags_select_individual_worktree_launch_parts(self) -> Non _restart_service_types_for_project( route=route, project_name="feature-a-1", - default_service_types={"backend", "frontend"}, + default_service_types={"backend", "frontend", "voice-runtime"}, ), {"frontend"}, ) @@ -376,7 +376,7 @@ def test_startup_only_flags_select_individual_worktree_launch_parts(self) -> Non _restart_service_types_for_project( route=route, project_name="feature-a-1", - default_service_types={"backend", "frontend"}, + default_service_types={"backend", "frontend", "voice-runtime"}, ), {"backend"}, ) @@ -387,7 +387,7 @@ def test_startup_only_flags_select_individual_worktree_launch_parts(self) -> Non _restart_service_types_for_project( route=route, project_name="feature-a-1", - default_service_types={"backend", "frontend"}, + default_service_types={"backend", "frontend", "voice-runtime"}, ), set(), ) diff --git a/tests/python/startup/test_support_module_decoupling.py b/tests/python/startup/test_support_module_decoupling.py index f58d1e238..7345c99f9 100644 --- a/tests/python/startup/test_support_module_decoupling.py +++ b/tests/python/startup/test_support_module_decoupling.py @@ -1013,6 +1013,95 @@ def test_start_project_services_skips_default_backend_frontend_when_no_local_sys self.assertEqual(len(warnings), 1) self.assertIn("No local app system is configured", str(warnings[0])) + def test_start_project_services_does_not_skip_docker_only_plan_app_system(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + project_root = root / "repo" + project_root.mkdir(parents=True, exist_ok=True) + run_dir = root / "runtime" / "run-1" + events: list[tuple[str, dict[str, object]]] = [] + prep_calls: list[str] = [] + + runtime = SimpleNamespace( + port_planner=_PortAllocatorStub(), + env={}, + config=SimpleNamespace( + raw={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}, + explicit_keys=("ENVCTL_BACKEND_DOCKER_IMAGE",), + backend_dir_name="backend", + frontend_dir_name="frontend", + additional_services=(), + backend_dependency_env_section_present=False, + frontend_dependency_env_section_present=False, + main_backend_dependency_env_section_present=False, + main_frontend_dependency_env_section_present=False, + trees_backend_dependency_env_section_present=False, + trees_frontend_dependency_env_section_present=False, + service_dependency_env_section_present={}, + mode_service_dependency_env_section_present={}, + all_app_service_names_for_mode=lambda mode, project_root=None: ("backend", "frontend"), + ), + services=SimpleNamespace(), + _invoke_envctl_hook=lambda **kwargs: SimpleNamespace(found=False, success=False, payload=None), + _run_dir_path=lambda run_id: run_dir, + _project_service_env=lambda context, requirements, route=None, service_name=None: {}, + _resolve_backend_env_file=lambda context, backend_cwd: (None, False), + _resolve_frontend_env_file=lambda context, frontend_cwd: None, + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_enabled_for_mode=lambda mode, service: True, + process_runner=SimpleNamespace(), + _prepare_backend_runtime=lambda **kwargs: prep_calls.append("backend"), + _prepare_frontend_runtime=lambda **kwargs: prep_calls.append("frontend"), + _service_command_source=lambda **kwargs: self.fail( + f"host command source must be skipped: {kwargs}" + ), + _command_exists=lambda command: False, + _emit=lambda event, **payload: events.append((event, payload)), + ) + orchestrator = SimpleNamespace( + runtime=runtime, + _restart_service_types_for_project=lambda **kwargs: {"backend", "frontend"}, + _suppress_timing_output=lambda route: True, + ) + context = SimpleNamespace( + name="Main", + root=project_root, + ports={ + "backend": SimpleNamespace(final=8000), + "frontend": SimpleNamespace(final=3000), + }, + ) + route = parse_route( + ["--plan", "feature-a", "--docker", "--entire-system", "--batch"], + env={}, + ) + + with mock.patch( + "envctl_engine.startup.service_project_execution.ServiceAttachRunner" + ) as runner_cls: + runner_cls.return_value.start.return_value = {} + records = start_project_services( + orchestrator, + context, + requirements=RequirementsResult(project="Main", components={}, health="healthy", failures=[]), + run_id="run-1", + route=route, + ) + + self.assertEqual(records, {}) + self.assertEqual(prep_calls, []) + self.assertTrue(runner_cls.call_args.kwargs["docker_mode"]) + self.assertEqual( + runner_cls.call_args.kwargs["prepared_launches"]["backend"].command_source, + "docker_image", + ) + self.assertFalse( + any( + event == "service.attach.skipped" and payload.get("reason") == "no_system_configured" + for event, payload in events + ) + ) + def test_start_project_services_keeps_missing_command_failure_for_explicit_enablement(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) diff --git a/tests/python/state/test_state_action_log_support.py b/tests/python/state/test_state_action_log_support.py index ee9500ab0..c58fef5d5 100644 --- a/tests/python/state/test_state_action_log_support.py +++ b/tests/python/state/test_state_action_log_support.py @@ -103,6 +103,30 @@ def test_clear_service_logs_truncates_available_files_and_reports_missing_paths( self.assertIn("Main Frontend: log missing", rendered) self.assertIn("Main Worker: log=n/a", rendered) + def test_clear_service_logs_advances_docker_cursor_without_snapshot_file(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + log_path = Path(tmp) / "docker-backend.log" + marker = Path(f"{log_path}.docker-since") + state = RunState( + run_id="run-docker", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=tmp, + log_path=str(log_path), + runtime_kind="docker", + ) + }, + ) + + summary = StateActionLogSupport.clear_service_logs(state, quiet=True) + + self.assertEqual(summary, (1, 0, 0, 0)) + self.assertFalse(log_path.exists()) + self.assertTrue(marker.is_file()) + if __name__ == "__main__": unittest.main() From 7095217c61f3a0fb8af7656010e51c3ffa9a510f Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:18:22 +0300 Subject: [PATCH 19/26] keep untested runner within architecture budget --- .../actions/action_test_execution_support.py | 9 +++++++++ python/envctl_engine/actions/action_test_runner.py | 8 +------- .../actions/test_action_command_orchestrator_targets.py | 5 +++++ tests/python/actions/test_action_test_runner.py | 6 ++++-- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/python/envctl_engine/actions/action_test_execution_support.py b/python/envctl_engine/actions/action_test_execution_support.py index dc4814dd7..0e067a233 100644 --- a/python/envctl_engine/actions/action_test_execution_support.py +++ b/python/envctl_engine/actions/action_test_execution_support.py @@ -31,6 +31,15 @@ def execution_mode(self) -> str: def multi_project(self) -> bool: return len(self.distinct_projects) > 1 + @property + def missing_execution_specs_message(self) -> str: + if self.untested and not self.target_contexts: + return ( + "No test command supports the all-untested scope. " + "Configure a test-all-trees.sh action command or select projects explicitly." + ) + return "No test command configured. Set Backend test command or Frontend test command in envctl config." + @dataclass(frozen=True, slots=True) class SuiteSpinnerDecision: diff --git a/python/envctl_engine/actions/action_test_runner.py b/python/envctl_engine/actions/action_test_runner.py index 5a06530a2..eaad7658b 100644 --- a/python/envctl_engine/actions/action_test_runner.py +++ b/python/envctl_engine/actions/action_test_runner.py @@ -82,13 +82,7 @@ def run_test_action( execution_specs = plan.execution_specs interactive_command = plan.interactive_command if not execution_specs: - if plan.untested and not plan.target_contexts: - print( - "No test command supports the all-untested scope. " - "Configure a test-all-trees.sh action command or select projects explicitly." - ) - return 1 - print("No test command configured. Set Backend test command or Frontend test command in envctl config.") + print(plan.missing_execution_specs_message) return 1 started_at = started_test_timer() diff --git a/tests/python/actions/test_action_command_orchestrator_targets.py b/tests/python/actions/test_action_command_orchestrator_targets.py index 9e29fd510..a53084ed3 100644 --- a/tests/python/actions/test_action_command_orchestrator_targets.py +++ b/tests/python/actions/test_action_command_orchestrator_targets.py @@ -230,6 +230,11 @@ def test_untested_scope_does_not_run_non_matrix_command_against_main(self) -> No self.assertEqual(targets, []) self.assertEqual(plan.target_contexts, []) self.assertEqual(plan.execution_specs, []) + self.assertEqual( + plan.missing_execution_specs_message, + "No test command supports the all-untested scope. " + "Configure a test-all-trees.sh action command or select projects explicitly.", + ) def test_projects_for_services_matches_additional_service_slug_from_state(self) -> None: runtime = _RuntimeStub() diff --git a/tests/python/actions/test_action_test_runner.py b/tests/python/actions/test_action_test_runner.py index eef8319a4..77ae61bee 100644 --- a/tests/python/actions/test_action_test_runner.py +++ b/tests/python/actions/test_action_test_runner.py @@ -60,8 +60,10 @@ def test_untested_scope_without_matrix_command_fails_without_running_main(self) return_value=SimpleNamespace( execution_specs=[], interactive_command=False, - untested=True, - target_contexts=[], + missing_execution_specs_message=( + "No test command supports the all-untested scope. " + "Configure a test-all-trees.sh action command or select projects explicitly." + ), ), ), patch("builtins.print") as printer, From ce794fae5103210490e8e30291cbf124ae45f660 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:34:36 +0300 Subject: [PATCH 20/26] fix scoped lifecycle and state review findings --- .../plan_agent/workflow_runtime_addresses.py | 37 ++++--- .../runtime/lifecycle_service_termination.py | 6 +- .../runtime/lifecycle_worktree_cleanup.py | 13 --- .../runtime/lifecycle_worktree_metadata.py | 12 ++ .../startup/resume_restore_project.py | 5 +- .../startup/run_reuse_fresh_start.py | 103 +++++++++++++++--- ...t_plan_agent_workflow_runtime_addresses.py | 37 +++++++ .../test_lifecycle_worktree_cleanup.py | 86 +++++++++++++++ .../python/startup/test_run_reuse_support.py | 57 +++++++++- .../startup/test_support_module_decoupling.py | 54 ++++++++- 10 files changed, 363 insertions(+), 47 deletions(-) diff --git a/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py b/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py index 99e6b5e41..43621fe42 100644 --- a/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py +++ b/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py @@ -61,29 +61,40 @@ def _latest_runtime_state(runtime: Any, *, worktree: CreatedPlanWorktree | None project_names = [worktree.name] if worktree is not None and str(worktree.name).strip() else None state_repository = optional_state_repository(runtime) if state_repository is not None and hasattr(state_repository, "load_latest"): - try: - state = call_state_loader(state_repository.load_latest, project_names=project_names) - except Exception: - state = None - if isinstance(state, RunState): - return state - try_loader = getattr(runtime, "_try_load_existing_state", None) - if callable(try_loader): - for mode in ("trees", "main"): + for selected_projects in _state_lookup_project_attempts(project_names): try: state = call_state_loader( - try_loader, - mode=mode, - strict_mode_match=True, - project_names=project_names, + state_repository.load_latest, + project_names=selected_projects, ) except Exception: state = None if isinstance(state, RunState): return state + try_loader = getattr(runtime, "_try_load_existing_state", None) + if callable(try_loader): + for mode in ("trees", "main"): + for selected_projects in _state_lookup_project_attempts(project_names): + try: + state = call_state_loader( + try_loader, + mode=mode, + strict_mode_match=True, + project_names=selected_projects, + ) + except Exception: + state = None + if isinstance(state, RunState): + return state return None +def _state_lookup_project_attempts(project_names: list[str] | None) -> tuple[list[str] | None, ...]: + if project_names is None: + return (None,) + return (project_names, None) + + def _dependency_address_lines(state: RunState, *, worktree: CreatedPlanWorktree | None = None) -> list[str]: rows: list[str] = [] seen: set[tuple[str, int]] = set() diff --git a/python/envctl_engine/runtime/lifecycle_service_termination.py b/python/envctl_engine/runtime/lifecycle_service_termination.py index ad1a87e35..aac50d64c 100644 --- a/python/envctl_engine/runtime/lifecycle_service_termination.py +++ b/python/envctl_engine/runtime/lifecycle_service_termination.py @@ -33,7 +33,7 @@ def terminate_started_services(runtime: Any, services: dict[str, object]) -> set if not terminated: failed.add(name) else: - _release_service_ports(runtime, name=name, service=service) + release_service_ports(runtime, name=name, service=service) return failed @@ -73,11 +73,11 @@ def terminate_one(item: tuple[str, object]) -> tuple[str, object, bool, int | No for name, service, terminated, _port in results: if terminated: - _release_service_ports(runtime, name=name, service=service) + release_service_ports(runtime, name=name, service=service) return {name for name, _service, terminated, _port in results if not terminated} -def _release_service_ports(runtime: Any, *, name: str, service: object) -> None: +def release_service_ports(runtime: Any, *, name: str, service: object) -> None: port_planner = getattr(runtime, "port_planner", None) if port_planner is None: return diff --git a/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py b/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py index a7e1b1ffa..07f726b84 100644 --- a/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py +++ b/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py @@ -141,7 +141,6 @@ def blast_worktree_before_delete( runtime._try_load_existing_state, mode=mode, strict_mode_match=True, - project_names=[normalized_project], ) if state is None: continue @@ -184,18 +183,6 @@ def blast_worktree_before_delete( target_ports.update(cleanup.collect_requirement_ports(requirement_entry)) release_requirement_ports(runtime, requirement_entry) - if not failed_services: - remaining_projects: set[str] = set() - for service_name in state.services: - project = runtime._project_name_from_service(service_name) - if project: - remaining_projects.add(project) - for project in list(state.requirements.keys()): - if project in remaining_projects: - continue - requirement_entry = state.requirements.pop(project) - release_requirement_ports(runtime, requirement_entry) - if not failed_services: artifact_paths.update(cleanup.prune_project_metadata(state, project_name=normalized_project)) diff --git a/python/envctl_engine/runtime/lifecycle_worktree_metadata.py b/python/envctl_engine/runtime/lifecycle_worktree_metadata.py index 297a5143f..dcc87fa09 100644 --- a/python/envctl_engine/runtime/lifecycle_worktree_metadata.py +++ b/python/envctl_engine/runtime/lifecycle_worktree_metadata.py @@ -8,6 +8,18 @@ def prune_project_metadata(state: RunState, *, project_name: str) -> list[Path]: removed_paths: list[Path] = [] + normalized_project = str(project_name).strip().casefold() + raw_project_names = state.metadata.get("project_names") + if isinstance(raw_project_names, list): + remaining_project_names = [ + name + for name in raw_project_names + if str(name).strip().casefold() != normalized_project + ] + if remaining_project_names: + state.metadata["project_names"] = remaining_project_names + else: + state.metadata.pop("project_names", None) for metadata_key in ("project_pr_links", "project_roots"): raw = state.metadata.get(metadata_key) if not isinstance(raw, dict): diff --git a/python/envctl_engine/startup/resume_restore_project.py b/python/envctl_engine/startup/resume_restore_project.py index efb52a0a4..233814ee7 100644 --- a/python/envctl_engine/startup/resume_restore_project.py +++ b/python/envctl_engine/startup/resume_restore_project.py @@ -5,6 +5,7 @@ import time from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.lifecycle_service_termination import release_service_ports from envctl_engine.state.models import RunState, ServiceRecord from envctl_engine.startup.resume_restore_policy import ( _configured_restore_service_types, @@ -349,9 +350,7 @@ def _stop_stale_services( ) if terminated: terminated_count += 1 - port = self.rt._service_port(service) # type: ignore[attr-defined] - if port is not None: - self.port_allocator.release(port) # type: ignore[attr-defined] + release_service_ports(self.rt, name=service_name, service=service) return terminated_count def _report_progress( diff --git a/python/envctl_engine/startup/run_reuse_fresh_start.py b/python/envctl_engine/startup/run_reuse_fresh_start.py index 32ff7f334..6f395c571 100644 --- a/python/envctl_engine/startup/run_reuse_fresh_start.py +++ b/python/envctl_engine/startup/run_reuse_fresh_start.py @@ -5,8 +5,9 @@ from typing import Any from envctl_engine.runtime.command_router import Route -from envctl_engine.shared.services import service_project_name +from envctl_engine.shared.services import service_project_name, service_slug_from_record from envctl_engine.startup.session import metadata_with_state_sources +from envctl_engine.startup.startup_selection_support import _restart_service_types_for_project def fresh_start_replacement_services( @@ -14,19 +15,52 @@ def fresh_start_replacement_services( selected_contexts: list[object], candidate_state: object, project_name_from_service: Callable[[str], str], + route: Route | None = None, + additional_services: tuple[object, ...] = (), ) -> set[str]: - """Select every old service owned by a project that is starting fresh.""" + """Select old services that overlap the requested fresh-start scope.""" - target_projects = {str(context.name).strip().casefold() for context in selected_contexts} - target_projects.discard("") + target_projects = { + str(context.name).strip().casefold(): str(context.name).strip() + for context in selected_contexts + if str(context.name).strip() + } if not target_projects: return set() - return { - service_name - for service_name, service in getattr(candidate_state, "services", {}).items() - if str(service_project_name(service) or project_name_from_service(service_name)).strip().casefold() - in target_projects + + services_by_project: dict[str, list[tuple[str, object, str]]] = { + project: [] for project in target_projects } + for service_name, service in getattr(candidate_state, "services", {}).items(): + project = str(service_project_name(service) or project_name_from_service(service_name)).strip().casefold() + if project not in target_projects: + continue + services_by_project[project].append( + (service_name, service, _service_type(service_name, service)) + ) + + selected: set[str] = set() + for project, project_services in services_by_project.items(): + if route is None or not _route_has_app_service_scope(route): + selected.update(name for name, _service, _service_type_name in project_services) + continue + configured_types = { + service_type_name + for _name, _service, service_type_name in project_services + if service_type_name + } + selected_types = _restart_service_types_for_project( + route=route, + project_name=target_projects[project], + default_service_types=configured_types, + additional_services=additional_services, + ) + selected.update( + name + for name, _service, service_type_name in project_services + if service_type_name in selected_types + ) + return selected @dataclass(frozen=True, slots=True) @@ -49,6 +83,10 @@ def replace(self) -> None: selected_contexts=list(self.session.selected_contexts), candidate_state=self.candidate_state, project_name_from_service=self.runtime._project_name_from_service, + route=route, + additional_services=tuple( + getattr(getattr(self.runtime, "config", None), "additional_services", ()) or () + ), ) ) target_projects = { @@ -65,10 +103,15 @@ def replace(self) -> None: if name not in selected_services } ) - preserved_requirements, requirements_to_release = _split_requirements( - self.candidate_state, - target_projects=target_projects, - ) + replace_requirements = _fresh_start_replaces_requirements(route) + if replace_requirements: + preserved_requirements, requirements_to_release = _split_requirements( + self.candidate_state, + target_projects=target_projects, + ) + else: + preserved_requirements = dict(self.candidate_state.requirements) + requirements_to_release = {} if selected_services: self.announce_session_identifiers(self.session) progress_message = ( @@ -182,3 +225,37 @@ def _failed_service_names(result: object) -> set[str]: if not isinstance(result, (set, frozenset)): return set() return {str(name).strip() for name in result if str(name).strip()} + + +def _route_has_app_service_scope(route: Route) -> bool: + runtime_scope = str(route.flags.get("runtime_scope") or "").strip().lower() + if runtime_scope in {"backend", "frontend", "fullstack", "dependencies"}: + return True + if isinstance(route.flags.get("services"), list) and bool(route.flags.get("services")): + return True + if isinstance(route.flags.get("restart_service_types"), list): + return True + return route.flags.get("launch_backend") is False or route.flags.get("launch_frontend") is False + + +def _fresh_start_replaces_requirements(route: Route) -> bool: + runtime_scope = str(route.flags.get("runtime_scope") or "").strip().lower() + if runtime_scope in {"dependencies", "entire-system"}: + return True + if _route_has_app_service_scope(route): + return False + if route.flags.get("launch_dependencies") is False: + return False + return True + + +def _service_type(service_name: str, service: object) -> str: + explicit = service_slug_from_record(service) + if explicit: + return explicit + lowered = str(service_name).strip().casefold() + for suffix in ("backend", "frontend"): + if lowered.endswith(f" {suffix}"): + return suffix + parts = lowered.rsplit(" ", 1) + return parts[-1] if len(parts) == 2 else "" diff --git a/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py b/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py index ebc0619ca..20e6c844e 100644 --- a/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py +++ b/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py @@ -43,6 +43,43 @@ def __init__(self, state: RunState | None) -> None: class PlanAgentWorkflowRuntimeAddressTests(unittest.TestCase): + def test_worktree_address_lookup_retries_unfiltered_for_base_project_state(self) -> None: + calls: list[list[str] | None] = [] + state = RunState( + run_id="run-base", + mode="trees", + requirements={ + "feature-a": RequirementsResult( + project="feature-a", + redis={"enabled": True, "final": 6381}, + ) + }, + ) + + class BaseProjectRepository: + def load_latest( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: list[str] | None = None, + ) -> RunState | None: + _ = mode, strict_mode_match + calls.append(project_names) + return state if project_names is None else None + + runtime = SimpleNamespace(state_repository=BaseProjectRepository()) + worktree = CreatedPlanWorktree( + name="feature-a-1", + root=Path("/repo/trees/feature-a/1"), + plan_file="feature-a.md", + ) + + section = _runtime_addresses_prompt_section(runtime, worktree=worktree) + + self.assertEqual(calls, [["feature-a-1"], None]) + self.assertIn("Redis (feature-a): redis://localhost:6381", section) + def test_builder_renders_dependency_and_service_addresses_for_matching_worktree(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: worktree_root = Path(tmpdir) / "repo" / "trees" / "feature-a-1" diff --git a/tests/python/runtime/test_lifecycle_worktree_cleanup.py b/tests/python/runtime/test_lifecycle_worktree_cleanup.py index 759666a97..8f43617d8 100644 --- a/tests/python/runtime/test_lifecycle_worktree_cleanup.py +++ b/tests/python/runtime/test_lifecycle_worktree_cleanup.py @@ -12,6 +12,92 @@ class LifecycleWorktreeCleanupTests(unittest.TestCase): + def test_worktree_cleanup_persists_siblings_from_the_full_multi_project_run(self) -> None: + loader_calls: list[dict[str, object]] = [] + saved_states: list[RunState] = [] + target_service = ServiceRecord( + name="Feature Backend", + type="backend", + cwd="/repo/feature/backend", + actual_port=8000, + project="Feature", + ) + sibling_service = ServiceRecord( + name="Sibling Backend", + type="backend", + cwd="/repo/sibling/backend", + actual_port=8020, + project="Sibling", + ) + full_state = RunState( + run_id="run-multi", + mode="trees", + services={target_service.name: target_service, sibling_service.name: sibling_service}, + requirements={ + "Feature": RequirementsResult(project="Feature", db={"enabled": True, "final": 5432}), + "Sibling": RequirementsResult(project="Sibling", db={"enabled": True, "final": 5452}), + "SiblingInfra": RequirementsResult( + project="SiblingInfra", + redis={"enabled": True, "final": 6399}, + ), + }, + metadata={ + "project_names": ["Feature", "Sibling", "SiblingInfra"], + "project_roots": { + "Feature": "/repo/feature", + "Sibling": "/repo/sibling", + "SiblingInfra": "/repo/sibling-infra", + }, + }, + ) + + def load_state(**kwargs: object) -> RunState | None: + loader_calls.append(dict(kwargs)) + if kwargs.get("mode") != "trees": + return None + if kwargs.get("project_names"): + return RunState( + run_id=full_state.run_id, + mode=full_state.mode, + services={target_service.name: target_service}, + requirements={"Feature": full_state.requirements["Feature"]}, + metadata={ + "project_names": ["Feature"], + "project_roots": {"Feature": "/repo/feature"}, + }, + ) + return full_state + + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _try_load_existing_state=load_state, + _project_name_from_service=lambda name: str(name).removesuffix(" Backend"), + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + port_planner=SimpleNamespace(release=lambda _port: None), + state_repository=SimpleNamespace( + save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) + ), + process_runner=SimpleNamespace(), + env={}, + ) + + blast_worktree_before_delete( + runtime, + project_name="Feature", + project_root=Path("/repo/feature"), + source_command="delete-worktree", + ) + + self.assertTrue(loader_calls) + self.assertTrue(all("project_names" not in call for call in loader_calls)) + self.assertEqual(set(saved_states[0].services), {"Sibling Backend"}) + self.assertEqual(set(saved_states[0].requirements), {"Sibling", "SiblingInfra"}) + self.assertEqual(saved_states[0].metadata["project_names"], ["Sibling", "SiblingInfra"]) + self.assertEqual( + saved_states[0].metadata["project_roots"], + {"Sibling": "/repo/sibling", "SiblingInfra": "/repo/sibling-infra"}, + ) + def test_blast_worktree_before_delete_updates_matching_project_state(self) -> None: events: list[tuple[str, dict[str, object]]] = [] saved_states: list[RunState] = [] diff --git a/tests/python/startup/test_run_reuse_support.py b/tests/python/startup/test_run_reuse_support.py index bbd34f3cc..d6ae837c6 100644 --- a/tests/python/startup/test_run_reuse_support.py +++ b/tests/python/startup/test_run_reuse_support.py @@ -306,6 +306,56 @@ def test_explicit_no_resume_replaces_matching_services_instead_of_leaking_a_seco self.assertEqual(terminated, [{"feature-a Backend"}]) self.assertEqual(progress, ["Auto-resume disabled; replacing 1 existing service(s)..."]) + def test_backend_scoped_fresh_start_preserves_frontend_and_existing_requirements(self) -> None: + route = Route( + command="start", + mode="trees", + flags={"no_resume": True, "runtime_scope": "backend"}, + ) + backend = SimpleNamespace(project="feature-a", type="backend") + frontend = SimpleNamespace(project="feature-a", type="frontend") + requirements = SimpleNamespace(project="feature-a") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": backend, "feature-a Frontend": frontend}, + requirements={"feature-a": requirements}, + metadata={}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + terminated: list[set[str]] = [] + released: list[object] = [] + runtime = SimpleNamespace( + config=SimpleNamespace(additional_services=()), + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda name: str(name).rsplit(" ", 1)[0], + _terminate_services_from_state=lambda _state, *, selected_services, **_kwargs: ( + terminated.append(set(selected_services)) or set() + ), + _release_requirement_ports=released.append, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda *_args: None, + terminate_restart_orphan_listeners=lambda **_kwargs: set(), + ) + + self.assertEqual(terminated, [{"feature-a Backend"}]) + self.assertEqual(session.preserved_services, {"feature-a Frontend": frontend}) + self.assertEqual(session.preserved_requirements, {"feature-a": requirements}) + self.assertEqual(released, []) + def test_explicit_no_resume_failure_keeps_previous_state_authoritative(self) -> None: route = Route(command="start", mode="trees", flags={"no_resume": True}) original_service = SimpleNamespace(project="feature-a") @@ -386,7 +436,12 @@ def test_fresh_start_aborts_when_orphan_exit_is_unconfirmed(self) -> None: self.assertEqual(session.preserved_requirements, {}) def test_explicit_no_resume_supersedes_requirement_only_state(self) -> None: - route = Route(command="start", mode="trees", flags={"no_resume": True}) + route = Route( + command="start", + mode="trees", + projects=["feature-a"], + flags={"no_resume": True}, + ) target_requirements = SimpleNamespace(project="feature-a") other_requirements = SimpleNamespace(project="feature-b") candidate_state = SimpleNamespace( diff --git a/tests/python/startup/test_support_module_decoupling.py b/tests/python/startup/test_support_module_decoupling.py index 10ac54eb0..c2caae7df 100644 --- a/tests/python/startup/test_support_module_decoupling.py +++ b/tests/python/startup/test_support_module_decoupling.py @@ -14,6 +14,7 @@ REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" from envctl_engine.startup.resume_restore_support import restore_missing # noqa: E402 +from envctl_engine.startup.resume_restore_project import ResumeProjectRestoreRunner # noqa: E402 import envctl_engine.startup.requirements_project_startup as requirements_project_startup_module # noqa: E402 from envctl_engine.startup.protocols import StartupOrchestratorLike # noqa: E402 from envctl_engine.startup.requirements_execution import ( # noqa: E402 @@ -34,7 +35,7 @@ from envctl_engine.startup.startup_orchestrator import StartupOrchestrator # noqa: E402 from envctl_engine.startup.startup_selection_support import _tree_preselected_projects_from_state # noqa: E402 from envctl_engine.config import AppServiceConfig # noqa: E402 -from envctl_engine.runtime.command_router import parse_route # noqa: E402 +from envctl_engine.runtime.command_router import Route, parse_route # noqa: E402 from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord # noqa: E402 @@ -68,6 +69,57 @@ def release(self, port: int) -> None: class StartupSupportModuleDecouplingTests(unittest.TestCase): + def test_resume_stale_service_releases_recorded_foreign_session_lock(self) -> None: + released: list[tuple[int, str, str]] = [] + + class RecordingPortAllocator: + session_id = "current-session" + + def release_owned(self, port: int, owner: str, *, expected_session: str) -> bool: + released.append((port, owner, expected_session)) + return True + + def release(self, _port: int, *, owner: str | None = None) -> None: + raise AssertionError(f"current-session release must not be used: {owner}") + + allocator = RecordingPortAllocator() + service = ServiceRecord( + name="Main Frontend", + type="frontend", + cwd="/tmp/repo/frontend", + actual_port=9000, + port_lock_session="prior-session", + project="Main", + ) + runtime = SimpleNamespace( + port_planner=allocator, + env={}, + config=SimpleNamespace(raw={}), + _terminate_service_record=lambda *_args, **_kwargs: True, + _project_name_from_service=lambda _name: "Main", + ) + runner = ResumeProjectRestoreRunner( + orchestrator=SimpleNamespace(), + rt=runtime, + state=RunState(run_id="run-1", mode="main"), + missing_services=[], + route_for_startup=Route(command="resume", mode="main"), + total_projects=1, + use_project_spinner_group=False, + project_spinner_group=SimpleNamespace(), + use_single_spinner=False, + active_spinner=SimpleNamespace(), + port_allocator=allocator, + ) + + terminated_count = runner._stop_stale_services( + service_names=[service.name], + original_services={service.name: service}, + ) + + self.assertEqual(terminated_count, 1) + self.assertEqual(released, [(9000, "Main:frontend", "prior-session")]) + def test_post_start_truth_failure_propagates_unterminated_service_records(self) -> None: requirements = RequirementsResult(project="Main", health="healthy") service = ServiceRecord( From 5883f637e52ce525b38e37e45b20bec269cc2bb9 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:41:05 +0300 Subject: [PATCH 21/26] fix review edge cases in test targeting and menu cleanup --- .../actions/action_target_support.py | 7 ++- .../actions/action_test_execution_support.py | 20 ++++++-- python/envctl_engine/planning/menu.py | 2 +- ...est_action_command_orchestrator_targets.py | 49 ++++++++++++++++++- .../planning/test_planning_menu_rendering.py | 48 +++++++++++++++++- 5 files changed, 118 insertions(+), 8 deletions(-) diff --git a/python/envctl_engine/actions/action_target_support.py b/python/envctl_engine/actions/action_target_support.py index 3945a27b8..ff1392842 100644 --- a/python/envctl_engine/actions/action_target_support.py +++ b/python/envctl_engine/actions/action_target_support.py @@ -272,9 +272,14 @@ def _request_from_route(self) -> _ActionTargetRequest: services = self.route.flags.get("services") unresolved_service_selectors: tuple[str, ...] = () if isinstance(services, list): + project_target_services = _normalized_service_targets(services) + if self.route.command in {"test", "test-focused"}: + project_target_services = [ + selector for selector in project_target_services if selector not in {"backend", "frontend"} + ] service_projects, unresolved_service_selectors = _resolve_projects_for_services( self.runtime, - services, + project_target_services, ) unresolved_service_selectors = tuple( selector diff --git a/python/envctl_engine/actions/action_test_execution_support.py b/python/envctl_engine/actions/action_test_execution_support.py index 0e067a233..9d040b225 100644 --- a/python/envctl_engine/actions/action_test_execution_support.py +++ b/python/envctl_engine/actions/action_test_execution_support.py @@ -64,7 +64,11 @@ def build(self) -> TestActionExecutionPlan: self._emit_scope_status() include_backend, include_frontend = self._selected_services() target_contexts = self._target_contexts() - _ensure_test_prereqs(self.orchestrator, target_contexts) + _ensure_test_prereqs( + self.orchestrator, + target_contexts, + aggregate_untested=self.untested and not target_contexts, + ) execution_specs = self._execution_specs( target_contexts=target_contexts, include_backend=include_backend, @@ -238,10 +242,18 @@ def print_test_execution_mode(orchestrator: Any, plan: TestActionExecutionPlan) print(orchestrator._colorize(text, fg=mode_color, bold=True)) -def _ensure_test_prereqs(orchestrator: Any, target_contexts: list[TestTargetContext]) -> None: +def _ensure_test_prereqs( + orchestrator: Any, + target_contexts: list[TestTargetContext], + *, + aggregate_untested: bool, +) -> None: + project_roots = [Path(context.project_root) for context in target_contexts] + if aggregate_untested: + project_roots.append(Path(orchestrator.runtime.config.base_dir)) seen_roots: set[Path] = set() - for context in target_contexts: - project_root = Path(context.project_root).resolve() + for root in project_roots: + project_root = root.resolve() if project_root in seen_roots: continue seen_roots.add(project_root) diff --git a/python/envctl_engine/planning/menu.py b/python/envctl_engine/planning/menu.py index e173a41d8..eb61569d7 100644 --- a/python/envctl_engine/planning/menu.py +++ b/python/envctl_engine/planning/menu.py @@ -109,7 +109,7 @@ def _run_legacy( finally: self._flush_input_buffer(fd=fd) self._restore_terminal_state(fd=fd, original_state=original_state) - self.flush_pending_input(fd=fd) + self._flush_input_buffer(fd=fd) sys.stdout.write("\033[0m\033[?25h\n") sys.stdout.flush() diff --git a/tests/python/actions/test_action_command_orchestrator_targets.py b/tests/python/actions/test_action_command_orchestrator_targets.py index a53084ed3..658d29272 100644 --- a/tests/python/actions/test_action_command_orchestrator_targets.py +++ b/tests/python/actions/test_action_command_orchestrator_targets.py @@ -197,13 +197,19 @@ def test_untested_scope_reaches_legacy_matrix_without_main_project_filter(self) runtime.split_command = lambda raw, *, replacements: raw.split() orchestrator = ActionCommandOrchestrator(runtime) route = Route(command="test", mode="trees", flags={"untested": True}) + prereq_roots: list[Path] = [] targets, error = orchestrator.resolve_targets(route, trees_only=False) - plan = build_test_action_execution_plan(orchestrator, route, targets) + with patch( + "envctl_engine.actions.action_test_execution_support.ensure_repo_local_test_prereqs", + side_effect=lambda root, emit_status: prereq_roots.append(Path(root)), + ): + plan = build_test_action_execution_plan(orchestrator, route, targets) self.assertIsNone(error) self.assertEqual(targets, []) self.assertEqual(plan.target_contexts, []) + self.assertEqual(prereq_roots, [Path("/tmp/repo").resolve()]) self.assertEqual(len(plan.execution_specs), 1) self.assertEqual(plan.execution_specs[0].project_name, "all-targets") self.assertEqual(plan.execution_specs[0].args, ["untested=true"]) @@ -259,6 +265,47 @@ def test_projects_for_services_matches_additional_service_slug_from_state(self) self.assertEqual(projects, ["Main"]) + def test_builtin_test_service_scopes_keep_current_worktree_when_state_resolves_service(self) -> None: + current = SimpleNamespace(name="feature-a-1", root=Path("/tmp/external-feature-a")) + for service_name in ("backend", "frontend"): + with self.subTest(service=service_name): + runtime = _RuntimeStub() + runtime.config = SimpleNamespace( + base_dir=Path("/tmp/repo"), + execution_root=current.root, + raw={}, + ) + runtime._latest_state = RunState( + run_id="run-1", + mode="trees", + services={ + f"feature-a-1 {service_name.title()}": ServiceRecord( + name=f"feature-a-1 {service_name.title()}", + type=service_name, + cwd=str(current.root), + requested_port=8000, + actual_port=8000, + status="running", + ) + }, + ) + runtime._project_name_from_service = lambda name: ( + "feature-a-1" if name.startswith("feature-a-1 ") else "" + ) + runtime._discover_projects = lambda mode: ( + [SimpleNamespace(name="Main", root=Path("/tmp/repo"))] if mode == "main" else [] + ) + orchestrator = ActionCommandOrchestrator(runtime) + orchestrator._resolve_current_worktree_target = lambda **_kwargs: current + route = Route(command="test", mode="main", flags={"services": [service_name]}) + + targets, error = orchestrator.resolve_targets(route, trees_only=False) + selected_services = orchestrator._test_service_selection(route, None, None) + + self.assertIsNone(error) + self.assertEqual(targets, [current]) + self.assertEqual(selected_services, (service_name == "backend", service_name == "frontend")) + def test_test_plan_action_delegates_to_owner(self) -> None: runtime = _RuntimeStub() runtime.config = SimpleNamespace(base_dir=Path("/tmp/repo"), raw={}) diff --git a/tests/python/planning/test_planning_menu_rendering.py b/tests/python/planning/test_planning_menu_rendering.py index b3d173168..db79f3de5 100644 --- a/tests/python/planning/test_planning_menu_rendering.py +++ b/tests/python/planning/test_planning_menu_rendering.py @@ -122,6 +122,7 @@ def test_run_flushes_pending_input_on_exit(self) -> None: patch("tty.setraw"), patch("os.read", return_value=b"\n"), patch.object(PlanningSelectionMenu, "flush_pending_input") as flush_mock, + patch.object(PlanningSelectionMenu, "_flush_input_buffer") as cleanup_flush_mock, patch.object(PlanningSelectionMenu, "_prompt_toolkit_enabled", return_value=False), redirect_stdout(StringIO()), ): @@ -133,7 +134,52 @@ def test_run_flushes_pending_input_on_exit(self) -> None: self.assertFalse(result.cancelled) self.assertEqual(result.selected_counts, {"a.md": 1}) - self.assertGreaterEqual(flush_mock.call_count, 2) + flush_mock.assert_called_once_with(fd=7) + self.assertEqual(cleanup_flush_mock.call_count, 2) + + def test_run_preserves_result_when_cleanup_drain_select_or_read_fails(self) -> None: + for key, expected_counts, expected_cancelled in ( + (b"\n", {"a.md": 1}, False), + (b"q", {}, True), + ): + for failing_operation in ("select", "read"): + with self.subTest(key=key, operation=failing_operation): + menu = PlanningSelectionMenu() + tcflush_calls = 0 + + def fail_cleanup_flush(_fd: int, _queue: int) -> None: + nonlocal tcflush_calls + tcflush_calls += 1 + if tcflush_calls > 1: + raise termios.error("stale fd") + + error = OSError("unreadable fd") + read_side_effect = [key, error, error] if failing_operation == "read" else None + select_side_effect = error if failing_operation == "select" else None + with ( + patch("sys.stdin.fileno", return_value=7), + patch("termios.tcgetattr", return_value=[0, 0, 0, 0, 0, 0]), + patch("termios.tcsetattr"), + patch("termios.tcflush", side_effect=fail_cleanup_flush), + patch("tty.setraw"), + patch("os.read", return_value=key, side_effect=read_side_effect), + patch( + "select.select", + return_value=([7], [], []), + side_effect=select_side_effect, + ), + patch.object(PlanningSelectionMenu, "_prompt_toolkit_enabled", return_value=False), + redirect_stdout(StringIO()), + ): + result = menu.run( + planning_files=["a.md"], + selected_counts={"a.md": 1}, + existing_counts={"a.md": 0}, + ) + + self.assertEqual(result.cancelled, expected_cancelled) + self.assertEqual(result.selected_counts, expected_counts) + self.assertGreaterEqual(tcflush_calls, 3) def test_run_submit_preserves_zero_counts_when_existing_worktrees_present(self) -> None: menu = PlanningSelectionMenu() From 8e3b1341822d2c1866cc90212fbbc0ecc168ecbf Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:45:55 +0300 Subject: [PATCH 22/26] fix final Docker runtime review edge cases --- .../actions/action_git_state_support.py | 8 +- .../runtime/docker_service_runtime.py | 21 ++++- .../runtime/lifecycle_dependency_stop.py | 10 +- .../startup/service_attach_execution.py | 18 ++++ .../actions/test_action_git_state_support.py | 28 +++++- .../runtime/test_docker_service_runtime.py | 27 ++++++ .../runtime/test_lifecycle_dependency_stop.py | 8 +- .../startup/test_service_attach_execution.py | 94 +++++++++++++++++++ 8 files changed, 205 insertions(+), 9 deletions(-) diff --git a/python/envctl_engine/actions/action_git_state_support.py b/python/envctl_engine/actions/action_git_state_support.py index 83fbe4738..4aa8b0944 100644 --- a/python/envctl_engine/actions/action_git_state_support.py +++ b/python/envctl_engine/actions/action_git_state_support.py @@ -93,6 +93,11 @@ def _origin_default_branch(git_root: Path, *, git_output: Callable[[Path, list[s def detect_pr_base_branch(git_root: Path, *, git_output: Callable[[Path, list[str]], str]) -> str: + for candidate in ("dev", "main", "master"): + for ref in (f"origin/{candidate}", candidate): + if git_output(git_root, ["rev-parse", "--verify", ref]).strip(): + return candidate + remote_heads = git_output( git_root, [ @@ -112,9 +117,6 @@ def detect_pr_base_branch(git_root: Path, *, git_output: Callable[[Path, list[st for candidate in ("dev", "main", "master"): if candidate in remote_branches: return candidate - for ref in (f"origin/{candidate}", candidate): - if git_output(git_root, ["rev-parse", "--verify", ref]).strip(): - return candidate return _origin_default_branch(git_root, git_output=git_output) or "master" diff --git a/python/envctl_engine/runtime/docker_service_runtime.py b/python/envctl_engine/runtime/docker_service_runtime.py index e52118116..54be280e0 100644 --- a/python/envctl_engine/runtime/docker_service_runtime.py +++ b/python/envctl_engine/runtime/docker_service_runtime.py @@ -32,6 +32,9 @@ _PUBLIC_ENV_KEYS = { "BACKEND_CORS_ORIGINS", "CORS_ORIGINS", + "CORS_ORIGINS_RAW", + "ENVCTL_SOURCE_FRONTEND_URL", + "FRONTEND_BASE_URL", "FRONTEND_URL", "PUBLIC_URL", } @@ -538,6 +541,7 @@ def _docker_build_arguments( def _container_env(self, values: Mapping[str, str]) -> dict[str, str]: result: dict[str, str] = {} + browser_facing_keys = self._browser_facing_env_keys() for raw_key, raw_value in values.items(): key = str(raw_key) if key in _HOST_ENV_KEYS or key.startswith("TERM_") or key.startswith("LC_"): @@ -549,7 +553,7 @@ def _container_env(self, values: Mapping[str, str]) -> dict[str, str]: }: result[key] = "0.0.0.0" continue - if key not in _PUBLIC_ENV_KEYS and not key.startswith(_PUBLIC_ENV_PREFIXES): + if key not in browser_facing_keys and not key.startswith(_PUBLIC_ENV_PREFIXES): value = re.sub( r"(://(?:[^/@\s]+@)?)(localhost|127\.0\.0\.1)(?=[:/]|$)", r"\1host.docker.internal", @@ -562,6 +566,21 @@ def _container_env(self, values: Mapping[str, str]) -> dict[str, str]: result["ENVCTL_HOST_GATEWAY"] = "host.docker.internal" return result + def _browser_facing_env_keys(self) -> set[str]: + keys = set(_PUBLIC_ENV_KEYS) + runtime_env = getattr(self.runtime, "env", {}) + config_raw = getattr(getattr(self.runtime, "config", None), "raw", {}) + cors_key = str( + runtime_env.get( + "ENVCTL_BACKEND_CORS_ENV_KEY", + config_raw.get("ENVCTL_BACKEND_CORS_ENV_KEY", ""), + ) + or "" + ).strip() + if cors_key: + keys.add(cors_key) + return keys + def _write_env_file(self, *, project_name: str, service_name: str, values: Mapping[str, str]) -> Path: run_dir = Path(self.runtime._run_dir_path(getattr(self.runtime, "_active_run_id", None) or "docker")) run_dir.mkdir(parents=True, exist_ok=True) diff --git a/python/envctl_engine/runtime/lifecycle_dependency_stop.py b/python/envctl_engine/runtime/lifecycle_dependency_stop.py index 4f7f5ac06..a2a9481e0 100644 --- a/python/envctl_engine/runtime/lifecycle_dependency_stop.py +++ b/python/envctl_engine/runtime/lifecycle_dependency_stop.py @@ -99,6 +99,7 @@ def stop_requirement_component_containers(runtime: Any, component: Mapping[str, if process_runner is None: return if str(component.get("id") or "").strip().lower() == "supabase": + compose_project = _supabase_compose_project(container_name) listed = process_runner.run( [ "docker", @@ -106,7 +107,7 @@ def stop_requirement_component_containers(runtime: Any, component: Mapping[str, "--all", "--quiet", "--filter", - f"label=com.docker.compose.project={container_name}", + f"label=com.docker.compose.project={compose_project}", ], timeout=30.0, ) @@ -125,3 +126,10 @@ def stop_requirement_component_containers(runtime: Any, component: Mapping[str, removed = process_runner.run(["docker", "rm", "--force", container_name], timeout=30.0) if removed.returncode != 0 and "No such container" not in str(removed.stderr or ""): raise RuntimeError(str(removed.stderr or removed.stdout or f"failed stopping {container_name}").strip()) + + +def _supabase_compose_project(container_name: str) -> str: + db_container_suffix = "-supabase-db-1" + if container_name.endswith(db_container_suffix): + return container_name[: -len(db_container_suffix)] + return container_name diff --git a/python/envctl_engine/startup/service_attach_execution.py b/python/envctl_engine/startup/service_attach_execution.py index 86aa1897d..fa3f2226e 100644 --- a/python/envctl_engine/startup/service_attach_execution.py +++ b/python/envctl_engine/startup/service_attach_execution.py @@ -278,6 +278,9 @@ def _container_actual_port( if ready: return requested if listener_expected else None detail = f"container {launch.container_name} exited or did not become ready" + cleanup_detail = self._remove_failed_container_launch(service_name, launch) + if cleanup_detail: + detail = f"{detail}; {cleanup_detail}" self.runtime._emit( "service.failure", project=self.project_name, @@ -288,6 +291,21 @@ def _container_actual_port( ) raise RuntimeError(detail) + def _remove_failed_container_launch( + self, + service_name: str, + launch: DockerServiceLaunch, + ) -> str | None: + try: + removed = DockerServiceRuntime(self.runtime, self.process_runtime).stop(launch.container_id) + except Exception as exc: # noqa: BLE001 - preserve the launch in state for a later stop retry + return f"failed to remove container: {exc}" + if not removed: + return "failed to remove container" + if self._container_launches.get(service_name) == launch: + self._container_launches.pop(service_name, None) + return None + def _annotate_container_records(self, records: dict[str, ServiceRecord]) -> None: for record in records.values(): service_name = str(record.service_slug or record.type) diff --git a/tests/python/actions/test_action_git_state_support.py b/tests/python/actions/test_action_git_state_support.py index 3afca2459..2d9974942 100644 --- a/tests/python/actions/test_action_git_state_support.py +++ b/tests/python/actions/test_action_git_state_support.py @@ -71,11 +71,35 @@ def fake_git_output(_git_root: Path, args: list[str]) -> str: self.assertEqual(resolve({"origin/master"}), "master") self.assertEqual(resolve({"master"}), "master") self.assertEqual(resolve(set()), "master") - self.assertEqual(resolve({"origin/main"}, {"dev", "main"}), "dev") - self.assertEqual(resolve({"dev"}, {"main"}), "dev") + self.assertEqual(resolve(set(), {"dev", "main"}), "dev") + self.assertEqual(resolve(set(), {"main", "master"}), "main") self.assertEqual(resolve(set(), origin_default="develop"), "develop") self.assertEqual(resolve(set(), origin_default="trunk", symbolic_default=False), "trunk") + def test_detect_pr_base_branch_does_not_probe_origin_when_local_candidate_exists(self) -> None: + calls: list[list[str]] = [] + + def local_main(_git_root: Path, args: list[str]) -> str: + calls.append(args) + if args == ["rev-parse", "--verify", "origin/main"]: + return "abc123\n" + if args[0] == "ls-remote": + raise AssertionError("remote must not be probed when a local base ref is available") + return "" + + self.assertEqual( + git_state.detect_pr_base_branch(Path("/repo"), git_output=local_main), + "main", + ) + self.assertEqual( + calls, + [ + ["rev-parse", "--verify", "origin/dev"], + ["rev-parse", "--verify", "dev"], + ["rev-parse", "--verify", "origin/main"], + ], + ) + def test_existing_pr_url_uses_gh_pr_list_for_open_branch(self) -> None: calls: list[tuple[list[str], str]] = [] diff --git a/tests/python/runtime/test_docker_service_runtime.py b/tests/python/runtime/test_docker_service_runtime.py index 0328c8fdd..84ed397aa 100644 --- a/tests/python/runtime/test_docker_service_runtime.py +++ b/tests/python/runtime/test_docker_service_runtime.py @@ -272,6 +272,33 @@ def test_configured_image_runs_without_build_and_preserves_browser_public_url(tm assert launch.image == "example/frontend:dev" +def test_container_env_preserves_browser_contract_urls_and_rewrites_internal_urls(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime( + tmp_path, + runner, + env={"ENVCTL_BACKEND_CORS_ENV_KEY": "ALLOWED_BROWSER_ORIGINS"}, + ) + + projected = DockerServiceRuntime(runtime, runner)._container_env( + { + "FRONTEND_BASE_URL": "http://localhost:5173", + "ENVCTL_SOURCE_FRONTEND_URL": "http://127.0.0.1:5173", + "CORS_ORIGINS_RAW": "http://localhost:5173,http://127.0.0.1:5173", + "ALLOWED_BROWSER_ORIGINS": "http://localhost:5173", + "DATABASE_URL": "postgres://user:pass@localhost:5432/app", + "REDIS_URL": "redis://127.0.0.1:6379/0", + } + ) + + assert projected["FRONTEND_BASE_URL"] == "http://localhost:5173" + assert projected["ENVCTL_SOURCE_FRONTEND_URL"] == "http://127.0.0.1:5173" + assert projected["CORS_ORIGINS_RAW"] == "http://localhost:5173,http://127.0.0.1:5173" + assert projected["ALLOWED_BROWSER_ORIGINS"] == "http://localhost:5173" + assert projected["DATABASE_URL"] == "postgres://user:pass@host.docker.internal:5432/app" + assert projected["REDIS_URL"] == "redis://host.docker.internal:6379/0" + + def test_remote_image_is_pulled_before_exposed_port_detection(tmp_path: Path) -> None: runner = _Runner() image = "example/frontend:remote" diff --git a/tests/python/runtime/test_lifecycle_dependency_stop.py b/tests/python/runtime/test_lifecycle_dependency_stop.py index fc5e7b4b3..da6f36971 100644 --- a/tests/python/runtime/test_lifecycle_dependency_stop.py +++ b/tests/python/runtime/test_lifecycle_dependency_stop.py @@ -45,7 +45,11 @@ def run(command, **_kwargs): # noqa: ANN001 ) stop_requirement_component_containers( runtime, - {"id": "supabase", "enabled": True, "container_name": "envctl-supabase-alpha"}, + { + "id": "supabase", + "enabled": True, + "container_name": "envctl-supabase-alpha-supabase-db-1", + }, ) self.assertIn(["docker", "rm", "--force", "envctl-redis-alpha"], calls) @@ -83,7 +87,7 @@ def run(command, **_kwargs): # noqa: ANN001 "--all", "--quiet", "--filter", - "label=com.docker.compose.project=envctl-feature-supabase-db-1", + "label=com.docker.compose.project=envctl-feature", ], ["docker", "rm", "--force", "envctl-feature-supabase-db-1"], ], diff --git a/tests/python/startup/test_service_attach_execution.py b/tests/python/startup/test_service_attach_execution.py index d9688aed7..4664803f3 100644 --- a/tests/python/startup/test_service_attach_execution.py +++ b/tests/python/startup/test_service_attach_execution.py @@ -106,6 +106,100 @@ def test_explicit_docker_mode_launches_and_annotates_container_service(self) -> ) self.assertEqual(resolved_calls, [{"service_name": "backend", "project_root": project_root, "port": 8001}]) + def test_failed_optional_docker_readiness_removes_container_before_degrading(self) -> None: + project_root = Path("/tmp/envctl-project") + optional_service = SimpleNamespace( + name="voice-runtime", + start_cmd="python voice.py --port {port}", + depends_on=(), + start_order=25, + expect_listener=True, + critical=False, + env_suffix="VOICE_RUNTIME", + ) + runtime = SimpleNamespace( + env={"ENVCTL_VOICE_RUNTIME_DOCKER_IMAGE": "example/voice:dev"}, + config=SimpleNamespace( + raw={}, + app_service_by_name=lambda name: optional_service if name == "voice-runtime" else None, + ), + services=ServiceManager(), + _conflict_remaining={}, + _emit=lambda event, **payload: None, + _split_command=lambda *args, **kwargs: self.fail("host command resolution must be skipped"), + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="Main", + project_root=project_root, + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=project_root / "backend", + frontend_cwd=project_root / "frontend", + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: {**extra, "PORT": str(port)}, + prepared_launches={ + "voice-runtime": PreparedServiceLaunch( + service_name="voice-runtime", + cwd=project_root / "voice-runtime", + log_path="/logs/voice.txt", + requested_port=8010, + env={"PORT": "8010"}, + command_source="configured", + ) + }, + selected_service_types={"voice-runtime"}, + additional_services=(optional_service,), + backend_listener_expected=True, + rebound_delta=0, + docker_mode=True, + ) + launch = DockerServiceLaunch( + container_id="failed-container-id", + container_name="envctl-app-main-voice-runtime", + image="example/voice:dev", + log_path="/logs/voice.txt", + ) + + with ( + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.launch", + return_value=launch, + ), + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.wait_until_ready", + return_value=False, + ), + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.stop", + return_value=True, + ) as stop_container, + ): + records = runner.start(attach_parallel=False, on_service_retry=lambda *args: None) + + record = records["Main Voice Runtime"] + self.assertEqual(record.status, "degraded") + self.assertTrue(record.degraded) + self.assertIsNone(record.container_id) + self.assertEqual(runner._container_launches, {}) + stop_container.assert_called_once_with("failed-container-id") + + runner._container_launches["voice-runtime"] = launch + with patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.stop", + return_value=False, + ): + self.assertEqual( + runner._remove_failed_container_launch("voice-runtime", launch), + "failed to remove container", + ) + self.assertEqual(runner._container_launches, {"voice-runtime": launch}) + def test_runner_builds_layered_descriptors_and_preserves_additional_urls(self) -> None: project_root = Path("/tmp/envctl-project") events: list[tuple[str, dict[str, object]]] = [] From 98ca67df510bd28fa513d37661b3f70d5fce047f Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:15:50 +0300 Subject: [PATCH 23/26] enable contributor CI and guidance for dev --- .github/workflows/tests.yml | 7 ++++--- docs/developer/contributing.md | 4 ++-- tests/python/runtime/test_tests_workflow_contract.py | 6 ++++++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c16a01e60..4e27655af 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,6 @@ name: Tests -# Runs the canonical contributor validation lane on every PR targeting `main`. +# Runs the canonical contributor validation lane on every PR targeting `dev` or `main`. # The lane matches `docs/developer/contributing.md`: # # uv run --extra dev pytest -q @@ -9,12 +9,12 @@ name: Tests # # `pull_request` already fires on PR open, reopen, and every push to the head # branch (the `synchronize` event), so a separate `push` trigger would just -# double the runs. Post-merge work on `main` is handled by `release.yml`. +# double the runs. Release publishing from `main` is handled explicitly by `release.yml`. # # Jobs run in parallel: # - pytest: the slow lane # - package: python -m build + release shipability gate -# - lint: ruff (non-blocking until main is green) +# - lint: ruff (non-blocking until the lint baseline is green) # # All three restore uv's cache through GitHub's official cache action keyed by # uv.lock so dependency installs reuse already-downloaded wheels without relying @@ -31,6 +31,7 @@ name: Tests on: pull_request: branches: + - dev - main permissions: diff --git a/docs/developer/contributing.md b/docs/developer/contributing.md index 649ba583e..02284bc23 100644 --- a/docs/developer/contributing.md +++ b/docs/developer/contributing.md @@ -8,7 +8,7 @@ This repository is Python-first at runtime. Most behavior changes therefore need - did the change accidentally break the supported Python runtime contract? ## Workflow -1. Create a branch from `main`. +1. Create a branch from `dev`. 2. Keep changes scoped to one objective. 3. Install the CLI in editable mode: @@ -35,7 +35,7 @@ uv run --extra dev python scripts/release_shipability_gate.py --repo . --check-t 6. Use narrower `unittest` targets only for focused local iteration; they are no longer the authoritative release-readiness lane. 7. Use conventional commits (`type(scope): subject`). -8. Open a PR with: +8. Open a PR targeting `dev` with: - `Summary` - `Validation` - `Impact` diff --git a/tests/python/runtime/test_tests_workflow_contract.py b/tests/python/runtime/test_tests_workflow_contract.py index 7cf055902..f05688b25 100644 --- a/tests/python/runtime/test_tests_workflow_contract.py +++ b/tests/python/runtime/test_tests_workflow_contract.py @@ -32,3 +32,9 @@ def test_tests_workflow_keeps_pr_checks_split_by_signal_type() -> None: assert "uv run --extra dev pytest -q" in workflow assert "uv run --extra dev python -m build" in workflow assert "uv tool run ruff check python tests scripts" in workflow + + +def test_tests_workflow_runs_for_dev_and_main_pull_requests() -> None: + workflow = _tests_workflow() + + assert "pull_request:\n branches:\n - dev\n - main" in workflow From d23a38c01c9c88f4bb8c61f842149697474af185 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:18:09 +0300 Subject: [PATCH 24/26] tolerate transient empty GitHub check rollups --- .../actions/action_ship_checks.py | 41 +++++++++++++++++++ .../python/actions/test_action_ship_checks.py | 40 ++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/python/envctl_engine/actions/action_ship_checks.py b/python/envctl_engine/actions/action_ship_checks.py index 56bc6e953..3167c02f0 100644 --- a/python/envctl_engine/actions/action_ship_checks.py +++ b/python/envctl_engine/actions/action_ship_checks.py @@ -99,8 +99,13 @@ class ShipCheckPoller: def run(self) -> dict[str, object]: next_progress_at = self.timing.progress_interval_seconds + last_observed_checks: Mapping[str, object] | None = None while True: result = self._query() + if _result_reports_target_checks(result): + last_observed_checks = result + elif result.get("state") == "no_checks_reported" and last_observed_checks is not None: + result = _pending_after_transient_empty_rollup(result, last_observed_checks) if _ship_check_result_is_terminal(result): return result @@ -238,6 +243,42 @@ def _ship_check_result_is_terminal(result: Mapping[str, object]) -> bool: return not _failed_check_logs_are_retryable(result) +def _result_reports_target_checks(result: Mapping[str, object]) -> bool: + synthetic_waiters = {"github_checks", "github_head_ref"} + for key in ("failing_checks", "passed_checks", "pending_checks"): + checks = result.get(key) + if not isinstance(checks, list): + continue + for check in checks: + if not isinstance(check, Mapping): + continue + name = str(check.get("name") or "").strip() + if name and name not in synthetic_waiters: + return True + return False + + +def _pending_after_transient_empty_rollup( + result: Mapping[str, object], + last_observed: Mapping[str, object], +) -> dict[str, object]: + pending = last_observed.get("pending_checks") + pending_checks = list(pending) if isinstance(pending, list) and pending else [ + {"name": "github_checks", "state": "WAITING"} + ] + passed = last_observed.get("passed_checks") + pr_checks = last_observed.get("pr_checks") + return { + **result, + "state": "checks_pending_timeout", + "failing_checks": [], + "passed_checks": list(passed) if isinstance(passed, list) else [], + "pending_checks": pending_checks, + "pr_checks": list(pr_checks) if isinstance(pr_checks, list) else [], + "error": "GitHub temporarily returned an empty check rollup after target checks were already observed.", + } + + def _check_progress_message( result: Mapping[str, object], *, diff --git a/tests/python/actions/test_action_ship_checks.py b/tests/python/actions/test_action_ship_checks.py index 4b70a2af6..1e4c1b32b 100644 --- a/tests/python/actions/test_action_ship_checks.py +++ b/tests/python/actions/test_action_ship_checks.py @@ -299,6 +299,46 @@ def fake_sleep(seconds: float) -> None: assert checks["duration_seconds"] == 2.0 assert updates == [] + +def test_github_pr_checks_ignores_transient_empty_rollup_after_checks_start(tmp_path: Path) -> None: + clock = {"now": 0.0} + outputs = [ + { + "headRefOid": "newsha", + "statusCheckRollup": [{"name": "pytest", "workflowName": "Tests", "status": "IN_PROGRESS"}], + }, + {"headRefOid": "newsha", "statusCheckRollup": []}, + { + "headRefOid": "newsha", + "statusCheckRollup": [ + {"name": "pytest", "workflowName": "Tests", "status": "COMPLETED", "conclusion": "SUCCESS"} + ], + }, + ] + + def fake_run(args: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + if args[:2] == ["/usr/bin/gh", "api"]: + return subprocess.CompletedProcess(args=args, returncode=0, stdout="[]", stderr="") + return subprocess.CompletedProcess(args=args, returncode=0, stdout=json.dumps(outputs.pop(0)), stderr="") + + def fake_sleep(seconds: float) -> None: + clock["now"] += seconds + + checks = github_pr_checks( + tmp_path, + expected_head_sha="newsha", + timeout_seconds=30.0, + poll_interval_seconds=1.0, + no_checks_grace_seconds=0.0, + run_command=fake_run, + monotonic=lambda: clock["now"], + sleep=fake_sleep, + ) + + assert checks["state"] == "checks_passed" + assert checks["duration_seconds"] == 2.0 + assert outputs == [] + def test_github_pr_checks_waits_for_expected_head_sha_before_accepting_rollup(tmp_path: Path) -> None: calls: list[list[str]] = [] outputs = [ From 50c688c28775aaad4f24dde7459e13a6fb375ff1 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:20:58 +0300 Subject: [PATCH 25/26] allow full CI three minutes to finish --- docs/reference/commands.md | 2 +- python/envctl_engine/actions/action_ship_checks.py | 2 +- tests/python/actions/test_action_ship_checks.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 5787b12d4..2df44f879 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -572,7 +572,7 @@ If GitHub has attached the pushed head commit but still has no target test check `no_checks_reported` immediately instead of waiting for the full timeout. The command returns the structured JSON result by default including the PR URL, `pr_created`, `operation_statuses`, `checks_state`, `passed_checks`, `failing_checks`, `pending_checks`, and `checks_error`. `--json` remains accepted as a compatibility no-op; -use `--human` only when compact terminal output is preferred. The check wait timeout defaults to 2 minutes; pending +use `--human` only when compact terminal output is preferred. The check wait timeout defaults to 3 minutes; pending checks after that window are reported as `checks_pending_timeout` without failing the handoff command, while actual failed checks still return a non-zero exit. The timeout can be tuned with `ENVCTL_SHIP_CHECK_TIMEOUT_SECONDS` and diff --git a/python/envctl_engine/actions/action_ship_checks.py b/python/envctl_engine/actions/action_ship_checks.py index 3167c02f0..26b0f1dd0 100644 --- a/python/envctl_engine/actions/action_ship_checks.py +++ b/python/envctl_engine/actions/action_ship_checks.py @@ -30,7 +30,7 @@ TERMINAL_SHIP_CHECK_STATES = {"checks_passed", "checks_failed", "gh_unavailable", "no_checks_reported"} SHIP_NO_CHECKS_GRACE_ENV = "ENVCTL_SHIP_NO_CHECKS_GRACE_SECONDS" -DEFAULT_CHECK_TIMEOUT_SECONDS = 120.0 +DEFAULT_CHECK_TIMEOUT_SECONDS = 180.0 DEFAULT_CHECK_POLL_INTERVAL_SECONDS = 5.0 DEFAULT_CHECK_PROGRESS_INTERVAL_SECONDS = 10.0 DEFAULT_NO_CHECKS_GRACE_SECONDS = 15.0 diff --git a/tests/python/actions/test_action_ship_checks.py b/tests/python/actions/test_action_ship_checks.py index 1e4c1b32b..aea5d77a6 100644 --- a/tests/python/actions/test_action_ship_checks.py +++ b/tests/python/actions/test_action_ship_checks.py @@ -77,8 +77,8 @@ def fail_run(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[s "duration_seconds": 0.0, } -def test_github_pr_checks_default_timeout_is_two_minute_wait_window() -> None: - assert action_ship_checks.DEFAULT_CHECK_TIMEOUT_SECONDS == 120.0 +def test_github_pr_checks_default_timeout_is_three_minute_wait_window() -> None: + assert action_ship_checks.DEFAULT_CHECK_TIMEOUT_SECONDS == 180.0 def test_github_pr_checks_default_no_checks_grace_is_fifteen_seconds() -> None: assert action_ship_checks.DEFAULT_NO_CHECKS_GRACE_SECONDS == 15.0 From 7194164f7b98d7e6073d0e6a985bad9958ffb8d6 Mon Sep 17 00:00:00 2001 From: kfiramar <83420275+kfiramar@users.noreply.github.com> Date: Sun, 12 Jul 2026 04:54:08 +0300 Subject: [PATCH 26/26] Consolidate open PRs and harden runtime lifecycle ownership --- contracts/python_runtime_gap_report.json | 2 +- contracts/runtime_feature_matrix.json | 4 +- .../actions/action_target_support.py | 7 +- .../plan_agent/workflow_runtime_addresses.py | 6 +- python/envctl_engine/pr_preview_controller.py | 137 ++- .../runtime/browser_runtime_diagnostics.py | 24 +- .../runtime/command_flag_storage.py | 1 + .../runtime/command_special_flags.py | 2 - .../runtime/docker_service_runtime.py | 512 +++++++++- .../runtime/endpoints_command_support.py | 4 + .../runtime/engine_runtime_artifacts.py | 12 +- .../runtime/engine_runtime_hooks.py | 8 +- .../runtime/engine_runtime_startup_facade.py | 13 +- .../runtime/engine_runtime_startup_support.py | 17 - .../runtime/inspection_support.py | 7 +- .../runtime/lifecycle_cleanup_orchestrator.py | 294 +++--- .../runtime/lifecycle_dependency_stop.py | 38 +- .../runtime/lifecycle_full_cleanup.py | 139 +++ .../runtime/lifecycle_requirement_ports.py | 16 +- .../lifecycle_service_stop_selection.py | 13 +- .../runtime/lifecycle_service_termination.py | 42 +- .../runtime/lifecycle_targeted_state.py | 10 +- .../runtime/lifecycle_worktree_cleanup.py | 40 +- .../runtime/requirement_reconcile_truth.py | 7 +- .../envctl_engine/runtime/service_manager.py | 179 +++- ...e_feature_lifecycle_command_definitions.py | 2 + python/envctl_engine/shared/ports.py | 16 +- .../shared/process_launch_support.py | 219 ++++- python/envctl_engine/shared/process_runner.py | 113 ++- python/envctl_engine/shared/protocols.py | 1 + python/envctl_engine/shared/services.py | 67 +- python/envctl_engine/startup/finalization.py | 38 +- .../startup/finalization_failure.py | 167 +++- .../startup/finalization_run_state.py | 48 +- python/envctl_engine/startup/lifecycle.py | 55 +- python/envctl_engine/startup/protocols.py | 9 +- .../startup/requirements_execution.py | 3 +- .../startup/restart_prestop_support.py | 209 +++- .../startup/resume_orchestrator.py | 20 +- .../startup/resume_restore_execution.py | 22 +- .../startup/resume_restore_policy.py | 44 +- .../startup/resume_restore_project.py | 123 ++- .../startup/resume_restore_results.py | 93 +- .../startup/run_reuse_dashboard_restore.py | 102 +- .../startup/run_reuse_decision.py | 7 +- .../startup/run_reuse_fresh_start.py | 126 ++- .../startup/run_reuse_identity.py | 13 +- .../startup/run_reuse_resolution.py | 131 +-- .../envctl_engine/startup/run_reuse_resume.py | 100 +- .../startup/selected_context_startup.py | 396 ++++++-- .../startup/service_attach_execution.py | 136 ++- python/envctl_engine/startup/session.py | 160 ++- .../startup/startup_execution_support.py | 13 +- .../startup/startup_selection_support.py | 519 +++++++++- python/envctl_engine/state/__init__.py | 42 +- .../state/action_health_support.py | 10 +- .../state/action_orchestrator.py | 26 +- python/envctl_engine/state/models.py | 2 + python/envctl_engine/state/project_runtime.py | 59 +- python/envctl_engine/state/repository.py | 92 +- python/envctl_engine/state/runtime_map.py | 259 +++-- .../envctl_engine/state/state_aggregation.py | 210 +++- .../ui/dashboard/dependency_rendering.py | 30 +- .../ui/dashboard/pr_link_rendering.py | 5 +- .../ui/dashboard/restart_selection_support.py | 11 +- .../ui/dashboard/snapshot_rendering.py | 5 +- .../ui/dashboard/stop_scope_support.py | 8 +- python/envctl_engine/ui/selection_support.py | 26 +- .../actions/test_action_target_support.py | 21 + ...t_plan_agent_workflow_runtime_addresses.py | 8 + .../runtime/lifecycle_parity_test_support.py | 5 +- .../runtime/test_command_flag_storage.py | 1 + .../runtime/test_docker_service_runtime.py | 930 +++++++++++++++++- .../runtime/test_engine_runtime_artifacts.py | 12 +- ...ine_runtime_command_parity_state_config.py | 20 + .../runtime/test_engine_runtime_hooks.py | 13 +- .../test_engine_runtime_lifecycle_support.py | 4 +- .../test_engine_runtime_real_startup.py | 4 +- .../test_engine_runtime_startup_support.py | 56 ++ .../test_engine_runtime_state_truth.py | 15 + ...t_lifecycle_cleanup_spinner_integration.py | 322 +++++- .../runtime/test_lifecycle_dependency_stop.py | 20 + .../test_lifecycle_parity_restore_startup.py | 2 +- .../test_lifecycle_parity_resume_policy.py | 12 +- .../test_lifecycle_requirement_ports.py | 42 + .../test_lifecycle_service_stop_selection.py | 39 + .../test_lifecycle_service_termination.py | 99 ++ .../test_lifecycle_worktree_cleanup.py | 139 ++- .../runtime/test_pr_preview_controller.py | 112 +++ .../test_runtime_browser_diagnostics.py | 13 + .../shared/test_ports_lock_reclamation.py | 73 +- .../test_process_runner_launch_policy.py | 309 +++++- .../test_process_runner_listener_detection.py | 29 +- .../shared/test_structure_runtime_layout.py | 10 + .../test_utility_consolidation_contract.py | 48 +- .../startup/test_frontend_projection.py | 161 +++ .../startup/test_restart_prestop_support.py | 700 ++++++++++++- .../startup/test_resume_restore_policy.py | 175 ++++ .../python/startup/test_run_reuse_identity.py | 27 + .../python/startup/test_run_reuse_support.py | 194 +++- .../startup/test_selected_context_startup.py | 327 +++++- .../startup/test_service_attach_execution.py | 146 +++ tests/python/startup/test_service_manager.py | 187 ++++ .../startup/test_startup_finalization.py | 337 ++++++- .../test_startup_fingerprint_replacement.py | 3 +- .../test_startup_orchestrator_flow_reuse.py | 10 +- .../test_startup_orchestrator_profiles.py | 107 ++ .../test_startup_restart_port_preservation.py | 12 +- .../test_startup_restart_service_scope.py | 12 +- .../test_startup_run_reuse_resolution.py | 64 +- .../startup/test_startup_selection_support.py | 100 ++ .../startup/test_support_module_decoupling.py | 319 +++++- .../state/test_project_runtime_resolution.py | 15 + .../test_state_action_orchestrator_logs.py | 29 + tests/python/state/test_state_aggregation.py | 170 +++- .../state/test_state_repository_contract.py | 177 +++- tests/python/state/test_state_roundtrip.py | 56 ++ .../ui/test_dashboard_pr_link_rendering.py | 21 + ...est_dashboard_restart_selection_support.py | 13 + tests/python/ui/test_selection_support.py | 24 + 120 files changed, 10002 insertions(+), 1013 deletions(-) create mode 100644 python/envctl_engine/runtime/lifecycle_full_cleanup.py diff --git a/contracts/python_runtime_gap_report.json b/contracts/python_runtime_gap_report.json index 8c7270b80..3a03a2972 100644 --- a/contracts/python_runtime_gap_report.json +++ b/contracts/python_runtime_gap_report.json @@ -2,7 +2,7 @@ "version": 1, "generated_at": "2026-07-10T21:33:31+00:00", "matrix_generated_at": "2026-07-10T21:30:49+00:00", - "matrix_sha256": "9968ef658488512f15c0ebda73d08c82da359ab389b1ff880b585dd47d5bb935", + "matrix_sha256": "6ecf2e6088f7d7a8926cc75e72d4376423b3a171c47406c9f995a5c368e48027", "summary": { "feature_count": 59, "gap_count": 0, diff --git a/contracts/runtime_feature_matrix.json b/contracts/runtime_feature_matrix.json index 44926b96c..d1e99da6b 100644 --- a/contracts/runtime_feature_matrix.json +++ b/contracts/runtime_feature_matrix.json @@ -420,6 +420,7 @@ "python_source_of_truth": [ "python/envctl_engine/runtime/engine_runtime.py", "python/envctl_engine/runtime/engine_runtime_lifecycle_support.py", + "python/envctl_engine/runtime/lifecycle_full_cleanup.py", "python/envctl_engine/runtime/lifecycle_blast_support.py", "python/envctl_engine/runtime/lifecycle_blast_ports.py", "python/envctl_engine/runtime/lifecycle_blast_processes.py", @@ -1312,7 +1313,8 @@ "shell_source_of_truth": [], "python_source_of_truth": [ "python/envctl_engine/runtime/engine_runtime.py", - "python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py" + "python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py", + "python/envctl_engine/runtime/lifecycle_full_cleanup.py" ], "evidence_tests": [ "tests/python/runtime/test_lifecycle_parity_stop_health.py" diff --git a/python/envctl_engine/actions/action_target_support.py b/python/envctl_engine/actions/action_target_support.py index ff1392842..2a52b5b59 100644 --- a/python/envctl_engine/actions/action_target_support.py +++ b/python/envctl_engine/actions/action_target_support.py @@ -5,6 +5,7 @@ from typing import Any, Callable, Mapping, Sequence, cast from envctl_engine.runtime.command_router import Route +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.ui.selection_types import TargetSelection @@ -114,7 +115,11 @@ def _projects_by_service_selector(runtime: Any, state: object | None) -> dict[st projects_by_selector: dict[str, list[str]] = {} services = getattr(state, "services", {}) if state is not None else {} for service_name, service in services.items(): - project = runtime.project_name_from_service(service_name) + project = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime.project_name_from_service, + ) if not project: continue service_type = str(getattr(service, "type", "") or "").strip().lower() diff --git a/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py b/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py index 43621fe42..396ea959d 100644 --- a/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py +++ b/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py @@ -98,7 +98,8 @@ def _state_lookup_project_attempts(project_names: list[str] | None) -> tuple[lis def _dependency_address_lines(state: RunState, *, worktree: CreatedPlanWorktree | None = None) -> list[str]: rows: list[str] = [] seen: set[tuple[str, int]] = set() - for project_name, requirements in state.requirements.items(): + for storage_key, requirements in state.requirements.items(): + project_name = str(getattr(requirements, "project", "") or storage_key).strip() if not _state_project_matches_worktree(project_name, worktree): continue for dependency_id in ("postgres", "redis", "supabase", "n8n"): @@ -146,6 +147,9 @@ def _state_project_matches_worktree(project_name: object, worktree: CreatedPlanW def _state_service_matches_worktree(service: object, worktree: CreatedPlanWorktree | None) -> bool: if worktree is None: return True + project = str(getattr(service, "project", "") or "").strip() + if project and _state_project_matches_worktree(project, worktree): + return True service_name = str(getattr(service, "name", "") or "").strip() worktree_name = str(worktree.name or "").strip() if worktree_name and service_name.casefold().startswith(f"{worktree_name.casefold()} "): diff --git a/python/envctl_engine/pr_preview_controller.py b/python/envctl_engine/pr_preview_controller.py index e86fe7146..3cee69855 100644 --- a/python/envctl_engine/pr_preview_controller.py +++ b/python/envctl_engine/pr_preview_controller.py @@ -6,6 +6,7 @@ import argparse from dataclasses import asdict, dataclass, field, fields, replace from datetime import UTC, datetime, timedelta +import hashlib import json import os from pathlib import Path @@ -857,6 +858,91 @@ def stable_system_python_bin() -> str | None: return shutil.which("python3") +def prepare_python_command_shim( + env: dict[str, str], + *, + runtime_root: Path, +) -> Path: + """Expose the selected Python 3 interpreter under the legacy ``python`` name. + + Imported PR branches can contain launch scripts written before envctl + propagated ``PYTHON_BIN``. Self-hosted runners commonly provide only + ``python3``. A controller-owned, private PATH shim lets those scripts use + the selected interpreter without modifying the imported worktree. + """ + + configured = str(env.get("PYTHON_BIN") or stable_system_python_bin() or "").strip() + if not configured: + raise RuntimeError("No Python 3 interpreter is available for PR preview startup") + candidate = Path(configured).expanduser() + if not candidate.is_absolute(): + resolved_command = shutil.which(configured, path=env.get("PATH")) + if not resolved_command: + raise RuntimeError(f"Configured preview Python interpreter is not executable: {configured}") + candidate = Path(resolved_command) + try: + python_bin = candidate.resolve(strict=True) + except OSError as exc: + raise RuntimeError(f"Configured preview Python interpreter does not exist: {configured}") from exc + if not python_bin.is_file() or not os.access(python_bin, os.X_OK): + raise RuntimeError(f"Configured preview Python interpreter is not executable: {python_bin}") + + try: + probe = subprocess.run( + [ + str(python_bin), + "-c", + "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ], + check=False, + capture_output=True, + text=True, + timeout=10.0, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise RuntimeError(f"Configured preview interpreter could not be validated: {python_bin}") from exc + reported_version = str(probe.stdout or "").strip() + if probe.returncode != 0 or re.fullmatch(r"3\.\d+", reported_version) is None: + raise RuntimeError(f"Configured preview interpreter is not Python 3: {python_bin}") + + _ensure_real_directory(runtime_root) + shim_root = runtime_root / "python-command-shims" + _ensure_real_directory(shim_root, private=True) + interpreter_key = hashlib.sha256(str(python_bin).encode("utf-8")).hexdigest()[:20] + shim_dir = shim_root / interpreter_key + _ensure_real_directory(shim_dir, private=True) + shim_path = shim_dir / "python" + temporary = shim_dir / f".python-{os.getpid()}-{secrets.token_hex(8)}" + try: + temporary.symlink_to(python_bin) + os.replace(temporary, shim_path) + finally: + temporary.unlink(missing_ok=True) + + env["PYTHON_BIN"] = str(python_bin) + env["ENVCTL_PYTHON_EXECUTABLE"] = str(python_bin) + existing_path = str(env.get("PATH") or "") + legacy_shim_dir = runtime_root / "python-command-shim" + path_parts = [ + part + for part in existing_path.split(os.pathsep) + if part and Path(part) != legacy_shim_dir and Path(part).parent != shim_root + ] + env["PATH"] = os.pathsep.join((str(shim_dir), *path_parts)) + return shim_path + + +def _ensure_real_directory(path: Path, *, private: bool = False) -> None: + try: + path.mkdir(parents=True, exist_ok=False, mode=0o700 if private else 0o755) + except FileExistsError: + pass + if path.is_symlink() or not path.is_dir(): + raise RuntimeError(f"PR preview runtime path is not a real directory: {path}") + if private: + path.chmod(0o700) + + def normalize_envctl_config_value(raw: str) -> str: value = raw.strip() if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: @@ -1989,7 +2075,16 @@ def run_import() -> CommandResult: start_env = self.envctl_command_env() start_env.update(pr_preview_start_env_overrides()) start_env.update(self.external_dependency_start_env(external_dependencies)) - self.reset_incompatible_backend_venv(project, start_env) + python_setup_error = "" + try: + prepare_python_command_shim( + start_env, + runtime_root=self.config.preview_root / "runtime", + ) + except (OSError, RuntimeError) as exc: + python_setup_error = str(exc) + if not python_setup_error: + self.reset_incompatible_backend_venv(project, start_env) public_urls = self.public_route_urls(pr.number) if public_urls.get("backend"): start_env["ENVCTL_BACKEND_ENV__FRONTEND_BASE_URL"] = public_urls["frontend"] @@ -2005,21 +2100,31 @@ def run_import() -> CommandResult: start_env["ENVCTL_FRONTEND_ENV__VITE_SUPABASE_URL"] = public_urls["supabase"] self.apply_repository_preview_env(start_env, pr_number=pr.number) - start_result = self.runner.run( - [ - self.config.envctl_bin, - "start", - "--trees", - "--project", - str(project["name"]), - "--headless", - "--entire-system", - "--isolated-deps", - "--copy-db-storage", - ], - cwd=self.config.control_repo, - check=False, - env=start_env, + start_argv = [ + self.config.envctl_bin, + "start", + "--trees", + "--project", + str(project["name"]), + "--headless", + "--entire-system", + "--isolated-deps", + "--copy-db-storage", + ] + start_result = ( + CommandResult( + argv=start_argv, + returncode=1, + stdout="", + stderr=f"PR preview Python compatibility setup failed: {python_setup_error}", + ) + if python_setup_error + else self.runner.run( + start_argv, + cwd=self.config.control_repo, + check=False, + env=start_env, + ) ) if start_result.returncode != 0: state = PreviewState( diff --git a/python/envctl_engine/runtime/browser_runtime_diagnostics.py b/python/envctl_engine/runtime/browser_runtime_diagnostics.py index 974778e6a..9ca96a759 100644 --- a/python/envctl_engine/runtime/browser_runtime_diagnostics.py +++ b/python/envctl_engine/runtime/browser_runtime_diagnostics.py @@ -79,15 +79,25 @@ def build_runtime_diagnostics( def project_names(state: RunState, *, runtime: Any | None) -> list[str]: - names: set[str] = set(state.requirements) + names: dict[str, str] = {} + + def add(raw_name: object) -> None: + name = str(raw_name or "").strip() + if name: + names.setdefault(name.casefold(), name) + + for storage_key, requirements in state.requirements.items(): + add(getattr(requirements, "project", "") or storage_key) roots = state.metadata.get("project_roots") if isinstance(roots, dict): - names.update(str(name) for name in roots) + for name in roots: + add(name) for service_name, service in state.services.items(): - names.add(service_project_name(service_name, service, runtime=runtime)) + add(service_project_name(service_name, service, runtime=runtime)) launch = launch_metadata(state) - names.update(launch) - return sorted(name for name in names if name) + for name in launch: + add(name) + return sorted(names.values(), key=lambda name: (name.casefold(), name)) def service_payload(service: ServiceRecord | None) -> dict[str, object]: @@ -159,6 +169,10 @@ def safe_launch_env(raw_launch: object, *, service_name: str) -> dict[str, str]: def requirements_for_project(state: RunState, project: str) -> RequirementsResult | None: if project in state.requirements: return state.requirements[project] + target = str(project).strip().casefold() + for storage_key, requirements in state.requirements.items(): + if str(requirements.project or storage_key).strip().casefold() == target: + return requirements if len(state.requirements) == 1: return next(iter(state.requirements.values())) return None diff --git a/python/envctl_engine/runtime/command_flag_storage.py b/python/envctl_engine/runtime/command_flag_storage.py index ddafa71b6..26463a55b 100644 --- a/python/envctl_engine/runtime/command_flag_storage.py +++ b/python/envctl_engine/runtime/command_flag_storage.py @@ -41,6 +41,7 @@ def boolean_flag_name(token: str) -> str: "--test-parallel": "test_parallel", "--service-parallel": "service_parallel", "--service-prep-parallel": "service_prep_parallel", + "--ignore-service-deps": "ignore_service_deps", "--refresh-cache": "refresh_cache", "--fast": "fast", "--fast-startup": "fast", diff --git a/python/envctl_engine/runtime/command_special_flags.py b/python/envctl_engine/runtime/command_special_flags.py index b232c950a..9056f0e7b 100644 --- a/python/envctl_engine/runtime/command_special_flags.py +++ b/python/envctl_engine/runtime/command_special_flags.py @@ -64,8 +64,6 @@ def handle_special_flag(flags: dict[str, object], token: str) -> None: "--no-requirements-parallel", }: flags["requirements_parallel"] = False - elif token == "--ignore-service-deps": - flags["ignore_service_deps"] = True elif token == "--only-backend": flags["launch_backend"] = True flags["launch_frontend"] = False diff --git a/python/envctl_engine/runtime/docker_service_runtime.py b/python/envctl_engine/runtime/docker_service_runtime.py index 54be280e0..c85f10773 100644 --- a/python/envctl_engine/runtime/docker_service_runtime.py +++ b/python/envctl_engine/runtime/docker_service_runtime.py @@ -2,13 +2,17 @@ import hashlib import json +import math import os import re import shlex +import tempfile import threading +import time +import uuid from dataclasses import dataclass from pathlib import Path -from typing import Any, Mapping, Sequence +from typing import Any, Callable, Mapping, Sequence from envctl_engine.shared.parsing import parse_bool @@ -40,6 +44,11 @@ } _DOCKER_INFO_LOCK = threading.Lock() _RUNTIME_SCOPE_LABEL = "io.envctl.runtime-scope" +_LAUNCH_TOKEN_LABEL = "io.envctl.launch-token" +_ENV_FILE_PID_PATTERN = re.compile( + r"^\.docker-env-[0-9a-f]{12}-p(?P[1-9][0-9]*)-[A-Za-z0-9_]+$" +) +_SECRET_SCRUB_FAILED = "secret scrub failed" def _docker_name(value: str, *, limit: int) -> str: @@ -59,6 +68,16 @@ class DockerServiceLaunch: container_name: str image: str log_path: str + launch_token: str = "" + cleanup_pending_since: float | None = None + + +class DockerServiceLaunchCleanupError(RuntimeError): + """A failed launch whose container cleanup could not be confirmed.""" + + def __init__(self, message: str, launch: DockerServiceLaunch) -> None: + super().__init__(message) + self.launch = launch def docker_service_mode_enabled(runtime: Any) -> bool: @@ -137,6 +156,7 @@ def launch( container_port: int, listener_expected: bool, log_path: str, + on_confirmed: Callable[[DockerServiceLaunch], None] | None = None, ) -> DockerServiceLaunch: self._require_docker() image = self._resolve_image( @@ -160,6 +180,10 @@ def launch( service_name=service_name, values=container_env, ) + launch_token = uuid.uuid4().hex + result: object + run_error: BaseException | None = None + docker_run_invoked = False try: docker_command = [ "docker", @@ -175,6 +199,8 @@ def launch( f"io.envctl.service={service_name}", "--label", f"{_RUNTIME_SCOPE_LABEL}={self._runtime_scope_identity()}", + "--label", + f"{_LAUNCH_TOKEN_LABEL}={launch_token}", "--add-host", "host.docker.internal:host-gateway", "--env-file", @@ -196,35 +222,262 @@ def launch( container_port=effective_container_port, ) ) + docker_run_invoked = True result = self.process_runtime.run( docker_command, cwd=project_root, env={**os.environ, "DOCKER_BUILDKIT": "1"}, timeout=self._timeout("ENVCTL_DOCKER_RUN_TIMEOUT_SECONDS", 120.0), ) + except BaseException as exc: # noqa: BLE001 - cancellation still requires token cleanup + run_error = exc + result = None finally: - env_file.unlink(missing_ok=True) - if result.returncode != 0: - detail = str(result.stderr or result.stdout or "docker run failed").strip() - raise RuntimeError(detail) - container_id = str(result.stdout or "").strip().splitlines()[-1] - if not container_id: - raise RuntimeError(f"docker run returned no container id for {project_name} {service_name}") - launch = DockerServiceLaunch( - container_id=container_id, + env_cleanup_error = self._scrub_env_file_safely(env_file) + if env_cleanup_error: + self._emit_best_effort( + "service.container.env_file_cleanup_warning", + project=project_name, + service=service_name, + detail=env_cleanup_error, + ) + provisional_launch = DockerServiceLaunch( + container_id=container_name, container_name=container_name, image=image, log_path=log_path, + launch_token=launch_token, + cleanup_pending_since=time.time(), ) - self.runtime._emit( - "service.container.start", - project=project_name, - service=service_name, - container_name=container_name, - image=image, - port=host_port if listener_expected else None, + if env_cleanup_error and _SECRET_SCRUB_FAILED in env_cleanup_error: + cleanup_detail = f"Docker env file cleanup failed: {env_cleanup_error}" + if not docker_run_invoked: + if run_error is not None: + raise RuntimeError(f"{run_error}; {cleanup_detail}") from run_error + raise RuntimeError(cleanup_detail) + self._raise_launch_error_after_cleanup( + cleanup_detail, + provisional_launch, + absence_can_confirm=self._definite_docker_rejection(result), + ) + if run_error is not None: + if not docker_run_invoked: + if isinstance(run_error, Exception): + raise RuntimeError(str(run_error)) from run_error + raise run_error + run_error_detail = str(run_error).strip() or type(run_error).__name__ + self._raise_launch_error_after_cleanup( + run_error_detail, + provisional_launch, + cause=run_error, + ) + if result is None: + self._raise_launch_error_after_cleanup("docker run returned no result", provisional_launch) + returncode = getattr(result, "returncode", None) + if not isinstance(returncode, int): + self._raise_launch_error_after_cleanup("docker run returned an invalid result", provisional_launch) + if returncode != 0: + detail = str( + getattr(result, "stderr", "") + or getattr(result, "stdout", "") + or "docker run failed" + ).strip() + self._raise_launch_error_after_cleanup( + detail, + provisional_launch, + absence_can_confirm=self._definite_docker_rejection(result), + ) + try: + output_lines = [ + line.strip() + for line in str(getattr(result, "stdout", "") or "").splitlines() + if line.strip() + ] + output_identity = output_lines[-1] if output_lines else "" + identity_ref = ( + output_identity + if re.fullmatch(r"[0-9a-fA-F]{64}", output_identity) + else container_name + ) + ownership, container_id = self._container_label_identity( + identity_ref, + label=_LAUNCH_TOKEN_LABEL, + expected_value=launch_token, + ) + except BaseException as exc: # noqa: BLE001 - post-run authority is still transactional + self._raise_launch_error_after_cleanup( + str(exc).strip() or type(exc).__name__, + provisional_launch, + cause=exc, + ) + if ownership != "owned" or container_id is None: + self._raise_launch_error_after_cleanup( + "docker run completed but the launched container identity could not be confirmed", + provisional_launch, + ) + try: + launch = DockerServiceLaunch( + container_id=container_id, + container_name=container_name, + image=image, + log_path=log_path, + launch_token=launch_token, + ) + if on_confirmed is not None: + on_confirmed(launch) + self._emit_best_effort( + "service.container.start", + project=project_name, + service=service_name, + container_name=container_name, + image=image, + port=host_port if listener_expected else None, + ) + return launch + except BaseException as exc: # noqa: BLE001 - confirmed identity was not handed off + self._raise_launch_error_after_cleanup( + str(exc).strip() or type(exc).__name__, + provisional_launch, + cause=exc, + ) + + def _raise_launch_error_after_cleanup( + self, + detail: str, + launch: DockerServiceLaunch, + *, + cause: BaseException | None = None, + absence_can_confirm: bool = False, + ) -> None: + cleanup_error: BaseException | None = None + try: + cleanup_confirmed = self._cleanup_failed_launch( + launch, + absence_can_confirm=absence_can_confirm, + ) + except BaseException as exc: # noqa: BLE001 - preserve authority across cleanup cancellation + cleanup_confirmed = False + cleanup_error = exc + if cleanup_confirmed: + if cause is not None: + if not isinstance(cause, Exception): + raise cause + raise RuntimeError(detail) from cause + raise RuntimeError(detail) + error = DockerServiceLaunchCleanupError( + f"{detail}; Docker cleanup could not confirm removal of {launch.container_name}", + launch, + ) + chained_cause = cause if cause is not None else cleanup_error + if chained_cause is not None: + raise error from chained_cause + raise error + + def _cleanup_failed_launch( + self, + launch: DockerServiceLaunch, + *, + absence_can_confirm: bool, + remove: bool = True, + ) -> bool: + deadline = time.monotonic() + self._timeout("ENVCTL_DOCKER_CLEANUP_DISCOVERY_SECONDS", 1.0) + while True: + try: + token_cleanup = self._cleanup_launch_token_matches( + launch.launch_token, + remove=remove, + ) + except BaseException: # noqa: BLE001 - cancellation cannot abandon authority + return False + if token_cleanup == "unknown": + return False + if token_cleanup == "removed": + return True + + try: + ownership, container_id = self._container_label_identity( + launch.container_name, + label=_LAUNCH_TOKEN_LABEL, + expected_value=launch.launch_token, + ) + except BaseException: # noqa: BLE001 - cancellation cannot abandon authority + return False + if ownership == "owned" and container_id is not None: + try: + return self._remove_container_identity(container_id, remove=remove) + except BaseException: # noqa: BLE001 - cancellation cannot abandon authority + return False + if ownership == "unknown": + return False + + remaining = deadline - time.monotonic() + if remaining <= 0: + # A token-filtered daemon query plus repeated name probes stayed + # absent for the whole discovery window. This covers definite + # Docker failures without losing a timeout-accepted container + # that becomes visible shortly after its client is terminated. + return absence_can_confirm + time.sleep(min(0.1, remaining)) + + @staticmethod + def _definite_docker_rejection(result: object | None) -> bool: + returncode = getattr(result, "returncode", None) + if not isinstance(returncode, int) or returncode in {0, 124}: + return False + detail = str(getattr(result, "stderr", "") or getattr(result, "stdout", "")).lower() + return "timed out" not in detail and "timeout" not in detail + + def _cleanup_launch_token_matches(self, launch_token: str, *, remove: bool) -> str: + matching_ids = self._container_ids_for_launch_token(launch_token) + if matching_ids is None: + return "unknown" + if not matching_ids: + return "absent" + removal_results: list[bool] = [] + for candidate_id in matching_ids: + ownership, confirmed_id = self._container_label_identity( + candidate_id, + label=_LAUNCH_TOKEN_LABEL, + expected_value=launch_token, + ) + if ownership == "absent": + removal_results.append(True) + elif ownership == "owned" and confirmed_id == candidate_id: + removal_results.append( + self._remove_container_identity(confirmed_id, remove=remove) + ) + else: + removal_results.append(False) + return "removed" if all(removal_results) else "unknown" + + def _container_ids_for_launch_token(self, launch_token: str) -> tuple[str, ...] | None: + if not launch_token: + return None + result = self.process_runtime.run( + [ + "docker", + "container", + "ls", + "--all", + "--quiet", + "--no-trunc", + "--filter", + f"label={_LAUNCH_TOKEN_LABEL}={launch_token}", + ], + timeout=10.0, + ) + if result.returncode != 0: + return None + identities = tuple( + dict.fromkeys( + line.strip() + for line in str(result.stdout or "").splitlines() + if line.strip() + ) ) - return launch + if any(re.fullmatch(r"[0-9a-fA-F]{64}", identity) is None for identity in identities): + return None + return identities def wait_until_ready(self, launch: DockerServiceLaunch, *, port: int, listener_expected: bool) -> bool: if not self.container_running(launch.container_id): @@ -280,30 +533,104 @@ def stop( *, remove: bool = True, verify_ownership: bool = False, + expected_launch_token: str | None = None, + pending_cleanup_since: float | None = None, ) -> bool: if not container: return True - if verify_ownership and not self._container_owned_by_runtime_scope(container): - return False - command = ["docker", "rm", "--force", container] if remove else ["docker", "stop", container] + target = container + if expected_launch_token: + ownership, container_id = self._container_label_identity( + container, + label=_LAUNCH_TOKEN_LABEL, + expected_value=expected_launch_token, + ) + if ownership == "owned" and container_id is not None: + target = container_id + elif ownership == "unknown": + return False + else: + token_cleanup = self._cleanup_launch_token_matches( + expected_launch_token, + remove=remove, + ) + if token_cleanup == "removed": + return True + if token_cleanup == "unknown" or ownership == "foreign": + return False + if pending_cleanup_since is None: + return True + try: + pending_timestamp = float(pending_cleanup_since) + except (TypeError, ValueError): + return False + if not math.isfinite(pending_timestamp) or pending_timestamp <= 0: + return False + grace = self._timeout("ENVCTL_DOCKER_PENDING_CLEANUP_GRACE_SECONDS", 30.0) + if time.time() - pending_timestamp < grace: + return False + provisional = DockerServiceLaunch( + container_id=container, + container_name=container, + image="", + log_path="", + launch_token=expected_launch_token, + cleanup_pending_since=pending_timestamp, + ) + return self._cleanup_failed_launch( + provisional, + absence_can_confirm=True, + remove=remove, + ) + elif verify_ownership: + ownership, container_id = self._container_label_identity( + container, + label=_RUNTIME_SCOPE_LABEL, + expected_value=self._runtime_scope_identity(), + ) + if ownership == "absent": + return True + if ownership != "owned" or container_id is None: + return False + target = container_id + return self._remove_container_identity(target, remove=remove) + + def _remove_container_identity(self, container_id: str, *, remove: bool) -> bool: + command = ["docker", "rm", "--force", container_id] if remove else ["docker", "stop", container_id] result = self.process_runtime.run(command, timeout=30.0) return result.returncode == 0 or "No such container" in str(result.stderr or "") - def _container_owned_by_runtime_scope(self, container: str) -> bool: + def _container_label_identity( + self, + container: str, + *, + label: str, + expected_value: str, + ) -> tuple[str, str | None]: + if not container or not expected_value: + return "unknown", None inspected = self.process_runtime.run( [ "docker", "container", "inspect", "--format", - f"{{{{index .Config.Labels {json.dumps(_RUNTIME_SCOPE_LABEL)}}}}}", + f"{{{{.Id}}}}|{{{{index .Config.Labels {json.dumps(label)}}}}}", container, ], timeout=10.0, ) if inspected.returncode != 0: - return "No such" in str(inspected.stderr or "") - return str(inspected.stdout or "").strip() == self._runtime_scope_identity() + return ("absent", None) if "No such" in str(inspected.stderr or "") else ("unknown", None) + raw_id, separator, actual_value = str(inspected.stdout or "").strip().partition("|") + container_id = raw_id.strip() + if not separator or re.fullmatch(r"[0-9a-fA-F]{64}", container_id) is None: + return "unknown", None + return ( + ("owned", container_id) + if actual_value.strip() == expected_value + else ("foreign", container_id) + ) def container_running(self, container: str) -> bool: if not container: @@ -411,7 +738,10 @@ def _prepare_container_name(self, name: str) -> None: "container", "inspect", "--format", - f"{{{{.State.Running}}}}|{{{{index .Config.Labels {json.dumps(_RUNTIME_SCOPE_LABEL)}}}}}", + ( + f"{{{{.Id}}}}|{{{{.State.Running}}}}|" + f"{{{{index .Config.Labels {json.dumps(_RUNTIME_SCOPE_LABEL)}}}}}" + ), name, ], timeout=10.0, @@ -420,19 +750,29 @@ def _prepare_container_name(self, name: str) -> None: if "No such" in str(inspect.stderr or "") or "not found" in str(inspect.stderr or "").lower(): return raise RuntimeError(str(inspect.stderr or inspect.stdout or f"failed to inspect {name}").strip()) - running_text, _separator, owner = str(inspect.stdout or "").strip().partition("|") + container_id, separator, remainder = str(inspect.stdout or "").strip().partition("|") + running_text, owner_separator, owner = remainder.partition("|") + if ( + not separator + or not owner_separator + or re.fullmatch(r"[0-9a-fA-F]{64}", container_id.strip()) is None + ): + raise RuntimeError(f"Docker returned an invalid identity while inspecting {name!r}") expected_owner = self._runtime_scope_identity() - if owner and owner != expected_owner: + if owner != expected_owner: raise RuntimeError( f"Docker container {name!r} is managed by another envctl runtime scope; " "stop it from that runtime before starting this one" ) - if running_text.lower() == "true": + normalized_running = running_text.strip().lower() + if normalized_running not in {"true", "false"}: + raise RuntimeError(f"Docker returned an invalid running state while inspecting {name!r}") + if normalized_running == "true": raise RuntimeError( f"Docker container {name!r} is already running but was not resumed from current state; " "run envctl stop for this project before starting it again" ) - self._remove_container(name) + self._remove_container(container_id.strip()) def _remove_container(self, name: str) -> None: result = self.process_runtime.run(["docker", "rm", "--force", name], timeout=30.0) @@ -584,12 +924,114 @@ def _browser_facing_env_keys(self) -> set[str]: def _write_env_file(self, *, project_name: str, service_name: str, values: Mapping[str, str]) -> Path: run_dir = Path(self.runtime._run_dir_path(getattr(self.runtime, "_active_run_id", None) or "docker")) run_dir.mkdir(parents=True, exist_ok=True) + stale_errors = self._scrub_stale_env_files(run_dir) + unsafe_errors = [detail for detail in stale_errors if _SECRET_SCRUB_FAILED in detail] + if unsafe_errors: + raise RuntimeError("Stale Docker env file cleanup failed: " + "; ".join(unsafe_errors)) + for detail in stale_errors: + self._emit_best_effort("service.container.env_file_cleanup_warning", detail=detail, stale=True) digest = hashlib.sha256(f"{project_name}\0{service_name}".encode()).hexdigest()[:12] - path = run_dir / f".docker-env-{digest}" - lines = [f"{key}={str(value).replace(chr(10), '')}" for key, value in sorted(values.items())] - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - path.chmod(0o600) - return path + descriptor, raw_path = tempfile.mkstemp( + prefix=f".docker-env-{digest}-p{os.getpid()}-", + dir=run_dir, + ) + path = Path(raw_path) + try: + lines = [ + f"{key}={str(value).replace(chr(10), '')}" + for key, value in sorted(values.items()) + ] + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + descriptor = -1 + handle.write("\n".join(lines) + "\n") + return path + except BaseException as write_error: + if descriptor >= 0: + try: + os.close(descriptor) + except BaseException: # noqa: BLE001 - continue to pathname scrub + pass + cleanup_error = self._scrub_env_file_safely(path) + if cleanup_error: + self._emit_best_effort( + "service.container.env_file_cleanup_warning", + project=project_name, + service=service_name, + detail=cleanup_error, + write_failed=True, + ) + raise RuntimeError( + f"Docker env file write failed: {write_error}; cleanup failed: {cleanup_error}" + ) from write_error + raise + + def _scrub_stale_env_files(self, run_dir: Path) -> list[str]: + errors: list[str] = [] + legacy_grace = self._timeout("ENVCTL_DOCKER_ENV_STALE_SECONDS", 300.0) + now = time.time() + for path in tuple(run_dir.glob(".docker-env-*")): + match = _ENV_FILE_PID_PATTERN.fullmatch(path.name) + if match is not None: + pid = int(match.group("pid")) + if pid == os.getpid() or self._pid_is_running(pid): + continue + else: + try: + if now - path.lstat().st_mtime < legacy_grace: + continue + except OSError as exc: + errors.append(f"{path}: {_SECRET_SCRUB_FAILED}: {exc}") + continue + detail = self._scrub_env_file_safely(path) + if detail: + errors.append(f"{path}: {detail}") + return errors + + @staticmethod + def _pid_is_running(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except (PermissionError, OSError): + return True + return True + + @staticmethod + def _scrub_env_file(path: Path) -> str | None: + try: + path.unlink(missing_ok=True) + return None + except OSError as unlink_error: + if not path.exists() and not path.is_symlink(): + return None + try: + flags = os.O_WRONLY | os.O_TRUNC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags) + os.close(descriptor) + except OSError as scrub_error: + return f"{unlink_error}; secret scrub failed: {scrub_error}" + try: + path.unlink(missing_ok=True) + return None + except OSError as retry_error: + return f"{unlink_error}; file was scrubbed but unlink retry failed: {retry_error}" + + @staticmethod + def _scrub_env_file_safely(path: Path) -> str | None: + try: + return DockerServiceRuntime._scrub_env_file(path) + except BaseException as exc: # noqa: BLE001 - secret cleanup must never be masked + return f"{_SECRET_SCRUB_FAILED}: {exc}" + + def _emit_best_effort(self, event: str, **payload: object) -> None: + try: + self.runtime._emit(event, **payload) + except BaseException: # noqa: BLE001 - diagnostics cannot interrupt authority handoff + pass def _setting(self, service_name: str, suffix: str) -> str: env = getattr(self.runtime, "env", {}) diff --git a/python/envctl_engine/runtime/endpoints_command_support.py b/python/envctl_engine/runtime/endpoints_command_support.py index c2c9ae51b..8b863e570 100644 --- a/python/envctl_engine/runtime/endpoints_command_support.py +++ b/python/envctl_engine/runtime/endpoints_command_support.py @@ -189,6 +189,10 @@ def _service_project_name(service_name: str, service: ServiceRecord, *, runtime: def _requirements_for_project(state: RunState, project: str) -> RequirementsResult | None: if project in state.requirements: return state.requirements[project] + target = str(project).strip().casefold() + for storage_key, requirements in state.requirements.items(): + if str(requirements.project or storage_key).strip().casefold() == target: + return requirements if len(state.requirements) == 1: return next(iter(state.requirements.values())) return None diff --git a/python/envctl_engine/runtime/engine_runtime_artifacts.py b/python/envctl_engine/runtime/engine_runtime_artifacts.py index 085212d0b..21cec76f4 100644 --- a/python/envctl_engine/runtime/engine_runtime_artifacts.py +++ b/python/envctl_engine/runtime/engine_runtime_artifacts.py @@ -6,7 +6,7 @@ import time from collections.abc import Mapping, Sequence from pathlib import Path -from typing import Any, Protocol +from typing import Any, Callable, Protocol from envctl_engine.runtime.runtime_context import optional_state_repository from envctl_engine.runtime.runtime_readiness import ( @@ -28,7 +28,14 @@ class _SummaryContext(Protocol): ports: Mapping[str, _FinalPort] -def write_artifacts(runtime: Any, state: object, contexts: list[object], *, errors: list[str]) -> None: +def write_artifacts( + runtime: Any, + state: object, + contexts: list[object], + *, + errors: list[str], + on_commit: Callable[[], None] | None = None, +) -> None: started = time.monotonic() cached = _cached_runtime_readiness_payload(runtime) paths = runtime.state_repository.save_run( @@ -43,6 +50,7 @@ def write_artifacts(runtime: Any, state: object, contexts: list[object], *, erro if cached is not None else (lambda run_dir: _write_pending_runtime_readiness_payload(runtime, run_dir=run_dir)) ), + on_commit=on_commit, ) runtime._emit( "artifacts.write", diff --git a/python/envctl_engine/runtime/engine_runtime_hooks.py b/python/envctl_engine/runtime/engine_runtime_hooks.py index 4ba8394ad..1eeb78a7e 100644 --- a/python/envctl_engine/runtime/engine_runtime_hooks.py +++ b/python/envctl_engine/runtime/engine_runtime_hooks.py @@ -7,7 +7,7 @@ from envctl_engine.shared.dependency_compose_assets import materialize_dependency_compose, supabase_managed_env from envctl_engine.shared.hooks import HookInvocationResult, legacy_shell_hook_issue, run_envctl_hook from envctl_engine.state.models import RequirementsResult, ServiceRecord -from envctl_engine.shared.parsing import parse_bool, parse_int +from envctl_engine.shared.parsing import parse_bool, parse_float_or_none, parse_int from envctl_engine.requirements.supabase import build_supabase_project_name from envctl_engine.requirements.core import dependency_definitions @@ -148,10 +148,16 @@ def services_from_hook_payload( actual_port=(actual_port if actual_port > 0 else None), log_path=(str(log_path) if log_path else None), status=status, + project=str(context.name), + service_slug=service_type.strip().lower(), runtime_kind=str(raw.get("runtime_kind") or "process"), container_id=str(raw.get("container_id") or "") or None, container_name=str(raw.get("container_name") or "") or None, container_image=str(raw.get("container_image") or "") or None, + container_launch_token=str(raw.get("container_launch_token") or "") or None, + container_cleanup_pending_since=parse_float_or_none( + raw.get("container_cleanup_pending_since") + ), ) records[name] = record if service_type == "backend" and actual_port > 0: diff --git a/python/envctl_engine/runtime/engine_runtime_startup_facade.py b/python/envctl_engine/runtime/engine_runtime_startup_facade.py index 2ed209e17..889de8dbe 100644 --- a/python/envctl_engine/runtime/engine_runtime_startup_facade.py +++ b/python/envctl_engine/runtime/engine_runtime_startup_facade.py @@ -2,7 +2,7 @@ from collections.abc import Sequence from pathlib import Path -from typing import Any, Mapping, cast +from typing import Any, Callable, Mapping, cast from envctl_engine.runtime.engine_runtime_artifacts import ( print_summary as runtime_print_summary, @@ -198,8 +198,15 @@ def _start_project_services( route=route, ) - def _write_artifacts(self, state: RunState, contexts: list[Any], *, errors: list[str]) -> None: - runtime_write_artifacts(self, state, contexts, errors=errors) + def _write_artifacts( + self, + state: RunState, + contexts: list[Any], + *, + errors: list[str], + on_commit: Callable[[], None] | None = None, + ) -> None: + runtime_write_artifacts(self, state, contexts, errors=errors, on_commit=on_commit) def _write_runtime_readiness_report( self, diff --git a/python/envctl_engine/runtime/engine_runtime_startup_support.py b/python/envctl_engine/runtime/engine_runtime_startup_support.py index 0cddda46b..83d471e3e 100644 --- a/python/envctl_engine/runtime/engine_runtime_startup_support.py +++ b/python/envctl_engine/runtime/engine_runtime_startup_support.py @@ -1,6 +1,5 @@ from __future__ import annotations -import threading from pathlib import Path from typing import Any, Callable, Mapping @@ -21,9 +20,6 @@ from envctl_engine.planning import discover_tree_projects -_PORT_REAP_LOCK = threading.Lock() - - def effective_start_mode(runtime: Any, route: Route) -> str: setup_worktree_requested = getattr(runtime, "_setup_worktree_requested", None) if ( @@ -210,7 +206,6 @@ def discover_projects( def reserve_project_ports(runtime: Any, context: ProjectContextLike, route: Route | None = None) -> None: - _reap_stale_port_locks_once(runtime) selected_app_services = _selected_app_services_for_port_reservation(runtime, context=context, route=route) for service_name, plan in context.ports.items(): if not _port_reservation_enabled( @@ -273,18 +268,6 @@ def reserve_project_ports(runtime: Any, context: ProjectContextLike, route: Rout runtime._emit("port.reserved", project=context.name, service=service_name, port=reserved) -def _reap_stale_port_locks_once(runtime: Any) -> None: - reaper = getattr(runtime.port_planner, "reap_stale_locks", None) - if not callable(reaper) or bool(getattr(runtime, "_startup_stale_port_locks_reaped", False)): - return - with _PORT_REAP_LOCK: - if bool(getattr(runtime, "_startup_stale_port_locks_reaped", False)): - return - reaped = int(reaper() or 0) - setattr(runtime, "_startup_stale_port_locks_reaped", True) - runtime._emit("port.lock.reap_stale", count=reaped) - - def _port_reservation_enabled( runtime: Any, *, diff --git a/python/envctl_engine/runtime/inspection_support.py b/python/envctl_engine/runtime/inspection_support.py index 08d6f88b1..ec35ac4ad 100644 --- a/python/envctl_engine/runtime/inspection_support.py +++ b/python/envctl_engine/runtime/inspection_support.py @@ -10,6 +10,7 @@ from envctl_engine.config.persistence import managed_values_from_local_state, managed_values_to_payload from envctl_engine.planning import PlanProjectPrediction from envctl_engine.runtime.command_router import Route, list_supported_commands +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.runtime.state_inspection_support import ( emit_cwd_runtime_mismatch_warnings as _emit_cwd_runtime_mismatch_warnings_impl, print_state as _print_state_impl, @@ -154,7 +155,11 @@ def _running_tree_project_names(*, runtime: Any, state: RunState) -> set[str]: if not isinstance(services, dict): return running for service_name, record in services.items(): - project_name = runtime._project_name_from_service(str(service_name)) + project_name = resolve_service_project_name( + str(service_name), + record, + project_name_from_service=runtime._project_name_from_service, + ) normalized = str(project_name).strip() if not normalized: continue diff --git a/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py b/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py index d3c6a4456..395f3655b 100644 --- a/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py +++ b/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py @@ -4,8 +4,8 @@ from typing import Any, Callable, Mapping, cast from envctl_engine.requirements.core import dependency_definitions -from envctl_engine.runtime.lifecycle_blast_support import LifecycleBlastCleanupSupport from envctl_engine.runtime.engine_runtime_lifecycle_support import release_requirement_ports +from envctl_engine.runtime.lifecycle_blast_support import LifecycleBlastCleanupSupport from envctl_engine.runtime.lifecycle_dependency_stop import ( release_requirement_component_ports, release_selected_dependency_components, @@ -14,10 +14,13 @@ stop_requirement_component_containers, ) from envctl_engine.runtime.lifecycle_requirement_ports import requirement_component_port_owners +from envctl_engine.runtime.lifecycle_full_cleanup import clear_runtime_state as clear_full_runtime_state from envctl_engine.runtime.lifecycle_service_stop_selection import ( + project_selectors_from_route, select_services_for_stop, service_matches_runtime_scope, ) +from envctl_engine.runtime.lifecycle_service_termination import unconfirmed_service_names from envctl_engine.runtime.lifecycle_stopped_metadata import ( has_dashboard_stopped_services, project_name_from_stopped_service, @@ -33,10 +36,10 @@ resolve_state_repository, ) from envctl_engine.shared.protocols import PortAllocator, ProcessRuntime, StateRepository +from envctl_engine.shared.services import service_matches_selector from envctl_engine.runtime.command_router import Route from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RequirementsResult, RunState -from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.ui.selection_support import ( interactive_selection_allowed, project_names_from_state, @@ -48,7 +51,10 @@ from envctl_engine.ui.spinner_service import emit_spinner_policy, resolve_spinner_policy -class LifecycleCleanupOrchestrator(LifecycleTargetedStateSupport, LifecycleBlastCleanupSupport): +class LifecycleCleanupOrchestrator( + LifecycleTargetedStateSupport, + LifecycleBlastCleanupSupport, +): def __init__(self, runtime: Any) -> None: self.runtime = runtime @@ -67,7 +73,7 @@ def operation() -> int: if completed: print("Stopped runtime state.") return 0 - print("Runtime cleanup is incomplete; failed services remain tracked.") + print("Runtime cleanup is incomplete; failed resources remain tracked.") return 1 message = ( @@ -152,13 +158,27 @@ def _execute_stop_state(self, route: Route, state: RunState) -> int: return self._execute_stop_dependencies(route, state) owned_projects = self._owned_project_names(state) - targeted_project_keys = frozenset( - str(project).strip().casefold() - for project in route.projects - if str(project).strip().casefold() in owned_projects - ) + dependency_project_keys, has_dependency_project_scope = self._dependency_scope_project_keys(state, route) + if route.flags.get("runtime_scope") == "entire-system" and has_dependency_project_scope: + targeted_project_keys = frozenset( + project for project in dependency_project_keys if project in owned_projects + ) + else: + targeted_project_keys = frozenset( + str(project).strip().casefold() + for project in route.projects + if str(project).strip().casefold() in owned_projects + ) preserve_stopped_dashboard_state = self._should_preserve_stopped_dashboard_state(route) - selected_dependencies = self._select_dependency_components_for_stop(state, route) + selected_dependencies = self._select_dependency_components_for_stop( + state, + route, + requested_projects=( + dependency_project_keys + if route.flags.get("runtime_scope") == "entire-system" and has_dependency_project_scope + else None + ), + ) selected = self._select_services_for_stop(state, route) if not selected and not selected_dependencies: if targeted_project_keys and self._targeted_projects_have_no_managed_resources( @@ -197,8 +217,7 @@ def operation() -> int: aggressive=False, verify_ownership=True, ) - if isinstance(termination_result, set): - failed_services = {str(name) for name in termination_result} + failed_services = unconfirmed_service_names(termination_result, selected) stopped_services = selected.difference(failed_services) if stopped_services and preserve_stopped_dashboard_state: @@ -316,18 +335,6 @@ def _project_root_from_stopped_service(service: object, *, service_type: str) -> return project_root_from_stopped_service(service, service_type=service_type) def _select_services_for_stop(self, state: RunState, route: Route) -> set[str]: - selectors_from_passthrough = getattr(self.runtime, "_selectors_from_passthrough", None) - - def selectors_from_passthrough_args(args: list[str]) -> Iterable[str]: - if not callable(selectors_from_passthrough): - return set() - result = selectors_from_passthrough(args) - if isinstance(result, str): - return {result} - if isinstance(result, Iterable): - return cast(Iterable[str], result) - return set() - def selected_services_from_ui_selection(selection: object, selected_state: RunState) -> set[str]: return self._services_from_selection(cast(TargetSelection, selection), selected_state) @@ -337,11 +344,44 @@ def selected_services_from_ui_selection(selection: object, selected_state: RunSt project_name_from_service_fn=lambda service_name: self.runtime._project_name_from_service( # type: ignore[attr-defined] service_name ), - selectors_from_passthrough_fn=selectors_from_passthrough_args, + selectors_from_passthrough_fn=self._passthrough_project_selectors, interactive_stop_selection_fn=self._interactive_stop_selection, services_from_selection_fn=selected_services_from_ui_selection, ) + def _passthrough_project_selectors(self, args: list[str]) -> set[str]: + selectors_from_passthrough = getattr(self.runtime, "_selectors_from_passthrough", None) + if callable(selectors_from_passthrough): + result = selectors_from_passthrough(args) + if isinstance(result, str): + return {result} + if isinstance(result, Iterable): + return {str(item).strip().casefold() for item in result if str(item).strip()} + selectors: set[str] = set() + for token in args: + if str(token).startswith("-"): + continue + selectors.update(part.strip().casefold() for part in str(token).split(",") if part.strip()) + return selectors + + def _dependency_scope_project_keys(self, state: RunState, route: Route) -> tuple[frozenset[str], bool]: + project_keys = project_selectors_from_route( + route, + selectors_from_passthrough_fn=self._passthrough_project_selectors, + ) + services_flag = route.flags.get("services") + has_selectors = bool(project_keys) + if isinstance(services_flag, list): + service_selectors = [str(selector).strip() for selector in services_flag if str(selector).strip()] + has_selectors = has_selectors or bool(service_selectors) + for selector in service_selectors: + for name, service in state.services.items(): + if name.casefold() == selector.casefold() or service_matches_selector(service, selector): + project = self._project_name_for_service(name, service) + if project: + project_keys.add(project) + return frozenset(project_keys), has_selectors + def _execute_stop_dependencies(self, route: Route, state: RunState) -> int: def operation() -> int: if not state.requirements: @@ -350,21 +390,42 @@ def operation() -> int: print("No dependencies found.") return 0 - for project in list(state.requirements): - requirements = state.requirements[project] + targeted_project_keys, has_targeted_scope = self._dependency_scope_project_keys(state, route) + owned_projects = self._owned_project_names(state) + matching_requirement_keys = { + storage_key + for storage_key, requirements in state.requirements.items() + if self._requirement_project_key(storage_key, requirements) in targeted_project_keys + } + if has_targeted_scope and not matching_requirement_keys: + print("No matching dependencies found.") + return 1 + removed_count = 0 + for storage_key in list(state.requirements): + requirements = state.requirements[storage_key] + requirement_project = self._requirement_project_key(storage_key, requirements) + if has_targeted_scope and requirement_project not in targeted_project_keys: + continue for definition in dependency_definitions(): component = requirements.component(definition.id) if bool(component.get("enabled", False)): self._stop_requirement_component_containers(component) - self._release_requirement_ports(state.requirements[project]) - state.requirements.pop(project, None) + self._release_requirement_ports(requirements) + state.requirements.pop(storage_key, None) + removed_count += 1 - if state.services: + if has_targeted_scope: + self._persist_targeted_project_state( + state, + owned_projects=owned_projects, + targeted_project_keys=targeted_project_keys, + ) + elif state.services or state.requirements: self._save_selected_stop_state(state) else: self._deactivate_runtime_state(state) - print("Stopped dependencies.") + print("Stopped dependencies." if removed_count else "No matching dependencies found.") return 0 return self._run_spinner_operation( @@ -374,8 +435,18 @@ def operation() -> int: operation=operation, ) - def _select_dependency_components_for_stop(self, state: RunState, route: Route) -> dict[str, set[str]]: - return select_dependency_components_for_stop(state, route) + def _select_dependency_components_for_stop( + self, + state: RunState, + route: Route, + *, + requested_projects: Iterable[str] | None = None, + ) -> dict[str, set[str]]: + return select_dependency_components_for_stop( + state, + route, + requested_projects=requested_projects, + ) def _release_selected_dependency_components( self, @@ -392,9 +463,10 @@ def _release_selected_dependency_components( def _stop_requirement_component_containers(self, component: Mapping[str, object]) -> None: stop_requirement_component_containers(self.runtime, component) - def _stop_requirement_component_containers_best_effort(self, component: Mapping[str, object]) -> None: + def _stop_requirement_component_containers_best_effort(self, component: Mapping[str, object]) -> bool: try: self._stop_requirement_component_containers(component) + return True except Exception as exc: # noqa: BLE001 component_id = str(component.get("id") or "dependency").strip() or "dependency" detail = str(exc).strip() or exc.__class__.__name__ @@ -404,20 +476,23 @@ def _stop_requirement_component_containers_best_effort(self, component: Mapping[ detail=detail, ) print(f"Warning: could not stop {component_id} container during cleanup: {detail}") + return False def _stop_requirement_containers( self, requirements: RequirementsResult, *, best_effort: bool = False, - ) -> None: + ) -> bool: + stopped = True for definition in dependency_definitions(): component = requirements.component(definition.id) if bool(component.get("enabled", False)): if best_effort: - self._stop_requirement_component_containers_best_effort(component) + stopped = self._stop_requirement_component_containers_best_effort(component) and stopped else: self._stop_requirement_component_containers(component) + return stopped def _release_requirement_component_ports( self, @@ -443,6 +518,21 @@ def _release_requirement_ports(self, requirements: RequirementsResult) -> None: return release_requirement_ports(self.runtime, requirements) + def _release_requirement_ports_best_effort(self, requirements: RequirementsResult) -> bool: + try: + self._release_requirement_ports(requirements) + return True + except Exception as exc: # noqa: BLE001 + project = str(getattr(requirements, "project", "") or "dependency").strip() or "dependency" + detail = str(exc).strip() or exc.__class__.__name__ + self.runtime._emit( # type: ignore[attr-defined] + "cleanup.requirement_ports.warning", + project=project, + detail=detail, + ) + print(f"Warning: could not release {project} dependency ports during cleanup: {detail}") + return False + @staticmethod def _service_matches_runtime_scope(name: str, service: object, runtime_scope: str) -> bool: return service_matches_runtime_scope(name, service, runtime_scope) @@ -469,125 +559,19 @@ def _project_names_from_state(self, state: RunState) -> list[object]: def _services_from_selection(self, selection: TargetSelection, state: RunState) -> set[str]: return services_from_selection(self.runtime, selection, state) - def clear_runtime_state(self, *, command: str, aggressive: bool = False, route: Route | None = None) -> bool: - rt = self.runtime - if command == "stop-all": - rt._emit("cleanup.stop_all", aggressive=aggressive) # type: ignore[attr-defined] - elif aggressive: - rt._emit("cleanup.blast", aggressive=aggressive) # type: ignore[attr-defined] - else: - rt._emit("cleanup.stop", aggressive=aggressive) # type: ignore[attr-defined] - state_repository = self._state_repository(rt) - load_all = getattr(state_repository, "load_all", None) - mode_filter = None - if ( - command == "stop-all" - and route is not None - and rt._state_lookup_strict_mode_match(route) # type: ignore[attr-defined] - ): - mode_filter = route.mode - if callable(load_all): - states = [state for state in load_all(mode=mode_filter) if isinstance(state, RunState)] - else: - states = [] - if not states: - state = call_state_loader( - rt._try_load_existing_state, # type: ignore[attr-defined] - mode=mode_filter, - strict_mode_match=mode_filter is not None, - ) - states = [state] if isinstance(state, RunState) else [] - seen_run_ids: set[str] = set() - failed_by_run: dict[str, set[str]] = {} - for state in states: - if state.run_id in seen_run_ids: - continue - seen_run_ids.add(state.run_id) - termination_result = rt._terminate_services_from_state( # type: ignore[attr-defined] - state, - selected_services=None, - aggressive=aggressive, - verify_ownership=not aggressive, - ) - failed_services = ( - {str(name) for name in termination_result} if isinstance(termination_result, set) else set() - ) - if not failed_services: - requirements_to_release = list(state.requirements.values()) - for requirements in requirements_to_release: - self._stop_requirement_containers( - requirements, - best_effort=command in {"stop-all", "blast-all"}, - ) - self._deactivate_runtime_state(state, release_session_when_empty=False) - for requirements in requirements_to_release: - self._release_requirement_ports(requirements) - continue - failed_by_run[state.run_id] = failed_services - state.services = {name: service for name, service in state.services.items() if name in failed_services} - failed_projects = { - project_name - for name, service in state.services.items() - if (project_name := self._project_name_for_service(name, service)) - } - requirements_to_release: list[RequirementsResult] = [] - retained_requirements: dict[str, RequirementsResult] = {} - for project, requirements in state.requirements.items(): - requirement_project = str(getattr(requirements, "project", "") or project).strip().casefold() - if not failed_projects or requirement_project in failed_projects: - retained_requirements[project] = requirements - else: - requirements_to_release.append(requirements) - for requirements in requirements_to_release: - self._stop_requirement_containers( - requirements, - best_effort=command in {"stop-all", "blast-all"}, - ) - state.requirements = {project: requirements for project, requirements in retained_requirements.items()} - state_repository.save_selected_stop_state( - state=state, - emit=rt._emit, # type: ignore[attr-defined] - runtime_map_builder=lambda raw_state: build_runtime_map(cast(RunState, raw_state)), - ) - for requirements in requirements_to_release: - self._release_requirement_ports(requirements) - - if failed_by_run: - rt._emit( # type: ignore[attr-defined] - "cleanup.incomplete", - failed_services={run_id: sorted(names) for run_id, names in failed_by_run.items()}, - ) - return False - - if aggressive and self.blast_all_ecosystem_enabled(): - self.blast_all_ecosystem_cleanup(route=route) - elif command == "stop-all" and route is not None and bool(route.flags.get("stop_all_remove_volumes")): - rt._emit("cleanup.stop_all.remove_volumes", requested=True) # type: ignore[attr-defined] - volume_route = Route( - command=route.command, - mode=route.mode, - raw_args=route.raw_args, - passthrough_args=route.passthrough_args, - projects=route.projects, - flags={ - **route.flags, - "blast_keep_worktree_volumes": False, - "blast_remove_main_volumes": True, - }, - ) - removed = self.blast_all_docker_cleanup(route=volume_route) - rt._emit("cleanup.stop_all.remove_volumes.finish", removed=removed) # type: ignore[attr-defined] - - if aggressive or command != "stop-all" or not self._has_active_runs(state_repository): - self._purge_runtime_state(aggressive=aggressive) - repository = self._state_repository(rt) - if aggressive and getattr(repository, "compat_mode", None) == getattr( - repository, - "COMPAT_READ_WRITE", - "compat_read_write", - ): - self.blast_all_purge_legacy_state_artifacts() - return True + def clear_runtime_state( + self, + *, + command: str, + aggressive: bool = False, + route: Route | None = None, + ) -> bool: + return clear_full_runtime_state( + self, + command=command, + aggressive=aggressive, + route=route, + ) @staticmethod def _has_active_runs(repository: object) -> bool: diff --git a/python/envctl_engine/runtime/lifecycle_dependency_stop.py b/python/envctl_engine/runtime/lifecycle_dependency_stop.py index 617243eb0..acbb4455a 100644 --- a/python/envctl_engine/runtime/lifecycle_dependency_stop.py +++ b/python/envctl_engine/runtime/lifecycle_dependency_stop.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterable, Mapping from typing import Any from envctl_engine.requirements.core import dependency_definitions @@ -9,9 +9,19 @@ from envctl_engine.state.models import RequirementsResult, RunState -def select_dependency_components_for_stop(state: RunState, route: Route) -> dict[str, set[str]]: +def select_dependency_components_for_stop( + state: RunState, + route: Route, + *, + requested_projects: Iterable[str] | None = None, +) -> dict[str, set[str]]: if route.flags.get("runtime_scope") == "entire-system": - requested_projects = {str(project).strip().casefold() for project in route.projects if str(project).strip()} + explicit_project_scope = requested_projects is not None + normalized_projects = { + str(project).strip().casefold() + for project in (route.projects if requested_projects is None else requested_projects) + if str(project).strip() + } return { project_name: { definition.id @@ -19,7 +29,8 @@ def select_dependency_components_for_stop(state: RunState, route: Route) -> dict if bool(requirements.component(definition.id).get("enabled", False)) } for project_name, requirements in state.requirements.items() - if not requested_projects or str(project_name).strip().casefold() in requested_projects + if (not explicit_project_scope and not normalized_projects) + or str(requirements.project or project_name).strip().casefold() in normalized_projects } raw_components = route.flags.get("stop_dependency_components") if not isinstance(raw_components, list): @@ -27,21 +38,26 @@ def select_dependency_components_for_stop(state: RunState, route: Route) -> dict known_definitions = {definition.id for definition in dependency_definitions()} selected: dict[str, set[str]] = {} - project_key_by_lower = {str(project).strip().casefold(): project for project in state.requirements} + project_keys_by_lower: dict[str, list[str]] = {} + for storage_key, requirements in state.requirements.items(): + project = str(requirements.project or storage_key).strip().casefold() + if project: + project_keys_by_lower.setdefault(project, []).append(storage_key) for raw in raw_components: project_name, separator, dependency_id = str(raw).partition(":") if not separator: continue - project_key = project_key_by_lower.get(project_name.strip().casefold()) - if project_key is None: + project_keys = project_keys_by_lower.get(project_name.strip().casefold(), []) + if not project_keys: continue normalized_dependency = dependency_id.strip().lower() if normalized_dependency not in known_definitions: continue - component = state.requirements[project_key].component(normalized_dependency) - if not bool(component.get("enabled", False)): - continue - selected.setdefault(project_key, set()).add(normalized_dependency) + for project_key in project_keys: + component = state.requirements[project_key].component(normalized_dependency) + if not bool(component.get("enabled", False)): + continue + selected.setdefault(project_key, set()).add(normalized_dependency) return selected diff --git a/python/envctl_engine/runtime/lifecycle_full_cleanup.py b/python/envctl_engine/runtime/lifecycle_full_cleanup.py new file mode 100644 index 000000000..38ac41134 --- /dev/null +++ b/python/envctl_engine/runtime/lifecycle_full_cleanup.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +from typing import Any, cast + +from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.lifecycle_service_termination import unconfirmed_service_names +from envctl_engine.state.lookup import call_state_loader +from envctl_engine.state.models import RequirementsResult, RunState +from envctl_engine.state.runtime_map import build_runtime_map + + +def clear_runtime_state( + owner: Any, + *, + command: str, + aggressive: bool = False, + route: Route | None = None, +) -> bool: + """Reconcile repository-wide stop and blast state.""" + + rt = owner.runtime + if command == "stop-all": + rt._emit("cleanup.stop_all", aggressive=aggressive) + elif aggressive: + rt._emit("cleanup.blast", aggressive=aggressive) + else: + rt._emit("cleanup.stop", aggressive=aggressive) + state_repository = owner._state_repository(rt) + load_all = getattr(state_repository, "load_all", None) + mode_filter = None + if command == "stop-all" and route is not None and rt._state_lookup_strict_mode_match(route): + mode_filter = route.mode + if callable(load_all): + states = [state for state in load_all(mode=mode_filter) if isinstance(state, RunState)] + else: + states = [] + if not states: + state = call_state_loader( + rt._try_load_existing_state, + mode=mode_filter, + strict_mode_match=mode_filter is not None, + ) + states = [state] if isinstance(state, RunState) else [] + seen_run_ids: set[str] = set() + failed_by_run: dict[str, set[str]] = {} + failed_requirements_by_run: dict[str, set[str]] = {} + for state in states: + if state.run_id in seen_run_ids: + continue + seen_run_ids.add(state.run_id) + termination_result = rt._terminate_services_from_state( + state, + selected_services=None, + aggressive=aggressive, + verify_ownership=not aggressive, + ) + failed_services = unconfirmed_service_names(termination_result, set(state.services)) + state.services = {name: service for name, service in state.services.items() if name in failed_services} + if failed_services: + failed_by_run[state.run_id] = failed_services + failed_projects = { + project_name + for name, service in state.services.items() + if (project_name := owner._project_name_for_service(name, service)) + } + requirements_to_release: list[tuple[str, RequirementsResult]] = [] + retained_requirements: dict[str, RequirementsResult] = {} + for storage_key, requirements in state.requirements.items(): + requirement_project = owner._requirement_project_key(storage_key, requirements) + if failed_projects and requirement_project in failed_projects: + retained_requirements[storage_key] = requirements + continue + containers_stopped = owner._stop_requirement_containers( + requirements, + best_effort=command in {"stop-all", "blast-all"}, + ) + if not containers_stopped: + retained_requirements[storage_key] = requirements + failed_requirements_by_run.setdefault(state.run_id, set()).add( + str(getattr(requirements, "project", "") or storage_key).strip() or storage_key + ) + else: + requirements_to_release.append((storage_key, requirements)) + + for storage_key, requirements in requirements_to_release: + if owner._release_requirement_ports_best_effort(requirements): + continue + retained_requirements[storage_key] = requirements + failed_requirements_by_run.setdefault(state.run_id, set()).add( + str(getattr(requirements, "project", "") or storage_key).strip() or storage_key + ) + + state.requirements = retained_requirements + if state.services or state.requirements: + state_repository.save_selected_stop_state( + state=state, + emit=rt._emit, + runtime_map_builder=lambda raw_state: build_runtime_map(cast(RunState, raw_state)), + ) + else: + owner._deactivate_runtime_state(state, release_session_when_empty=False) + + if failed_by_run or failed_requirements_by_run: + rt._emit( + "cleanup.incomplete", + failed_services={run_id: sorted(names) for run_id, names in failed_by_run.items()}, + failed_requirements={run_id: sorted(names) for run_id, names in failed_requirements_by_run.items()}, + ) + return False + + if aggressive and owner.blast_all_ecosystem_enabled(): + owner.blast_all_ecosystem_cleanup(route=route) + elif command == "stop-all" and route is not None and bool(route.flags.get("stop_all_remove_volumes")): + rt._emit("cleanup.stop_all.remove_volumes", requested=True) + volume_route = Route( + command=route.command, + mode=route.mode, + raw_args=route.raw_args, + passthrough_args=route.passthrough_args, + projects=route.projects, + flags={ + **route.flags, + "blast_keep_worktree_volumes": False, + "blast_remove_main_volumes": True, + }, + ) + removed = owner.blast_all_docker_cleanup(route=volume_route) + rt._emit("cleanup.stop_all.remove_volumes.finish", removed=removed) + + if aggressive or command != "stop-all" or not owner._has_active_runs(state_repository): + owner._purge_runtime_state(aggressive=aggressive) + repository = owner._state_repository(rt) + if aggressive and getattr(repository, "compat_mode", None) == getattr( + repository, + "COMPAT_READ_WRITE", + "compat_read_write", + ): + owner.blast_all_purge_legacy_state_artifacts() + return True diff --git a/python/envctl_engine/runtime/lifecycle_requirement_ports.py b/python/envctl_engine/runtime/lifecycle_requirement_ports.py index 5d51d01b5..ab5e206d1 100644 --- a/python/envctl_engine/runtime/lifecycle_requirement_ports.py +++ b/python/envctl_engine/runtime/lifecycle_requirement_ports.py @@ -6,6 +6,10 @@ from envctl_engine.requirements.core import dependency_definitions from envctl_engine.state.models import RequirementsResult, RunState +from envctl_engine.state.project_runtime import ( + requirement_key_for_project as _requirement_key_for_project, + requirement_keys_for_project as _requirement_keys_for_project, +) def release_requirement_ports(runtime: Any, requirements: RequirementsResult) -> None: @@ -153,10 +157,8 @@ def component_port_values(component: Mapping[str, object]) -> set[int]: def requirement_key_for_project(state: RunState, project_name: str) -> str | None: - target = str(project_name).strip().lower() - if not target: - return None - for key in state.requirements: - if str(key).strip().lower() == target: - return key - return None + return _requirement_key_for_project(state, project_name) + + +def requirement_keys_for_project(state: RunState, project_name: str) -> list[str]: + return _requirement_keys_for_project(state, project_name) diff --git a/python/envctl_engine/runtime/lifecycle_service_stop_selection.py b/python/envctl_engine/runtime/lifecycle_service_stop_selection.py index 273c58233..023e75c4f 100644 --- a/python/envctl_engine/runtime/lifecycle_service_stop_selection.py +++ b/python/envctl_engine/runtime/lifecycle_service_stop_selection.py @@ -4,7 +4,7 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.lifecycle_dependency_stop import select_dependency_components_for_stop -from envctl_engine.shared.services import service_matches_selector +from envctl_engine.shared.services import service_matches_selector, service_project_name from envctl_engine.state.models import RunState @@ -79,7 +79,10 @@ def _selected_services_from_route( has_selectors = has_selectors or bool(services_flag) selected.update(_services_matching_service_selectors(state, services_flag)) - project_names = _project_selectors(route, selectors_from_passthrough_fn=selectors_from_passthrough_fn) + project_names = project_selectors_from_route( + route, + selectors_from_passthrough_fn=selectors_from_passthrough_fn, + ) has_selectors = has_selectors or bool(project_names) if project_names: selected.update( @@ -105,7 +108,7 @@ def _services_matching_service_selectors(state: RunState, services_flag: list[ob return selected -def _project_selectors( +def project_selectors_from_route( route: Route, *, selectors_from_passthrough_fn: Callable[[list[str]], Iterable[str]], @@ -123,8 +126,8 @@ def _services_matching_project_selectors( project_name_from_service_fn: Callable[[str], str], ) -> set[str]: selected: set[str] = set() - for name in state.services: - project = project_name_from_service_fn(name).lower() + for name, service in state.services.items(): + project = (service_project_name(service) or project_name_from_service_fn(name)).lower() if project and project in project_names: selected.add(name) return selected diff --git a/python/envctl_engine/runtime/lifecycle_service_termination.py b/python/envctl_engine/runtime/lifecycle_service_termination.py index 092d863e2..1720390f6 100644 --- a/python/envctl_engine/runtime/lifecycle_service_termination.py +++ b/python/envctl_engine/runtime/lifecycle_service_termination.py @@ -2,7 +2,7 @@ import concurrent.futures import os -from collections.abc import Mapping +from collections.abc import Collection, Mapping from dataclasses import dataclass from pathlib import Path from typing import Any @@ -16,6 +16,38 @@ from envctl_engine.runtime.lifecycle_requirement_ports import release_port_reservation +def unconfirmed_service_names(result: object, service_names: Collection[str]) -> set[str]: + """Normalize a termination adapter result without ever assuming success. + + Only a collection containing known service identities is authoritative. An + absent/scalar result or any unknown identity fails closed for the complete + requested set so callers never discard state for a process whose exit was + not positively confirmed. + """ + + requested = {str(name).strip() for name in service_names if str(name).strip()} + if not isinstance(result, Collection) or isinstance(result, (str, bytes, Mapping)): + return requested + raw_reported = list(result) + if any(not str(name).strip() for name in raw_reported): + return requested + reported = {str(name).strip() for name in raw_reported} + return reported if reported.issubset(requested) else requested + + +def failed_listener_pids(result: object) -> set[int] | None: + """Return failed listener PIDs, or ``None`` when cleanup was not confirmed.""" + + if not isinstance(result, Collection) or isinstance(result, (str, bytes, Mapping)): + return None + failed: set[int] = set() + for value in result: + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + return None + failed.add(value) + return failed + + def terminate_started_services(runtime: Any, services: dict[str, object]) -> set[str]: """Terminate startup-owned services and report every unconfirmed exit.""" @@ -127,9 +159,17 @@ def terminate_service_record(runtime: Any, service: object, *, aggressive: bool, container = str( getattr(service, "container_id", "") or getattr(service, "container_name", "") or "" ).strip() + launch_token = str(getattr(service, "container_launch_token", "") or "").strip() + pending_since = getattr(service, "container_cleanup_pending_since", None) stopped = DockerServiceRuntime(runtime, runtime.process_runner).stop( container, verify_ownership=verify_ownership, + **({"expected_launch_token": launch_token} if launch_token else {}), + **( + {"pending_cleanup_since": pending_since} + if isinstance(pending_since, (int, float)) + else {} + ), ) follower_stopped = True pid = getattr(service, "pid", None) diff --git a/python/envctl_engine/runtime/lifecycle_targeted_state.py b/python/envctl_engine/runtime/lifecycle_targeted_state.py index 1d7cf805f..35960e4ff 100644 --- a/python/envctl_engine/runtime/lifecycle_targeted_state.py +++ b/python/envctl_engine/runtime/lifecycle_targeted_state.py @@ -3,6 +3,7 @@ from collections.abc import Iterable, Mapping from typing import Any, cast +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.state.models import RequirementsResult, RunState from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.state.state_aggregation import filter_state_to_owned_projects @@ -14,10 +15,11 @@ class LifecycleTargetedStateSupport: runtime: Any = cast(Any, None) def _project_name_for_service(self, name: str, service: object) -> str: - explicit_project = str(getattr(service, "project", "") or "").strip() - if explicit_project: - return explicit_project.casefold() - return str(self.runtime._project_name_from_service(name) or "").strip().casefold() + return resolve_service_project_name( + name, + service, + project_name_from_service=self.runtime._project_name_from_service, + ).casefold() def _save_selected_stop_state( self, diff --git a/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py b/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py index 07f726b84..27de6d04b 100644 --- a/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py +++ b/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py @@ -6,12 +6,15 @@ from pathlib import Path from typing import Any, Protocol +from envctl_engine.requirements.core import dependency_definitions +from envctl_engine.runtime.lifecycle_dependency_stop import stop_requirement_component_containers from envctl_engine.runtime.lifecycle_requirement_ports import ( release_requirement_ports, - requirement_key_for_project, + requirement_keys_for_project, requirement_port_values, ) from envctl_engine.runtime.lifecycle_service_termination import service_port +from envctl_engine.runtime.lifecycle_service_termination import unconfirmed_service_names from envctl_engine.runtime.lifecycle_worktree_containers import ( legacy_container_name as _legacy_container_name, ) @@ -30,6 +33,7 @@ from envctl_engine.runtime.lifecycle_worktree_processes import ( blast_tree_listener_ports as _blast_tree_listener_ports, ) +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RequirementsResult, RunState from envctl_engine.state.runtime_map import build_runtime_map @@ -147,8 +151,13 @@ def blast_worktree_before_delete( selected_services = { name - for name in list(state.services.keys()) - if runtime._project_name_from_service(name).strip().lower() == target_lower + for name, service in list(state.services.items()) + if resolve_service_project_name( + name, + service, + project_name_from_service=runtime._project_name_from_service, + ).lower() + == target_lower } for service_name in selected_services: service = state.services.get(service_name) @@ -160,8 +169,8 @@ def blast_worktree_before_delete( log_path_raw = str(getattr(service, "log_path", "") or "").strip() if blast_mode and log_path_raw: artifact_paths.add(Path(log_path_raw)) - requirement_key = requirement_key_for_project(state, normalized_project) - if not selected_services and requirement_key is None: + requirement_keys = requirement_keys_for_project(state, normalized_project) + if not selected_services and not requirement_keys: continue termination_result = runtime._terminate_services_from_state( @@ -170,16 +179,23 @@ def blast_worktree_before_delete( aggressive=True, verify_ownership=False, ) - failed_services = ( - {str(name) for name in termination_result} if isinstance(termination_result, (set, frozenset)) else set() - ) + failed_services = unconfirmed_service_names(termination_result, selected_services) stopped_services = selected_services.difference(failed_services) for service_name in stopped_services: state.services.pop(service_name, None) - if requirement_key is not None and not failed_services: - requirement_entry = state.requirements.pop(requirement_key, None) - if requirement_entry is not None: + requirements_removed = 0 + if requirement_keys and not failed_services: + for requirement_key in requirement_keys: + requirement_entry = state.requirements.get(requirement_key) + if requirement_entry is None: + continue + for definition in dependency_definitions(): + component = requirement_entry.component(definition.id) + if bool(component.get("enabled", False)): + stop_requirement_component_containers(runtime, component) + state.requirements.pop(requirement_key, None) + requirements_removed += 1 target_ports.update(cleanup.collect_requirement_ports(requirement_entry)) release_requirement_ports(runtime, requirement_entry) @@ -208,7 +224,7 @@ def blast_worktree_before_delete( project=normalized_project, mode=mode, services_removed=len(stopped_services), - requirements_removed=(requirement_key is not None and not failed_services), + requirements_removed=requirements_removed, ) if failed_services: warning = ( diff --git a/python/envctl_engine/runtime/requirement_reconcile_truth.py b/python/envctl_engine/runtime/requirement_reconcile_truth.py index da910b260..c4be27c17 100644 --- a/python/envctl_engine/runtime/requirement_reconcile_truth.py +++ b/python/envctl_engine/runtime/requirement_reconcile_truth.py @@ -118,9 +118,9 @@ def requirement_truth_identity( state_key: str, requirements: RequirementsResult, ) -> tuple[str, Path | None]: - truth_project = state_key - shared_scope = str(state.metadata.get("dashboard_dependency_scope", "")).strip().lower() == "shared" requirement_project = str(getattr(requirements, "project", "") or "").strip() + truth_project = requirement_project or state_key + shared_scope = str(state.metadata.get("dashboard_dependency_scope", "")).strip().lower() == "shared" shared_project = str(state.metadata.get("dashboard_shared_dependency_project", "") or "").strip() if shared_scope and requirement_project: truth_project = requirement_project @@ -134,7 +134,8 @@ def requirement_truth_identity( def requirement_truth_issues(runtime: Any, state: RunState) -> list[dict[str, object]]: issues: list[dict[str, object]] = [] - for project, requirements in state.requirements.items(): + for storage_key, requirements in state.requirements.items(): + project = str(getattr(requirements, "project", "") or storage_key).strip() for definition in dependency_definitions(): component_name = definition.id component_data = requirements.component(component_name) diff --git a/python/envctl_engine/runtime/service_manager.py b/python/envctl_engine/runtime/service_manager.py index ccaf3c5e3..a5240bfe8 100644 --- a/python/envctl_engine/runtime/service_manager.py +++ b/python/envctl_engine/runtime/service_manager.py @@ -101,9 +101,12 @@ def start_service_with_retry( started_at=time.time(), listener_expected=listener_expected, ) - except Exception as exc: # noqa: BLE001 - error = str(exc) - if not _terminate_pid(pid, process_runner=self.process_runner): + except BaseException as exc: # noqa: BLE001 - cancellation must not leak the launched PID + cleanup_confirmed = _terminate_pid_safely( + pid, + process_runner=self.process_runner, + ) + if not cleanup_confirmed: record = _failed_start_record( project=project, service_type=service_type, @@ -111,12 +114,19 @@ def start_service_with_retry( pid=pid, requested_port=current_port if listener_expected else None, listener_expected=listener_expected, - failure_detail=error, + failure_detail=str(exc), ) + unterminated = {record.name: record} + if not isinstance(exc, Exception): + _retain_unterminated_services(exc, unterminated) + raise raise ServiceCleanupError( - f"{error}; cleanup could not confirm exit of PID {pid}", - {record.name: record}, + f"{exc}; cleanup could not confirm exit of PID {pid}", + unterminated, ) from exc + if not isinstance(exc, Exception): + raise + error = str(exc) elif isinstance(pid, int) and pid > 0: if not _terminate_pid(pid, process_runner=self.process_runner): record = _failed_start_record( @@ -245,36 +255,39 @@ def start_record(descriptor: ServiceStartDescriptor) -> ServiceRecord: partial_records: list[ServiceRecord] = [] try: if parallel_start and len(descriptors) > 1: - worker_count = max_workers or len(descriptors) - with concurrent.futures.ThreadPoolExecutor(max_workers=min(len(descriptors), worker_count)) as executor: - future_map = { - executor.submit(start_record, descriptor): index for index, descriptor in enumerate(descriptors) - } - records_by_index: dict[int, ServiceRecord] = {} - failures_by_index: dict[int, Exception] = {} - for future in concurrent.futures.as_completed(future_map): - index = future_map[future] - try: - records_by_index[index] = future.result() - except Exception as exc: # noqa: BLE001 - failures_by_index[index] = exc - - partial_records.extend(records_by_index[index] for index in sorted(records_by_index)) - if failures_by_index: - primary_failure = failures_by_index[min(failures_by_index)] - unterminated = _unterminated_services_from_failures(failures_by_index.values()) - if unterminated: - raise ServiceCleanupError(str(primary_failure), unterminated) from primary_failure - raise primary_failure + records_by_index, failures_by_index, interruption = _start_parallel_records( + descriptors=descriptors, + start_record=start_record, + max_workers=max_workers, + ) + partial_records.extend(records_by_index[index] for index in sorted(records_by_index)) + primary_failure = _primary_parallel_failure( + failures_by_index, + interruption=interruption, + ) + if primary_failure is not None: + unterminated = _unterminated_services_from_failures(failures_by_index.values()) + if unterminated: + if not isinstance(primary_failure, Exception): + _retain_unterminated_services(primary_failure, unterminated) + raise primary_failure + raise ServiceCleanupError(str(primary_failure), unterminated) from primary_failure + raise primary_failure else: for descriptor in descriptors: partial_records.append(start_record(descriptor)) - except Exception as exc: + except BaseException as exc: # noqa: BLE001 - cancellation still requires transactional rollback unterminated = _unterminated_services_from_failures((exc,)) - for record in partial_records: - if not _terminate_pid(record.pid, process_runner=self.process_runner): - unterminated[record.name] = record + unterminated.update( + _cleanup_service_records( + partial_records, + process_runner=self.process_runner, + ) + ) if unterminated: + if not isinstance(exc, Exception): + _retain_unterminated_services(exc, unterminated) + raise names = ", ".join(sorted(unterminated)) raise ServiceCleanupError( f"{exc}; cleanup could not confirm exit for: {names}", @@ -302,6 +315,110 @@ def _terminate_pid(pid: int | None, *, process_runner: object | None = None) -> ) +def _terminate_pid_safely(pid: int | None, *, process_runner: object | None = None) -> bool: + try: + return _terminate_pid(pid, process_runner=process_runner) + except BaseException: # noqa: BLE001 - cleanup must not mask the triggering cancellation + return False + + +def _cleanup_service_records( + records: Iterable[ServiceRecord], + *, + process_runner: object | None, +) -> dict[str, ServiceRecord]: + unterminated: dict[str, ServiceRecord] = {} + for record in records: + if not _terminate_pid_safely(record.pid, process_runner=process_runner): + unterminated[record.name] = record + return unterminated + + +def _retain_unterminated_services( + failure: BaseException, + services: dict[str, ServiceRecord], +) -> None: + retained = _unterminated_services_from_failures((failure,)) + retained.update(services) + try: + failure.unterminated_services = retained # type: ignore[attr-defined] + except Exception: # noqa: BLE001 - built-in cancellation types normally allow metadata + failure.add_note("cleanup could not confirm exit for: " + ", ".join(sorted(retained))) + + +def _start_parallel_records( + *, + descriptors: tuple[ServiceStartDescriptor, ...], + start_record: Callable[[ServiceStartDescriptor], ServiceRecord], + max_workers: int | None, +) -> tuple[dict[int, ServiceRecord], dict[int, BaseException], BaseException | None]: + records_by_index: dict[int, ServiceRecord] = {} + failures_by_index: dict[int, BaseException] = {} + future_map: dict[concurrent.futures.Future[ServiceRecord], int] = {} + collected: set[concurrent.futures.Future[ServiceRecord]] = set() + interruption: BaseException | None = None + worker_count = max_workers or len(descriptors) + executor = concurrent.futures.ThreadPoolExecutor(max_workers=min(len(descriptors), worker_count)) + + def collect(future: concurrent.futures.Future[ServiceRecord]) -> None: + collected.add(future) + index = future_map[future] + try: + records_by_index[index] = future.result() + except BaseException as exc: # noqa: BLE001 - a worker cancellation is transaction data + failures_by_index[index] = exc + + try: + try: + for index, descriptor in enumerate(descriptors): + future = executor.submit(start_record, descriptor) + future_map[future] = index + for future in concurrent.futures.as_completed(future_map): + collect(future) + except BaseException as exc: # noqa: BLE001 - main-thread cancellation must drain submitted launches + interruption = exc + for future in future_map: + if future not in collected: + future.cancel() + finally: + try: + executor.shutdown(wait=True, cancel_futures=interruption is not None) + except BaseException as exc: # noqa: BLE001 - preserve the first cancellation and retry teardown + if interruption is None: + interruption = exc + for future in future_map: + future.cancel() + try: + executor.shutdown(wait=True, cancel_futures=True) + except BaseException as shutdown_exc: # noqa: BLE001 - retain diagnostics on the original failure + assert interruption is not None + interruption.add_note(f"parallel executor shutdown failed: {shutdown_exc}") + + for future in future_map: + if future in collected or future.cancelled(): + continue + if future.done(): + collect(future) + return records_by_index, failures_by_index, interruption + + +def _primary_parallel_failure( + failures_by_index: dict[int, BaseException], + *, + interruption: BaseException | None, +) -> BaseException | None: + if interruption is not None: + return interruption + cancellation_indexes = [ + index for index, failure in failures_by_index.items() if not isinstance(failure, Exception) + ] + if cancellation_indexes: + return failures_by_index[min(cancellation_indexes)] + if failures_by_index: + return failures_by_index[min(failures_by_index)] + return None + + def _failed_start_record( *, project: str, diff --git a/python/envctl_engine/runtime_feature_lifecycle_command_definitions.py b/python/envctl_engine/runtime_feature_lifecycle_command_definitions.py index 76fc32727..e803bbd16 100644 --- a/python/envctl_engine/runtime_feature_lifecycle_command_definitions.py +++ b/python/envctl_engine/runtime_feature_lifecycle_command_definitions.py @@ -82,6 +82,7 @@ python_source_of_truth=( "python/envctl_engine/runtime/engine_runtime.py", "python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py", + "python/envctl_engine/runtime/lifecycle_full_cleanup.py", ), evidence_tests=("tests/python/runtime/test_lifecycle_parity_stop_health.py",), parity_status="verified_python", @@ -95,6 +96,7 @@ python_source_of_truth=( "python/envctl_engine/runtime/engine_runtime.py", "python/envctl_engine/runtime/engine_runtime_lifecycle_support.py", + "python/envctl_engine/runtime/lifecycle_full_cleanup.py", "python/envctl_engine/runtime/lifecycle_blast_support.py", "python/envctl_engine/runtime/lifecycle_blast_ports.py", "python/envctl_engine/runtime/lifecycle_blast_processes.py", diff --git a/python/envctl_engine/shared/ports.py b/python/envctl_engine/shared/ports.py index 36552d9ef..783705ddc 100644 --- a/python/envctl_engine/shared/ports.py +++ b/python/envctl_engine/shared/ports.py @@ -413,14 +413,28 @@ def _normalize_project_identity(project: str) -> str: def _reserve_port(self, port: int, owner: str) -> bool: lock_path = self._lock_path(port) with self._port_guard(port): + availability_confirmed = False if lock_path.exists(): if not self._lock_is_stale(lock_path): return False stale_payload = self._read_lock_payload(lock_path) + if stale_payload is None or stale_payload.get("owner") != owner: + # A targeted startup may reclaim a dead allocator session + # only for the same logical project/service. A dead or + # corrupt foreign lock is still durable ownership evidence + # and requires an explicit all-scope cleanup operation. + return False + # The allocator process recorded in an otherwise stale lock + # can exit while its launched service is still listening. Do + # not erase that last ownership record merely because the + # allocator PID is gone. + if not self._is_port_available(port): + return False + availability_confirmed = True lock_path.unlink(missing_ok=True) self._emit_reclaim(port=port, owner=owner, payload=stale_payload) - if not self._is_port_available(port): + if not availability_confirmed and not self._is_port_available(port): return False payload = { diff --git a/python/envctl_engine/shared/process_launch_support.py b/python/envctl_engine/shared/process_launch_support.py index 10a684383..f57ae247e 100644 --- a/python/envctl_engine/shared/process_launch_support.py +++ b/python/envctl_engine/shared/process_launch_support.py @@ -1,6 +1,8 @@ from __future__ import annotations import hashlib +import os +import signal import subprocess import time from dataclasses import dataclass @@ -23,6 +25,20 @@ class LaunchRecord: launched_at_monotonic: float +class ProcessHandoffCleanupError(RuntimeError): + """A spawned process whose exit could not be confirmed after handoff failed.""" + + def __init__(self, *, process: object, launch_intent: str) -> None: + raw_pid = getattr(process, "pid", None) + self.pid = raw_pid if isinstance(raw_pid, int) and not isinstance(raw_pid, bool) and raw_pid > 0 else None + self.process = process + self.launch_intent = launch_intent + identity = f"PID {self.pid}" if self.pid is not None else "an unknown PID" + super().__init__( + f"{launch_intent} failed and cleanup could not confirm exit of {identity}" + ) + + def _hash_command(command: Sequence[str]) -> tuple[str, int]: rendered = "\0".join(str(part) for part in command) return hashlib.sha256(rendered.encode("utf-8", errors="ignore")).hexdigest(), len(rendered) @@ -40,6 +56,126 @@ def _stdio_policy_name(target: object, *, inherited_label: str) -> str: return "file" +def _process_exited(process: object, *, timeout: float) -> bool: + wait = getattr(process, "wait", None) + if callable(wait): + try: + wait(timeout=max(timeout, 0.0)) + return True + except subprocess.TimeoutExpired: + return False + except (ChildProcessError, ProcessLookupError): + return True + except BaseException: # noqa: BLE001 - cleanup must preserve the original failure + pass + poll = getattr(process, "poll", None) + if not callable(poll): + return False + try: + return poll() is not None + except BaseException: # noqa: BLE001 - cleanup must preserve the original failure + return False + + +def _process_pid(process: object) -> int | None: + pid = getattr(process, "pid", None) + return ( + pid + if ( + isinstance(pid, int) + and not isinstance(pid, bool) + and pid > 0 + and pid not in {os.getpid(), os.getppid()} + ) + else None + ) + + +def _process_group_exists(pid: int) -> bool | None: + try: + os.killpg(pid, 0) + return True + except ProcessLookupError: + return False + except PermissionError: + return True + except BaseException: # noqa: BLE001 - cleanup must preserve the original failure + return None + + +def _wait_for_process_group_exit(pid: int, *, timeout: float) -> bool: + deadline = time.monotonic() + max(timeout, 0.0) + while True: + exists = _process_group_exists(pid) + if exists is False: + return True + if exists is None or time.monotonic() >= deadline: + return False + try: + time.sleep(min(0.01, max(deadline - time.monotonic(), 0.0))) + except BaseException: # noqa: BLE001 - cancellation cannot abandon authority + return False + + +def _signal_spawned_process( + process: object, + sig: signal.Signals, + *, + process_group: bool, +) -> None: + pid = _process_pid(process) + if process_group and pid is not None: + try: + os.killpg(pid, sig) + return + except BaseException: # noqa: BLE001 - fall back to the Popen handle + pass + method_name = "terminate" if sig == signal.SIGTERM else "kill" + method = getattr(process, method_name, None) + if callable(method): + try: + method() + except BaseException: # noqa: BLE001 - cleanup must preserve the original failure + pass + + +def _terminate_unhanded_process( + process: object, + *, + term_timeout: float = 0.5, + kill_timeout: float = 1.0, + process_group: bool = True, +) -> bool: + """Best-effort terminate and reap a spawned process whose handoff failed.""" + + pid = _process_pid(process) + process_exited = _process_exited(process, timeout=0.0) + group_exited = ( + _process_group_exists(pid) is False + if process_group and pid is not None + else True + ) + if process_exited and group_exited: + return True + _signal_spawned_process(process, signal.SIGTERM, process_group=process_group) + process_exited = _process_exited(process, timeout=term_timeout) + group_exited = ( + _process_group_exists(pid) is False + if process_group and pid is not None + else True + ) + if process_exited and group_exited: + return True + _signal_spawned_process(process, signal.SIGKILL, process_group=process_group) + process_exited = _process_exited(process, timeout=kill_timeout) + group_exited = ( + _wait_for_process_group_exit(pid, timeout=kill_timeout) + if process_group and pid is not None + else True + ) + return process_exited and group_exited + + class ProcessLaunchMixin: _emit: Callable[..., Any] | None _launch_records: list[LaunchRecord] @@ -56,6 +192,7 @@ def _record_launch( stderr_policy: str, controller_input_owner_allowed: bool, active: bool, + emit_event: bool = True, ) -> None: command_hash, command_length = _hash_command(command) record = LaunchRecord( @@ -72,7 +209,7 @@ def _record_launch( launched_at_monotonic=time.monotonic(), ) self._launch_records.append(record) - if callable(self._emit): + if emit_event and callable(self._emit): self._emit( "process.launch", component="shared.process_runner", @@ -88,6 +225,15 @@ def _record_launch( active=bool(active), ) + def _mark_launch_inactive(self, process: object) -> None: + pid = getattr(process, "pid", None) + if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0: + return + for record in reversed(self._launch_records): + if record.pid == pid and record.active: + record.active = False + return + @staticmethod def _prepare_log_targets( *, @@ -135,6 +281,7 @@ def start_background( stdout_path=stdout_path, stderr_path=stderr_path, ) + process: subprocess.Popen[str] | None = None try: process = subprocess.Popen( list(cmd), @@ -160,11 +307,21 @@ def start_background( active=True, ) return process + except BaseException as exc: # noqa: BLE001 - a failed handoff must not orphan the child + if process is not None: + if _terminate_unhanded_process(process): + self._mark_launch_inactive(process) + else: + raise ProcessHandoffCleanupError( + process=process, + launch_intent="background_service", + ) from exc + raise finally: for handle in opened_handles: try: handle.close() - except OSError: + except BaseException: # noqa: BLE001 - diagnostics cleanup cannot orphan the child continue def start( @@ -194,29 +351,41 @@ def start_interactive_child( stdout: int | TextIO | None = None, stderr: int | TextIO | None = None, ) -> subprocess.Popen[str]: - process = subprocess.Popen( - list(cmd), - cwd=str(cwd) if cwd is not None else None, - env=dict(env) if env is not None else None, - text=True, - encoding="utf-8", - errors="replace", - stdin=stdin, - stdout=stdout, - stderr=stderr, - ) - self._record_launch( - launch_intent="interactive_child", - command=cmd, - pid=getattr(process, "pid", None), - cwd=cwd, - stdin_policy=_stdio_policy_name(stdin, inherited_label="inherit"), - stdout_policy=_stdio_policy_name(stdout, inherited_label="inherit"), - stderr_policy=_stdio_policy_name(stderr, inherited_label="inherit"), - controller_input_owner_allowed=True, - active=True, - ) - return process + process: subprocess.Popen[str] | None = None + try: + process = subprocess.Popen( + list(cmd), + cwd=str(cwd) if cwd is not None else None, + env=dict(env) if env is not None else None, + text=True, + encoding="utf-8", + errors="replace", + stdin=stdin, + stdout=stdout, + stderr=stderr, + ) + self._record_launch( + launch_intent="interactive_child", + command=cmd, + pid=getattr(process, "pid", None), + cwd=cwd, + stdin_policy=_stdio_policy_name(stdin, inherited_label="inherit"), + stdout_policy=_stdio_policy_name(stdout, inherited_label="inherit"), + stderr_policy=_stdio_policy_name(stderr, inherited_label="inherit"), + controller_input_owner_allowed=True, + active=True, + ) + return process + except BaseException as exc: # noqa: BLE001 - a failed handoff must not orphan the child + if process is not None: + if _terminate_unhanded_process(process, process_group=False): + self._mark_launch_inactive(process) + else: + raise ProcessHandoffCleanupError( + process=process, + launch_intent="interactive_child", + ) from exc + raise def tracked_launches(self, *, active_only: bool = False) -> list[dict[str, object]]: launches: list[dict[str, object]] = [] diff --git a/python/envctl_engine/shared/process_runner.py b/python/envctl_engine/shared/process_runner.py index 06b15a16f..b525d4f98 100644 --- a/python/envctl_engine/shared/process_runner.py +++ b/python/envctl_engine/shared/process_runner.py @@ -7,12 +7,14 @@ import subprocess import time as time from pathlib import Path -from typing import Any, Callable, Mapping, Sequence, TextIO +from typing import Any, Callable, Mapping, NoReturn, Sequence, TextIO from envctl_engine.shared.process_launch_support import ( LaunchRecord as LaunchRecord, + ProcessHandoffCleanupError as ProcessHandoffCleanupError, ProcessLaunchMixin, _hash_command as _hash_command, _stdio_policy_name as _stdio_policy_name, + _terminate_unhanded_process, ) from envctl_engine.shared.process_lifecycle_probe import ProcessLifecycleProbeMixin from envctl_engine.shared.process_streaming_support import ProcessStreamingMixin @@ -25,6 +27,35 @@ def __init__(self, *, emit: Callable[..., Any] | None = None) -> None: self._emit = emit self._launch_records: list[LaunchRecord] = [] + def _raise_unconfirmed_process_cleanup( + self, + *, + process: object, + command: Sequence[str], + cwd: str | Path | None, + stdin: object, + cause: BaseException, + ) -> NoReturn: + try: + self._record_launch( + launch_intent="foreground_cleanup_unconfirmed", + command=command, + pid=getattr(process, "pid", None), + cwd=cwd, + stdin_policy=_stdio_policy_name(stdin, inherited_label="inherit"), + stdout_policy="pipe", + stderr_policy="pipe", + controller_input_owner_allowed=False, + active=True, + emit_event=False, + ) + except BaseException: # noqa: BLE001 - the durable error still carries the process + pass + raise ProcessHandoffCleanupError( + process=process, + launch_intent="foreground_completion", + ) from cause + def run( self, cmd: Sequence[str], @@ -55,30 +86,14 @@ def run( process_started_callback(int(process.pid)) except Exception: pass - stdout, stderr = process.communicate(timeout=timeout) - return subprocess.CompletedProcess( - args=command, - returncode=process.returncode, - stdout=stdout, - stderr=stderr, - ) - except subprocess.TimeoutExpired as exc: - stdout = exc.stdout if isinstance(exc.stdout, str) else "" - stderr_prefix = exc.stderr if isinstance(exc.stderr, str) else "" - if process is not None and process.pid > 0: - try: - os.killpg(process.pid, signal.SIGTERM) - except OSError: - pass - try: - extra_stdout, extra_stderr = process.communicate(timeout=2.0) - if isinstance(extra_stdout, str): - stdout = f"{stdout}{extra_stdout}" - if isinstance(extra_stderr, str): - stderr_prefix = f"{stderr_prefix}{extra_stderr}" - except subprocess.TimeoutExpired: + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout if isinstance(exc.stdout, str) else "" + stderr_prefix = exc.stderr if isinstance(exc.stderr, str) else "" + if process.pid > 0: try: - os.killpg(process.pid, signal.SIGKILL) + os.killpg(process.pid, signal.SIGTERM) except OSError: pass try: @@ -88,14 +103,52 @@ def run( if isinstance(extra_stderr, str): stderr_prefix = f"{stderr_prefix}{extra_stderr}" except subprocess.TimeoutExpired: - pass - return self._timeout_result( - command, - timeout=timeout, + try: + os.killpg(process.pid, signal.SIGKILL) + except OSError: + pass + try: + extra_stdout, extra_stderr = process.communicate(timeout=2.0) + if isinstance(extra_stdout, str): + stdout = f"{stdout}{extra_stdout}" + if isinstance(extra_stderr, str): + stderr_prefix = f"{stderr_prefix}{extra_stderr}" + except subprocess.TimeoutExpired: + pass + cleanup_confirmed = _terminate_unhanded_process(process) + if not cleanup_confirmed: + self._raise_unconfirmed_process_cleanup( + process=process, + command=command, + cwd=cwd, + stdin=stdin, + cause=exc, + ) + return self._timeout_result( + command, + timeout=timeout, + stdout=stdout, + stderr_prefix=stderr_prefix, + ) + return subprocess.CompletedProcess( + args=command, + returncode=process.returncode, stdout=stdout, - stderr_prefix=stderr_prefix, + stderr=stderr, ) - except OSError: + except BaseException as exc: # noqa: BLE001 - a failed completion must not orphan the child + if isinstance(exc, ProcessHandoffCleanupError): + raise + if process is not None: + if _terminate_unhanded_process(process): + raise + self._raise_unconfirmed_process_cleanup( + process=process, + command=command, + cwd=cwd, + stdin=stdin, + cause=exc, + ) raise def run_probe( diff --git a/python/envctl_engine/shared/protocols.py b/python/envctl_engine/shared/protocols.py index fd2d8fa0b..e050af301 100644 --- a/python/envctl_engine/shared/protocols.py +++ b/python/envctl_engine/shared/protocols.py @@ -94,6 +94,7 @@ def save_run( emit: Callable[..., None], runtime_map_builder: Callable[[object], dict[str, object]], write_runtime_readiness_report: Callable[[Path], None] | None = None, + on_commit: Callable[[], None] | None = None, ) -> object: ... def save_resume_state( diff --git a/python/envctl_engine/shared/services.py b/python/envctl_engine/shared/services.py index ab470a7d2..ef11d5da3 100644 --- a/python/envctl_engine/shared/services.py +++ b/python/envctl_engine/shared/services.py @@ -1,16 +1,15 @@ from __future__ import annotations -from envctl_engine.state.models import RunState +from collections.abc import Callable + +from envctl_engine.state.models import RunState, ServiceRecord def frontend_env_for_project(state: RunState, project: str, *, host: str = "localhost") -> dict[str, str]: + backend = service_for_project_type(state, project=project, service_type="backend") backend_port = None - for service in state.services.values(): - if project_name_from_service_name(service.name) != project: - continue - if service.type != "backend": - continue - backend_port = service.actual_port if service.actual_port is not None else service.requested_port + if backend is not None: + backend_port = backend.actual_port if backend.actual_port is not None else backend.requested_port if backend_port is None: return {} backend_url = f"http://{host}:{backend_port}" @@ -47,9 +46,63 @@ def service_project_name(service: object) -> str: explicit = str(getattr(service, "project", "") or "").strip() if explicit: return explicit + typed_project = _typed_service_project_name(service) + if typed_project: + return typed_project return project_name_from_service_name(str(getattr(service, "name", "") or "")) +def _typed_service_project_name(service: object) -> str: + name = str(getattr(service, "name", "") or "").strip() + display = service_display_name(service_slug_from_record(service)) + suffix = f" {display}" if display else "" + if suffix and name.casefold().endswith(suffix.casefold()): + return name[: -len(suffix)].strip() + collision_marker = f" {display} " if display else "" + if collision_marker: + marker_index = name.casefold().rfind(collision_marker.casefold()) + if marker_index > 0: + collision_suffix = name[marker_index + len(collision_marker) :].strip().casefold() + if collision_suffix.startswith(("restart collision", "resume collision", "state collision")): + return name[:marker_index].strip() + return "" + + +def service_for_project_type(state: RunState, *, project: str, service_type: str) -> ServiceRecord | None: + project_key = str(project).strip().casefold() + type_key = str(service_type).strip().lower() + for service in state.services.values(): + if service_project_name(service).casefold() != project_key: + continue + if service_slug_from_record(service) == type_key: + return service + return None + + +def resolve_service_project_name( + service_name: str, + service: object | None, + *, + project_name_from_service: Callable[[str], object] | None = None, +) -> str: + """Resolve project ownership without discarding authoritative record metadata.""" + explicit = str(getattr(service, "project", "") or "").strip() + if explicit: + return explicit + typed_project = _typed_service_project_name(service) if service is not None else "" + if typed_project: + return typed_project + if project_name_from_service is not None: + legacy = str(project_name_from_service(service_name) or "").strip() + if legacy: + return legacy + if service is not None: + record_project = service_project_name(service) + if record_project: + return record_project + return project_name_from_service_name(str(service_name)) + + def service_matches_selector(service: object, selector: str) -> bool: target = str(selector or "").strip().lower() if not target: diff --git a/python/envctl_engine/startup/finalization.py b/python/envctl_engine/startup/finalization.py index 0796de926..fdafc8af2 100644 --- a/python/envctl_engine/startup/finalization.py +++ b/python/envctl_engine/startup/finalization.py @@ -3,6 +3,7 @@ import time import sys from collections.abc import Callable +import inspect from envctl_engine.planning.plan_agent.tmux_transport import attach_plan_agent_terminal from envctl_engine.runtime.command_router import Route @@ -105,7 +106,7 @@ def finalize_successful_startup( run_state = build_success_run_state(runtime, session) emit_preserved_service_merge(session) artifacts_started = time.monotonic() - runtime._write_artifacts(run_state, session.selected_contexts, errors=session.errors) + _write_artifacts_at_commit_boundary(runtime, session, run_state) emit_phase(session, "artifacts_write", artifacts_started, status="ok") release_lifecycle_operation(runtime) if requirements_timing_enabled(session.effective_route) and not suppress_timing_output(session.effective_route): @@ -310,11 +311,44 @@ def finalize_plan_agent_degraded_handoff_artifacts( validate_plan_agent_handoff(session, phase="degraded_finalization") run_state = build_success_run_state(runtime, session) artifacts_started = time.monotonic() - runtime._write_artifacts(run_state, session.selected_contexts, errors=session.errors) + _write_artifacts_at_commit_boundary(runtime, session, run_state) emit_phase(session, "artifacts_write", artifacts_started, status="degraded") return run_state +def _write_artifacts_at_commit_boundary( + runtime: StartupRuntime, + session: StartupSession, + run_state: RunState, +) -> None: + writer = runtime._write_artifacts + + def mark_committed() -> None: + session.authority_committed = True + + try: + parameters = inspect.signature(writer).parameters.values() + except (TypeError, ValueError): + parameters = () + # Treat the callback as an explicit protocol capability. A generic + # ``**kwargs`` proxy can forward into a legacy writer or raise after doing + # durable work; probing it and retrying would risk executing a commit twice. + supports_callback = any(parameter.name == "on_commit" for parameter in parameters) + if supports_callback: + writer( + run_state, + session.selected_contexts, + errors=session.errors, + on_commit=mark_committed, + ) + else: + writer(run_state, session.selected_contexts, errors=session.errors) + # Legacy adapters cannot expose their internal commit point. Preserve the + # historical contract by marking successful returns as committed too. + if not session.authority_committed: + mark_committed() + + def render_project_startup_warnings( runtime: StartupRuntime, *, diff --git a/python/envctl_engine/startup/finalization_failure.py b/python/envctl_engine/startup/finalization_failure.py index 75a14cf5c..81fd3cf4f 100644 --- a/python/envctl_engine/startup/finalization_failure.py +++ b/python/envctl_engine/startup/finalization_failure.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path import sys import time @@ -12,7 +12,7 @@ from envctl_engine.startup.finalization_run_state import _build_run_state from envctl_engine.requirements.core import dependency_definitions from envctl_engine.runtime.lifecycle_requirement_ports import component_port_values -from envctl_engine.state.models import RunState +from envctl_engine.state.models import RunState, ServiceRecord from envctl_engine.ui.color_policy import colors_enabled from envctl_engine.ui.path_links import local_paths_in_text, render_paths_in_terminal_text from envctl_engine.ui.status_symbols import STATUS_FAILURE @@ -42,8 +42,10 @@ def finalize(self) -> int: final_error = self._final_error() self.session.failure_message = final_error self.session.errors.append(final_error) + self._normalize_colliding_service_state() self._emit_failure(final_error) self._terminate_started_services() + self._prepare_collision_failure_state() self._release_ports() self._write_failure_artifacts(final_error) self._print_final_status(final_error) @@ -56,6 +58,83 @@ def _final_error(self) -> str: return self.error return f"Startup failed: {self.error}" + def _normalize_colliding_service_state(self) -> None: + """Give every newly started process a unique durable storage key. + + This runs before cleanup so duplicate hook-provided names cannot be + collapsed in the termination request. The ServiceRecord keeps its + authoritative project/service metadata; only its storage identity is + disambiguated. + """ + + replacement_maps = [*self.session.services_by_project.values(), self.session.unterminated_services] + reserved_names = set(self.session.preserved_services) + for services in replacement_maps: + reserved_names.update(services) + assigned_names = set(self.session.preserved_services) + identity_records: dict[int, tuple[str, ServiceRecord]] = {} + collision_rows = list(self.session.service_state_collision_rows) + recorded_replacements = { + str(row.get("replacement_name") or "").strip() + for row in collision_rows + if isinstance(row, dict) + } + + def normalized_services( + services: dict[str, ServiceRecord], + *, + fallback_project: str = "", + ) -> dict[str, ServiceRecord]: + normalized: dict[str, ServiceRecord] = {} + for original_name, service in services.items(): + prior_record = identity_records.get(id(service)) + if prior_record is not None: + prior_name, prior_service = prior_record + normalized[prior_name] = prior_service + continue + stored_name = original_name + stored_service = service + if stored_name in assigned_names: + stored_name = _collision_service_name( + original_name, + service, + reserved_names.union(assigned_names), + ) + if isinstance(stored_service, ServiceRecord): + stored_service = replace( + stored_service, + name=stored_name, + project=stored_service.project or fallback_project or None, + ) + self.session.service_state_collisions.add(original_name) + if stored_name not in recorded_replacements: + collision_rows.append( + { + "original_name": original_name, + "replacement_name": stored_name, + "replacement_pid": getattr(service, "pid", None), + "replacement_project": getattr(service, "project", None), + } + ) + recorded_replacements.add(stored_name) + assigned_names.add(stored_name) + reserved_names.add(stored_name) + identity_records[id(service)] = (stored_name, stored_service) + normalized[stored_name] = stored_service + return normalized + + normalized_by_project: dict[str, dict[str, ServiceRecord]] = {} + for project, services in self.session.services_by_project.items(): + normalized_by_project[project] = normalized_services( + dict(services), + fallback_project=project, + ) + self.session.services_by_project = normalized_by_project + self.session.unterminated_services = normalized_services( + dict(self.session.unterminated_services) + ) + self.session.service_state_collision_rows = collision_rows + def _emit_failure(self, final_error: str) -> None: failure_payload: dict[str, object] = { "mode": self.session.runtime_mode, @@ -74,8 +153,8 @@ def _terminate_started_services(self) -> None: started_services.update( { service_name: service - for project_name in self.session.started_context_names - for service_name, service in self.session.services_by_project.get(project_name, {}).items() + for project_services in self.session.services_by_project.values() + for service_name, service in project_services.items() } ) if started_services: @@ -103,6 +182,64 @@ def _terminate_started_services(self) -> None: service.status = "termination_failed" service.degraded = True + def _prepare_collision_failure_state(self) -> None: + collision_names = set(self.session.service_state_collisions) + retained_service_names = set(self.session.unterminated_services) + for services in self.session.services_by_project.values(): + retained_service_names.update(services) + collision_rows: list[dict[str, object]] = [] + for row in self.session.service_state_collision_rows: + normalized_row = dict(row) + replacement_name = str(normalized_row.get("replacement_name") or "").strip() + normalized_row["replacement_exit_unconfirmed"] = replacement_name in retained_service_names + collision_rows.append(normalized_row) + recorded_originals = { + str(row.get("original_name") or "").strip() + for row in collision_rows + if isinstance(row, dict) + } + collision_rows.extend( + { + "original_name": name, + "replacement_exit_unconfirmed": False, + } + for name in sorted(collision_names - recorded_originals) + ) + + occupied_projects = set(self.session.preserved_requirements).union( + self.session.requirements_by_project + ) + remapped_requirements = {} + requirement_rows = list(self.session.requirement_state_collision_rows) + for project, requirements in self.session.requirements_by_project.items(): + preserved = self.session.preserved_requirements.get(project) + if preserved is None: + remapped_requirements[project] = requirements + continue + if requirements == preserved: + continue + synthetic_project = _collision_project_name(project, occupied_projects) + occupied_projects.add(synthetic_project) + # The dictionary key is only a durable storage identity. Keep the + # authoritative project on the value so later port release and + # container cleanup use the owner that acquired the resources. + remapped_requirements[synthetic_project] = requirements + requirement_rows.append( + { + "original_project": project, + "replacement_project": synthetic_project, + "replacement_requirements_retained": True, + } + ) + self.session.requirements_by_project = remapped_requirements + collision_rows.extend(requirement_rows) + if collision_rows: + self.session.base_metadata["startup_state_collisions"] = collision_rows + # A collision-aware failure state can now retain both the previous + # authority and any replacement resource whose cleanup was unconfirmed. + if collision_rows or collision_names: + self.session.preserve_existing_state_on_failure = False + def _release_ports(self) -> None: if self.session.unterminated_services or _has_retained_managed_requirements(self.session): try: @@ -165,6 +302,28 @@ def _print_final_status(self, final_error: str) -> None: pass +def _collision_service_name(name: str, service: object, occupied: set[str]) -> str: + pid = getattr(service, "pid", None) + suffix = f"Restart Collision {pid}" if isinstance(pid, int) and pid > 0 else "Restart Collision" + base = f"{name} {suffix}" + candidate = base + index = 2 + while candidate in occupied: + candidate = f"{base} {index}" + index += 1 + return candidate + + +def _collision_project_name(project: str, occupied: set[str]) -> str: + base = f"{project} Restart Collision" + candidate = base + index = 2 + while candidate in occupied: + candidate = f"{base} {index}" + index += 1 + return candidate + + def finalize_failed_startup( *, runtime: StartupRuntime, diff --git a/python/envctl_engine/startup/finalization_run_state.py b/python/envctl_engine/startup/finalization_run_state.py index 326e5850d..4e18017dc 100644 --- a/python/envctl_engine/startup/finalization_run_state.py +++ b/python/envctl_engine/startup/finalization_run_state.py @@ -1,17 +1,22 @@ from __future__ import annotations import shlex +from pathlib import Path from typing import cast from envctl_engine.dashboard_metadata import ( APP_SERVICE_TYPES, DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY, + dashboard_project_configured_services_from_metadata, serialize_dashboard_project_configured_services, ) from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_env import effective_dependency_scope +from envctl_engine.shared.services import service_slug_from_record from envctl_engine.startup.protocols import ProjectContextLike, StartupRuntime +from envctl_engine.startup.run_reuse_dashboard_restore import metadata_without_dashboard_stopped_services from envctl_engine.startup.run_reuse_support import build_startup_identity_metadata +from envctl_engine.startup.service_execution_policy import additional_service_enabled_for_context from envctl_engine.startup.session import StartupSession from envctl_engine.state.models import RunState @@ -99,8 +104,25 @@ def _build_run_state(runtime: StartupRuntime, session: StartupSession, *, failed if requested_dependency_scope is not None: metadata["dependency_scope_requested"] = str(requested_dependency_scope) project_configured_services = _project_configured_services_metadata(runtime, session) - if project_configured_services: - metadata[DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY] = project_configured_services + existing_configured_services = dashboard_project_configured_services_from_metadata(metadata) + selected_project_keys = { + str(context.name).strip().casefold() + for context in session.selected_contexts + if str(getattr(context, "name", "")).strip() + } + merged_configured_services = { + project: service_types + for project, service_types in existing_configured_services.items() + if project.casefold() not in selected_project_keys + } + merged_configured_services.update(project_configured_services) + serialized_configured_services = serialize_dashboard_project_configured_services( + merged_configured_services + ) + if serialized_configured_services: + metadata[DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY] = serialized_configured_services + else: + metadata.pop(DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY, None) shared_dependency_project = _shared_dependency_dashboard_project(session) if shared_dependency_project: metadata["dashboard_dependency_scope"] = "shared" @@ -191,6 +213,18 @@ def _build_run_state(runtime: StartupRuntime, session: StartupSession, *, failed metadata["plan_agent_session_name"] = session_name if attach_command: metadata["plan_agent_attach_command"] = attach_command + if not failed: + restored_types: dict[str, set[str]] = {} + for project_name, services in session.services_by_project.items(): + for service in services.values(): + project = str(getattr(service, "project", "") or project_name).strip() + service_type = service_slug_from_record(service) + if project and service_type: + restored_types.setdefault(project, set()).add(service_type) + metadata = metadata_without_dashboard_stopped_services( + metadata, + restored_service_types_by_project=restored_types, + ) run_state = RunState( run_id=session.run_id, mode=session.runtime_mode, @@ -213,6 +247,16 @@ def _project_configured_services_metadata(runtime: StartupRuntime, session: Star for service_type in APP_SERVICE_TYPES if runtime._service_enabled_for_mode(session.runtime_mode, service_type) ] + service_types.extend( + str(getattr(service, "name", "") or "").strip().lower() + for service in tuple(getattr(runtime.config, "additional_services", ()) or ()) + if str(getattr(service, "name", "") or "").strip() + and additional_service_enabled_for_context( + service, + mode=session.runtime_mode, + project_root=Path(context.root), + ) + ) if service_types: configured[str(context.name)] = service_types return serialize_dashboard_project_configured_services(configured) diff --git a/python/envctl_engine/startup/lifecycle.py b/python/envctl_engine/startup/lifecycle.py index a82529b67..5a49ea71e 100644 --- a/python/envctl_engine/startup/lifecycle.py +++ b/python/envctl_engine/startup/lifecycle.py @@ -6,6 +6,7 @@ from envctl_engine.planning.plan_agent.launch import launch_plan_agent_terminals from envctl_engine.planning.plan_agent.omx_transport import validate_plan_agent_attach_target from envctl_engine.planning.plan_agent.tmux_transport import attach_plan_agent_terminal +from envctl_engine.runtime.lifecycle_operation_lease import release_lifecycle_operation from envctl_engine.startup.context_selection import select_startup_contexts from envctl_engine.startup.dependency_bootstrap import prepare_project_dependencies from envctl_engine.startup.disabled_startup_resolution import resolve_disabled_startup_mode_with_runtime @@ -95,9 +96,59 @@ def execute_startup_lifecycle(orchestrator: Any, route: Any) -> int: ), ) except RuntimeError as exc: - return finalize_failure(error=str(exc)) + return _finalize_startup_exception( + runtime=runtime, + session=session, + error=exc, + finalize_failure=finalize_failure, + ) except Exception as exc: - return finalize_failure(error=str(exc)) + return _finalize_startup_exception( + runtime=runtime, + session=session, + error=exc, + finalize_failure=finalize_failure, + ) + except BaseException as exc: # noqa: BLE001 - cancellation still requires failure finalization + try: + _finalize_startup_exception( + runtime=runtime, + session=session, + error=exc, + finalize_failure=finalize_failure, + ) + except BaseException as finalization_error: # noqa: BLE001 + if finalization_error is not exc: + try: + exc.add_note(f"startup cancellation finalization also failed: {finalization_error}") + except Exception: # noqa: BLE001 + pass + raise + + +def _finalize_startup_exception( + *, + runtime: Any, + session: StartupSession, + error: BaseException, + finalize_failure: Any, +) -> int: + if not session.authority_committed: + return int(finalize_failure(error=str(error))) + # The durable active revision is already committed. Presentation, terminal + # attachment, or dashboard failures after this boundary must not terminate + # its services or rewrite it as a failed startup. + release_lifecycle_operation(runtime) + try: + runtime._emit( + "startup.post_commit_error", + run_id=session.run_id, + error=str(error), + ) + except Exception: # noqa: BLE001 + pass + print(f"Startup state is active, but post-start handling failed: {error}") + return 1 def _pre_start_phases( diff --git a/python/envctl_engine/startup/protocols.py b/python/envctl_engine/startup/protocols.py index 61b3f05a0..f04872afc 100644 --- a/python/envctl_engine/startup/protocols.py +++ b/python/envctl_engine/startup/protocols.py @@ -163,7 +163,14 @@ def _wait_for_service_listener( ) -> None: ... def _refresh_service_listener_pids(self, service: ServiceRecord) -> None: ... def _run_interactive_dashboard_loop(self, state: RunState) -> int: ... - def _write_artifacts(self, state: RunState, contexts: list[ProjectContextLike], *, errors: list[str]) -> None: ... + def _write_artifacts( + self, + state: RunState, + contexts: list[ProjectContextLike], + *, + errors: list[str], + on_commit: Callable[[], None] | None = None, + ) -> None: ... def _print_summary(self, state: RunState, contexts: list[ProjectContextLike]) -> None: ... def _should_enter_post_start_interactive(self, route: Route) -> bool: ... def _run_pr_action(self, route: Route, targets: list[ProjectContextLike]) -> int: ... diff --git a/python/envctl_engine/startup/requirements_execution.py b/python/envctl_engine/startup/requirements_execution.py index 0845bcace..b2c8054dc 100644 --- a/python/envctl_engine/startup/requirements_execution.py +++ b/python/envctl_engine/startup/requirements_execution.py @@ -19,6 +19,7 @@ from envctl_engine.startup.startup_selection_support import restart_include_requirements from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RequirementsResult +from envctl_engine.state.project_runtime import requirements_for_project _DOCKER_SOCKET_PATTERNS = ( re.compile(r"unix://(?P[^\s;]+docker\.sock)"), @@ -69,7 +70,7 @@ def requirements_for_restart_context( project_names=[context.name], ) if previous is not None: - existing = previous.requirements.get(context.name) + existing = requirements_for_project(previous, context.name) if isinstance(existing, RequirementsResult): rt._emit( "requirements.restart.reuse", diff --git a/python/envctl_engine/startup/restart_prestop_support.py b/python/envctl_engine/startup/restart_prestop_support.py index ad978d581..1fafb72c5 100644 --- a/python/envctl_engine/startup/restart_prestop_support.py +++ b/python/envctl_engine/startup/restart_prestop_support.py @@ -6,17 +6,24 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.docker_service_runtime import state_uses_docker_services from envctl_engine.runtime.lifecycle_requirement_ports import release_port_reservation +from envctl_engine.runtime.lifecycle_service_termination import failed_listener_pids, unconfirmed_service_names from envctl_engine.state.lookup import call_state_loader from envctl_engine.runtime.runtime_context import resolve_port_allocator, resolve_process_runtime from envctl_engine.shared.process_cwd import process_cwd -from envctl_engine.shared.services import service_project_name, service_slug_from_record +from envctl_engine.shared.services import ( + service_project_name, + service_slug_from_record, +) from envctl_engine.startup.startup_selection_support import ( - _restart_selected_services, + _configured_service_projects_for_selector, + _observed_service_types_for_selector, restart_include_requirements, + restart_selected_services_for_type_map, + restart_service_types_by_project, + state_project_names, restart_target_projects, restart_target_projects_for_selected_services, ) -from envctl_engine.startup.run_reuse_dashboard_restore import metadata_without_dashboard_stopped_services from envctl_engine.startup.session import metadata_with_state_sources @@ -39,6 +46,7 @@ class RestartPrestopSelection: selected_services: set[str] target_projects: set[str] include_requirements: bool + service_types_by_project: dict[str, set[str]] = field(default_factory=dict) @dataclass(frozen=True, slots=True) @@ -103,37 +111,58 @@ def restart_start_route( selected_services: set[str], target_projects: set[str], include_requirements: bool, + service_types_by_project: dict[str, set[str]] | None = None, ) -> Route: + type_map = ( + { + project: sorted(service_types) + for project, service_types in sorted(service_types_by_project.items(), key=lambda item: item[0].casefold()) + } + if service_types_by_project is not None + else None + ) return Route( command="start", mode=restart_lookup_mode, raw_args=route.raw_args, passthrough_args=route.passthrough_args, - projects=route.projects, + projects=sorted(target_projects, key=str.casefold) if target_projects else route.projects, flags={ **route.flags, "_restart_request": True, "_restart_selected_services": sorted(selected_services), "_restart_target_projects": sorted(target_projects), "_restart_include_requirements": include_requirements, + **({"_restart_service_types_by_project": type_map} if type_map is not None else {}), }, ) def restart_prestop_selection(*, state: Any, route: Route, runtime: Any) -> RestartPrestopSelection: - selected_services = _restart_selected_services(state=state, route=route) target_projects = restart_target_projects(state=state, route=route, runtime=runtime) include_requirements = restart_include_requirements(route) - if include_requirements and not target_projects: + if include_requirements and not target_projects and not isinstance(route.flags.get("services"), list): target_projects = restart_target_projects_for_selected_services( - selected_services=selected_services, + selected_services=set(), state=state, runtime=runtime, ) + service_types_by_project = restart_service_types_by_project( + state=state, + route=route, + runtime=runtime, + target_projects=target_projects, + ) + selected_services = restart_selected_services_for_type_map( + state=state, + runtime=runtime, + service_types_by_project=service_types_by_project, + ) return RestartPrestopSelection( selected_services=selected_services, target_projects=target_projects, include_requirements=include_requirements, + service_types_by_project=service_types_by_project, ) @@ -159,11 +188,35 @@ def handle_restart_prestop( session.runtime_mode = restart_lookup_mode return None session.restart_state = resumed + previous_preserve_flag = bool(getattr(session, "preserve_existing_state_on_failure", False)) + source_services = { + **dict(getattr(resumed, "services", {})), + **dict(session.preserved_services), + } + source_requirements = { + **dict(getattr(resumed, "requirements", {})), + **dict(session.preserved_requirements), + } + source_metadata = metadata_with_state_sources( + {**dict(getattr(resumed, "metadata", {})), **dict(session.base_metadata)}, + resumed, + ) + session.preserved_services = source_services + session.preserved_requirements = source_requirements + session.base_metadata = source_metadata + session.preserve_existing_state_on_failure = True selection = restart_prestop_selection(state=resumed, route=route, runtime=runtime) selected_services = selection.selected_services target_projects = selection.target_projects include_requirements = selection.include_requirements + _validate_restart_selection( + state=resumed, + route=route, + runtime=runtime, + target_projects=target_projects, + service_types_by_project=selection.service_types_by_project, + ) runtime._emit( "restart.selection", include_requirements=include_requirements, @@ -177,6 +230,12 @@ def handle_restart_prestop( prestop_policy, context={"component": "startup_orchestrator", "op_id": "restart.prestop"}, ) + preservation = restart_prestop_preservation( + resumed, + selected_services=selected_services, + include_requirements=include_requirements, + target_projects=target_projects, + ) with ( use_spinner_policy_fn(prestop_policy), spinner_factory("Restarting services...", enabled=use_prestop_spinner) as prestop_spinner, @@ -196,43 +255,34 @@ def handle_restart_prestop( aggressive=False, verify_ownership=True, ) - failed_services = _failed_service_names(termination_result) + failed_services = unconfirmed_service_names(termination_result, selected_services) if failed_services: session.preserve_existing_state_on_failure = True raise RuntimeError( "Restart aborted because existing services could not be stopped: " + ", ".join(sorted(failed_services)) ) - failed_orphan_pids = terminate_restart_orphan_listeners( + orphan_result = terminate_restart_orphan_listeners( state=resumed, selected_services=selected_services, aggressive=True, ) + failed_orphan_pids = failed_listener_pids(orphan_result) + if failed_orphan_pids is None: + session.preserve_existing_state_on_failure = True + raise RuntimeError("Restart aborted because orphan listener cleanup was not confirmed") if failed_orphan_pids: session.preserve_existing_state_on_failure = True raise RuntimeError( "Restart aborted because orphan listeners could not be stopped: " + ", ".join(str(pid) for pid in sorted(failed_orphan_pids)) ) - preservation = restart_prestop_preservation( - resumed, - selected_services=selected_services, - include_requirements=include_requirements, - target_projects=target_projects, - ) for requirements in preservation.requirements_to_release.values(): runtime._release_requirement_ports(requirements) session.preserved_requirements = dict(preservation.preserved_requirements) session.preserved_services = dict(preservation.preserved_services) - requested_service_names = { - str(name).strip() for name in route.flags.get("services", []) if str(name).strip() - } - inherited_metadata = metadata_without_dashboard_stopped_services( - {**dict(resumed.metadata), **dict(session.base_metadata)}, - restored_service_names=requested_service_names, - ) session.base_metadata = metadata_with_state_sources( - inherited_metadata, + {**dict(resumed.metadata), **dict(session.base_metadata)}, resumed, ) if use_prestop_spinner: @@ -245,6 +295,10 @@ def handle_restart_prestop( message="Restart pre-stop complete", ) except Exception: + session.preserved_services = source_services + session.preserved_requirements = source_requirements + session.base_metadata = source_metadata + session.preserve_existing_state_on_failure = True if use_prestop_spinner: prestop_spinner.fail("Restart pre-stop failed") runtime._emit( @@ -269,15 +323,104 @@ def handle_restart_prestop( selected_services=selected_services, target_projects=target_projects, include_requirements=include_requirements, + service_types_by_project=selection.service_types_by_project, ) session.runtime_mode = restart_lookup_mode + session.preserve_existing_state_on_failure = previous_preserve_flag return None -def _failed_service_names(result: object) -> set[str]: - if not isinstance(result, (set, frozenset)): - return set() - return {str(name).strip() for name in result if str(name).strip()} +def _validate_restart_selection( + *, + state: Any, + route: Route, + runtime: Any, + target_projects: set[str], + service_types_by_project: dict[str, set[str]] | None = None, +) -> None: + requested_projects = {str(project).strip() for project in route.projects if str(project).strip()} + if requested_projects: + known_projects = {name.casefold() for name in state_project_names(runtime=runtime, state=state)} + unknown_projects = sorted( + project for project in requested_projects if project.casefold() not in known_projects + ) + if unknown_projects: + raise RuntimeError("No active restart target found for project(s): " + ", ".join(unknown_projects)) + + service_selectors = route.flags.get("services") + requested_services = [ + str(selector).strip() + for selector in (service_selectors if isinstance(service_selectors, list) else []) + if str(selector).strip() + ] + requested_project_keys = {project.casefold() for project in requested_projects} + selector_matches: dict[str, dict[str, set[str]]] = {} + unmatched_services: list[str] = [] + for selector in requested_services: + matches: dict[str, set[str]] = {} + for source in ( + _observed_service_types_for_selector( + state=state, + runtime=runtime, + selector=selector, + ), + _configured_service_projects_for_selector( + state=state, + runtime=runtime, + selector=selector, + ), + ): + for project, service_types in source.items(): + matches.setdefault(project, set()).update(service_types) + scoped_matches = { + project: service_types + for project, service_types in matches.items() + if not requested_project_keys or project.casefold() in requested_project_keys + } + selector_matches[selector] = scoped_matches + if not scoped_matches: + unmatched_services.append(selector) + if unmatched_services: + raise RuntimeError("No matching services found for restart: " + ", ".join(unmatched_services)) + if requested_services and not target_projects: + possible_projects = { + project.casefold() + for matches in selector_matches.values() + for project in matches + } + if len(possible_projects) > 1: + raise RuntimeError( + "Restart service selector is ambiguous across active projects; add --project: " + + ", ".join(requested_services) + ) + raise RuntimeError("No active restart target found for service(s): " + ", ".join(requested_services)) + + if service_types_by_project is None: + service_types_by_project = restart_service_types_by_project( + state=state, + route=route, + runtime=runtime, + target_projects=target_projects, + ) + target_keys = {project.casefold() for project in target_projects} + final_by_key = { + project.casefold(): set(service_types) + for project, service_types in service_types_by_project.items() + } + filtered_services = [ + selector + for selector, matches in selector_matches.items() + if not any( + project.casefold() in target_keys + and bool(service_types.intersection(final_by_key.get(project.casefold(), set()))) + for project, service_types in matches.items() + ) + ] + if filtered_services: + raise RuntimeError( + "Restart service selector conflicts with the requested launch scope: " + + ", ".join(filtered_services) + ) def restart_prestop_preservation( @@ -289,11 +432,15 @@ def restart_prestop_preservation( ) -> RestartPrestopPreservation: requirements_to_release: dict[str, object] = {} preserved_requirements: dict[str, object] = {} - for project_name, requirements in getattr(state, "requirements", {}).items(): - if include_requirements and (not target_projects or project_name in target_projects): - requirements_to_release[project_name] = requirements + target_project_keys = {str(project).strip().casefold() for project in target_projects if str(project).strip()} + for storage_name, requirements in getattr(state, "requirements", {}).items(): + project_name = str(getattr(requirements, "project", "") or storage_name).strip() + if include_requirements and ( + not target_project_keys or project_name.casefold() in target_project_keys + ): + requirements_to_release[storage_name] = requirements else: - preserved_requirements[project_name] = requirements + preserved_requirements[storage_name] = requirements preserved_services = { name: service for name, service in getattr(state, "services", {}).items() if name not in selected_services } diff --git a/python/envctl_engine/startup/resume_orchestrator.py b/python/envctl_engine/startup/resume_orchestrator.py index 45e512fa6..6ef5a440f 100644 --- a/python/envctl_engine/startup/resume_orchestrator.py +++ b/python/envctl_engine/startup/resume_orchestrator.py @@ -7,6 +7,7 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_startup_support import mark_run_reused from envctl_engine.runtime.lifecycle_operation_lease import release_lifecycle_operation +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.state.models import RunState from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.runtime_map import build_runtime_map @@ -118,17 +119,22 @@ def emit_phase(phase: str, started_at: float, **extra: object) -> None: ) restore_started = time.monotonic() restore_errors = self.restore_missing(state, missing_services, route=route) + restored_projects = { + project.casefold() + for name in missing_services + if ( + project := resolve_service_project_name( + name, + state.services.get(name), + project_name_from_service=rt._project_name_from_service, # type: ignore[attr-defined] + ) + ) + } emit_phase( "restore_missing", restore_started, status="error" if restore_errors else "ok", - project_count=len( - { - rt._project_name_from_service(name) - for name in missing_services - if rt._project_name_from_service(name) - } - ), # type: ignore[attr-defined] + project_count=len(restored_projects), error_count=len(restore_errors), ) if legacy_resume: diff --git a/python/envctl_engine/startup/resume_restore_execution.py b/python/envctl_engine/startup/resume_restore_execution.py index d5be89152..54fdb6847 100644 --- a/python/envctl_engine/startup/resume_restore_execution.py +++ b/python/envctl_engine/startup/resume_restore_execution.py @@ -8,6 +8,7 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.docker_service_runtime import state_uses_docker_services +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.state.models import RunState from envctl_engine.startup.resume_restore_project import ResumeProjectRestoreRunner, _round_ms from envctl_engine.startup.resume_restore_results import apply_restore_result @@ -213,15 +214,24 @@ def _execute_restore_missing( def _projects_to_restore(rt: object, *, state: RunState, missing_services: list[str]) -> list[str]: - projects_to_restore: set[str] = set() - for service_name in missing_services: - project = rt._project_name_from_service(service_name) # type: ignore[attr-defined] + projects_to_restore: dict[str, str] = {} + + def add_project(raw_project: object) -> None: + project = str(raw_project or "").strip() if project: - projects_to_restore.add(project) + projects_to_restore.setdefault(project.casefold(), project) + + for service_name in missing_services: + project = resolve_service_project_name( + service_name, + state.services.get(service_name), + project_name_from_service=rt._project_name_from_service, # type: ignore[attr-defined] + ) + add_project(project) for project, requirements in state.requirements.items(): if not rt._requirements_ready(requirements): # type: ignore[attr-defined] - projects_to_restore.add(project) - return sorted(projects_to_restore) + add_project(getattr(requirements, "project", "") or project) + return sorted(projects_to_restore.values(), key=lambda project: (project.casefold(), project)) def _route_for_resume_restore( diff --git a/python/envctl_engine/startup/resume_restore_policy.py b/python/envctl_engine/startup/resume_restore_policy.py index a751cc429..950b17a96 100644 --- a/python/envctl_engine/startup/resume_restore_policy.py +++ b/python/envctl_engine/startup/resume_restore_policy.py @@ -8,7 +8,33 @@ from envctl_engine.requirements.core import dependency_definitions from envctl_engine.shared.parsing import parse_bool from envctl_engine.shared.protocols import PortAllocator, StateRepository +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord +from envctl_engine.state.project_runtime import ( + requirement_key_for_project as _requirement_key_for_project, + requirements_for_project as _requirements_for_project, +) + + +def _store_requirements_for_project( + state: RunState, + *, + project: str, + requirements: RequirementsResult, +) -> str: + """Update one project without creating case-only duplicate keys.""" + + target = str(project).strip().casefold() + key = _requirement_key_for_project(state, project) or str(project).strip() + if not key: + raise ValueError("project is required when storing restored requirements") + state.requirements[key] = requirements + for alias in list(state.requirements): + if alias == key: + continue + if str(alias).strip().casefold() == target: + state.requirements.pop(alias, None) + return key def _restore_parallel_config( @@ -204,7 +230,7 @@ def context_for_project(orchestrator, state: RunState, project: str) -> object | rt = orchestrator.runtime discovered = rt._discover_projects(mode=state.mode) # type: ignore[attr-defined] for context in discovered: - if context.name.lower() == project.lower(): + if context.name.casefold() == project.casefold(): orchestrator.apply_ports_to_context(context, state) return context @@ -235,7 +261,12 @@ def project_root(orchestrator, state: RunState, project: str) -> Path | None: return rt.config.base_dir # type: ignore[attr-defined] for service_name, service in state.services.items(): - if rt._project_name_from_service(service_name).lower() != project.lower(): # type: ignore[attr-defined] + service_project = resolve_service_project_name( + service_name, + service, + project_name_from_service=rt._project_name_from_service, # type: ignore[attr-defined] + ) + if service_project.casefold() != project.casefold(): continue cwd_raw = getattr(service, "cwd", None) if not isinstance(cwd_raw, str) or not cwd_raw.strip(): @@ -258,7 +289,7 @@ def apply_ports_to_context(orchestrator, context: object, state: RunState) -> No if not isinstance(context_ports, dict): return - requirements = state.requirements.get(project) + requirements = _requirements_for_project(state, project) if requirements is not None: for definition in dependency_definitions(): plan = context_ports.get(definition.resources[0].legacy_port_key) @@ -267,7 +298,12 @@ def apply_ports_to_context(orchestrator, context: object, state: RunState) -> No rt._set_plan_port_from_component(plan, requirements.component(definition.id)) # type: ignore[attr-defined] for service_name, service in state.services.items(): - if rt._project_name_from_service(service_name).lower() != project.lower(): # type: ignore[attr-defined] + service_project = resolve_service_project_name( + service_name, + service, + project_name_from_service=rt._project_name_from_service, # type: ignore[attr-defined] + ) + if service_project.casefold() != project.casefold(): continue service_type = (getattr(service, "type", "") or "").strip().lower() port = getattr(service, "actual_port", None) diff --git a/python/envctl_engine/startup/resume_restore_project.py b/python/envctl_engine/startup/resume_restore_project.py index 233814ee7..94c14465e 100644 --- a/python/envctl_engine/startup/resume_restore_project.py +++ b/python/envctl_engine/startup/resume_restore_project.py @@ -6,9 +6,11 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.lifecycle_service_termination import release_service_ports -from envctl_engine.state.models import RunState, ServiceRecord +from envctl_engine.shared.services import resolve_service_project_name +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord from envctl_engine.startup.resume_restore_policy import ( _configured_restore_service_types, + _requirements_for_project, _requirements_reuse_decision, _reserve_application_service_ports, _resume_terminate_aggressive, @@ -17,6 +19,21 @@ ) +class _StaleServiceTerminationError(RuntimeError): + def __init__( + self, + *, + unconfirmed_service_names: list[str], + confirmed_service_names: list[str], + ) -> None: + self.unconfirmed_service_names = list(unconfirmed_service_names) + self.confirmed_service_names = list(confirmed_service_names) + super().__init__( + "stale service termination could not be confirmed: " + + ", ".join(sorted(self.unconfirmed_service_names)) + ) + + @dataclass(slots=True) class ResumeProjectRestoreRunner: orchestrator: object @@ -102,15 +119,20 @@ def mark_step( ) project_service_names = [ name - for name in list(self.state.services.keys()) - if self.rt._project_name_from_service(name).lower() == project.lower() # type: ignore[attr-defined] + for name, service in list(self.state.services.items()) + if resolve_service_project_name( + name, + service, + project_name_from_service=self.rt._project_name_from_service, # type: ignore[attr-defined] + ).casefold() + == project.casefold() ] original_services: dict[str, ServiceRecord] = { name: self.state.services[name] for name in project_service_names if name in self.state.services } missing_set = set(self.missing_services) missing_project_service_names = [name for name in project_service_names if name in missing_set] - original_requirements = self.state.requirements.get(project) + original_requirements = _requirements_for_project(self.state, project) context_root = getattr(context, "root", None) requirements_reused, requirements_reuse_reason = _requirements_reuse_decision( self.orchestrator, @@ -145,21 +167,42 @@ def mark_step( elif configured_service_types: selected_service_types = set(configured_service_types) - existing_requirements = original_requirements - if existing_requirements is not None and not requirements_reused: + confirmed_old_service_names: list[str] = [] + replacement_requirements: RequirementsResult | None = None + try: self._report_progress( project, - f"{prefix} Releasing previous requirement ports...", - single_message=f"{prefix} Releasing previous requirement ports for {project}...", + f"{prefix} Stopping stale services...", + single_message=f"{prefix} Stopping stale services for {project}...", ) - release_started = time.monotonic() - self.rt._release_requirement_ports(existing_requirements) # type: ignore[attr-defined] + stop_started = time.monotonic() + terminated_count = self._stop_stale_services( + service_names=selected_service_names, + original_services=original_services, + ) + confirmed_old_service_names = list(selected_service_names) mark_step( - "release_requirement_ports", - duration_ms=_round_ms(time.monotonic() - release_started), + "stop_stale_services", + duration_ms=_round_ms(time.monotonic() - stop_started), + stale_service_count=len(selected_service_names), + terminated_count=terminated_count, + aggressive=_resume_terminate_aggressive(self.orchestrator, self.rt), ) - try: + existing_requirements = original_requirements + if existing_requirements is not None and not requirements_reused: + self._report_progress( + project, + f"{prefix} Releasing previous requirement ports...", + single_message=f"{prefix} Releasing previous requirement ports for {project}...", + ) + release_started = time.monotonic() + self.rt._release_requirement_ports(existing_requirements) # type: ignore[attr-defined] + mark_step( + "release_requirement_ports", + duration_ms=_round_ms(time.monotonic() - release_started), + ) + if requirements_reused and requirements_for_services is not None: requirements = requirements_for_services mark_step("start_requirements", duration_ms=0.0, status="reused") @@ -169,6 +212,7 @@ def mark_step( requirements = self.rt._start_requirements_for_project( context, mode=self.state.mode, route=project_route_for_startup ) # type: ignore[attr-defined] + replacement_requirements = requirements mark_step( "start_requirements", duration_ms=_round_ms(time.monotonic() - requirements_started), @@ -195,26 +239,10 @@ def mark_step( "error": error_message, "steps": step_durations_ms, "total_ms": project_total, + "remove_service_names": confirmed_old_service_names, + "requirements": requirements, } - self._report_progress( - project, - f"{prefix} Stopping stale services...", - single_message=f"{prefix} Stopping stale services for {project}...", - ) - stop_started = time.monotonic() - terminated_count = self._stop_stale_services( - service_names=selected_service_names, - original_services=original_services, - ) - mark_step( - "stop_stale_services", - duration_ms=_round_ms(time.monotonic() - stop_started), - stale_service_count=len(selected_service_names), - terminated_count=terminated_count, - aggressive=_resume_terminate_aggressive(self.orchestrator, self.rt), - ) - self._report_progress( project, f"{prefix} Reserving service ports...", @@ -275,7 +303,25 @@ def mark_step( "requirements": requirements, "remove_service_names": selected_service_names, } - except RuntimeError as exc: + except Exception as exc: # noqa: BLE001 + confirmed_from_error = getattr(exc, "confirmed_service_names", None) + if isinstance(confirmed_from_error, list): + confirmed_old_service_names = [ + name for name in confirmed_from_error if isinstance(name, str) and name + ] + exception_requirements = getattr(exc, "requirements", None) + if isinstance(exception_requirements, RequirementsResult): + replacement_requirements = exception_requirements + raw_unterminated = getattr(exc, "unterminated_services", None) + unterminated_services = ( + { + name: service + for name, service in raw_unterminated.items() + if isinstance(name, str) and isinstance(service, ServiceRecord) + } + if isinstance(raw_unterminated, dict) + else {} + ) mark_step("exception", status="error", error=str(exc)) self._mark_failure( project, @@ -295,6 +341,9 @@ def mark_step( "error": str(exc), "steps": step_durations_ms, "total_ms": project_total, + "remove_service_names": confirmed_old_service_names, + "requirements": replacement_requirements, + "unterminated_services": unterminated_services, } def _route_with_spinner_updates(self, project: str, prefix: str) -> Route: @@ -338,6 +387,7 @@ def _stop_stale_services( original_services: dict[str, ServiceRecord], ) -> int: terminated_count = 0 + unconfirmed: list[str] = [] aggressive_terminate = _resume_terminate_aggressive(self.orchestrator, self.rt) for service_name in service_names: service = original_services.get(service_name) @@ -351,6 +401,15 @@ def _stop_stale_services( if terminated: terminated_count += 1 release_service_ports(self.rt, name=service_name, service=service) + else: + unconfirmed.append(service_name) + if unconfirmed: + raise _StaleServiceTerminationError( + unconfirmed_service_names=unconfirmed, + confirmed_service_names=[ + name for name in service_names if name in original_services and name not in unconfirmed + ], + ) return terminated_count def _report_progress( diff --git a/python/envctl_engine/startup/resume_restore_results.py b/python/envctl_engine/startup/resume_restore_results.py index 2ba46d10d..1a589c15e 100644 --- a/python/envctl_engine/startup/resume_restore_results.py +++ b/python/envctl_engine/startup/resume_restore_results.py @@ -1,7 +1,12 @@ from __future__ import annotations from collections.abc import Mapping +from dataclasses import replace +from envctl_engine.startup.resume_restore_policy import ( + _requirements_for_project, + _store_requirements_for_project, +) from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord @@ -22,29 +27,95 @@ def apply_restore_result( if timing_enabled: timing_lines.append(format_project_timing_line(project, step_map, project_total)) - error_message = str(result.get("error") or "").strip() - if error_message: - mark_restore_failure_requirements(state, project=project, error=error_message) - errors.append(f"{project}: {error_message}") - return - remove_names_raw = result.get("remove_service_names") if isinstance(remove_names_raw, list): for service_name in remove_names_raw: if isinstance(service_name, str): + existing = state.services.get(service_name) + existing_project = str(getattr(existing, "project", "") or "").strip() + if existing_project and existing_project.casefold() != project.casefold(): + continue state.services.pop(service_name, None) project_services_raw = result.get("project_services") if isinstance(project_services_raw, dict): - for service_name, record in project_services_raw.items(): - if isinstance(service_name, str) and isinstance(record, ServiceRecord): - state.services[service_name] = record + _store_project_services(state, project=project, services=project_services_raw) requirements_result = result.get("requirements") if isinstance(requirements_result, RequirementsResult): - state.requirements[project] = requirements_result + _store_requirements_for_project( + state, + project=project, + requirements=requirements_result, + ) + raw_unterminated = result.get("unterminated_services") + if isinstance(raw_unterminated, dict): + stored_names = _store_unterminated_services(state, raw_unterminated) + if stored_names: + existing_failed = state.metadata.get("termination_failed_services") + failed_names = ( + [name for name in existing_failed if isinstance(name, str)] + if isinstance(existing_failed, list) + else [] + ) + state.metadata["termination_failed_services"] = sorted(set(failed_names).union(stored_names)) + + error_message = str(result.get("error") or "").strip() + if error_message: + mark_restore_failure_requirements(state, project=project, error=error_message) + errors.append(f"{project}: {error_message}") + + +def _store_unterminated_services(state: RunState, services: dict[object, object]) -> list[str]: + occupied = set(state.services) + stored_names: list[str] = [] + for raw_name, service in services.items(): + if not isinstance(service, ServiceRecord): + continue + name = str(raw_name).strip() or service.name + stored_name = name + if stored_name in occupied: + stored_name = _unique_resume_collision_name(name, service, occupied) + occupied.add(stored_name) + state.services[stored_name] = replace( + service, + name=stored_name, + status="termination_failed", + degraded=True, + ) + stored_names.append(stored_name) + return stored_names + + +def _store_project_services(state: RunState, *, project: str, services: dict[object, object]) -> None: + occupied = set(state.services) + for raw_name, service in services.items(): + if not isinstance(service, ServiceRecord): + continue + name = str(raw_name).strip() or service.name + stored_service = service if str(service.project or "").strip() else replace(service, project=project) + stored_name = name + if stored_name in occupied: + stored_name = _unique_resume_collision_name(name, stored_service, occupied) + stored_service = replace(stored_service, name=stored_name) + elif stored_service.name != stored_name: + stored_service = replace(stored_service, name=stored_name) + occupied.add(stored_name) + state.services[stored_name] = stored_service + + +def _unique_resume_collision_name(name: str, service: ServiceRecord, occupied: set[str]) -> str: + pid = service.pid + suffix = f"Resume Collision {pid}" if isinstance(pid, int) and pid > 0 else "Resume Collision" + base = f"{name} {suffix}" + candidate = base + index = 2 + while candidate in occupied: + candidate = f"{base} {index}" + index += 1 + return candidate def mark_restore_failure_requirements(state: RunState, *, project: str, error: str) -> None: - existing = state.requirements.get(project) + existing = _requirements_for_project(state, project) if not isinstance(existing, RequirementsResult): return existing.health = "degraded" diff --git a/python/envctl_engine/startup/run_reuse_dashboard_restore.py b/python/envctl_engine/startup/run_reuse_dashboard_restore.py index 759efee4b..515af1e59 100644 --- a/python/envctl_engine/startup/run_reuse_dashboard_restore.py +++ b/python/envctl_engine/startup/run_reuse_dashboard_restore.py @@ -4,7 +4,9 @@ from dataclasses import dataclass from typing import Any +from envctl_engine.dashboard_metadata import normalize_dashboard_service_types from envctl_engine.runtime.command_router import Route +from envctl_engine.shared.services import service_display_name from envctl_engine.startup.run_reuse_decision import mark_run_reused from envctl_engine.startup.session import metadata_with_state_sources @@ -18,15 +20,16 @@ def dashboard_stopped_service_entries(state: object) -> list[dict[str, str]]: if not isinstance(item, Mapping): continue project = str(item.get("project", "") or "").strip() - service_type = str(item.get("type", "") or "").strip().lower() + normalized_types = normalize_dashboard_service_types([item.get("type", "")]) + service_type = normalized_types[0] if normalized_types else "" name = str(item.get("name", "") or "").strip() - if not project or service_type not in {"backend", "frontend"}: + if not project or not service_type: continue entries.append( { "project": project, "type": service_type, - "name": name or f"{project} {service_type.title()}", + "name": name or f"{project} {service_display_name(service_type)}", } ) return entries @@ -35,19 +38,32 @@ def dashboard_stopped_service_entries(state: object) -> list[dict[str, str]]: def metadata_without_dashboard_stopped_services( metadata: Mapping[str, object], *, - restored_service_names: set[str], + restored_service_names: set[str] | None = None, + restored_service_types_by_project: Mapping[str, object] | None = None, ) -> dict[str, object]: updated = dict(metadata) raw = updated.get("dashboard_stopped_services") if not isinstance(raw, list): return updated + restored_names = { + str(selector).strip().casefold().removeprefix("service:").strip() + for selector in (restored_service_names or set()) + if str(selector).strip() + } + restored_types = { + str(project).strip().casefold(): set(normalize_dashboard_service_types(service_types)) + for project, service_types in (restored_service_types_by_project or {}).items() + if str(project).strip() and isinstance(service_types, (list, tuple, set, frozenset)) + } remaining: list[object] = [] for item in raw: if not isinstance(item, Mapping): remaining.append(item) continue name = str(item.get("name", "") or "").strip() - if name in restored_service_names: + project = str(item.get("project", "") or "").strip() + service_type = str(item.get("type", "") or "").strip().lower() + if name.casefold() in restored_names or service_type in restored_types.get(project.casefold(), set()): continue remaining.append(dict(item)) if remaining: @@ -106,10 +122,7 @@ def _apply_restore(self, *, restore_entries: list[dict[str, str]], contexts_to_s stopped_service_types = sorted({entry["type"] for entry in restore_entries}) target_project_names = sorted({entry["project"] for entry in restore_entries}, key=str.casefold) self.session.base_metadata = metadata_with_state_sources( - metadata_without_dashboard_stopped_services( - mark_run_reused(self.candidate_state.metadata, reason="restore_stopped_services"), - restored_service_names=set(stopped_service_names), - ), + mark_run_reused(self.candidate_state.metadata, reason="restore_stopped_services"), self.candidate_state, ) self.session.preserved_services = dict(self.candidate_state.services) @@ -119,6 +132,16 @@ def _apply_restore(self, *, restore_entries: list[dict[str, str]], contexts_to_s target_project_names=target_project_names, stopped_service_names=stopped_service_names, stopped_service_types=stopped_service_types, + stopped_service_types_by_project={ + project: sorted( + { + entry["type"] + for entry in restore_entries + if entry["project"].casefold() == project.casefold() + } + ) + for project in target_project_names + }, ) self._emit_restore( target_project_names=target_project_names, @@ -131,6 +154,7 @@ def _apply_restart_route( target_project_names: list[str], stopped_service_names: list[str], stopped_service_types: list[str], + stopped_service_types_by_project: dict[str, list[str]], ) -> None: route = self.session.effective_route self.session.effective_route = Route( @@ -145,39 +169,47 @@ def _apply_restart_route( "_restore_dashboard_stopped_services": True, "services": stopped_service_names, "restart_service_types": stopped_service_types, + "_restart_service_types_by_project": stopped_service_types_by_project, "restart_include_requirements": False, }, ) def _emit_restore(self, *, target_project_names: list[str], stopped_service_names: list[str]) -> None: route = self.session.effective_route - self.emit_phase( - self.session, - "auto_resume_evaluate", - self.reuse_started, - status="restore_stopped_services", - match_mode="exact" if self.decision_kind == "resume_exact" else "subset", - stopped_service_count=len(stopped_service_names), - target_projects=target_project_names, - ) - self.runtime._emit( - "state.auto_resume.restore_stopped_services", - run_id=self.candidate_state.run_id, - mode=self.session.runtime_mode, - command=route.command, - projects=target_project_names, - services=stopped_service_names, - ) - self.runtime._emit( - "state.run_reuse.applied", - run_id=self.candidate_state.run_id, - mode=self.session.runtime_mode, - command=route.command, - decision_kind="restore_stopped_services", - reason="dashboard_stopped_services", - restored_projects=target_project_names, - restored_services=stopped_service_names, + diagnostics = ( + lambda: self.emit_phase( + self.session, + "auto_resume_evaluate", + self.reuse_started, + status="restore_stopped_services", + match_mode="exact" if self.decision_kind == "resume_exact" else "subset", + stopped_service_count=len(stopped_service_names), + target_projects=target_project_names, + ), + lambda: self.runtime._emit( + "state.auto_resume.restore_stopped_services", + run_id=self.candidate_state.run_id, + mode=self.session.runtime_mode, + command=route.command, + projects=target_project_names, + services=stopped_service_names, + ), + lambda: self.runtime._emit( + "state.run_reuse.applied", + run_id=self.candidate_state.run_id, + mode=self.session.runtime_mode, + command=route.command, + decision_kind="restore_stopped_services", + reason="dashboard_stopped_services", + restored_projects=target_project_names, + restored_services=stopped_service_names, + ), ) + for diagnostic in diagnostics: + try: + diagnostic() + except Exception: # noqa: BLE001 + pass def prepare_dashboard_stopped_service_restore( diff --git a/python/envctl_engine/startup/run_reuse_decision.py b/python/envctl_engine/startup/run_reuse_decision.py index fbb715981..2cf58de0b 100644 --- a/python/envctl_engine/startup/run_reuse_decision.py +++ b/python/envctl_engine/startup/run_reuse_decision.py @@ -5,6 +5,7 @@ from typing import Any, Mapping from envctl_engine.runtime.command_router import Route +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RunState from envctl_engine.startup.run_reuse_identity import ( @@ -366,7 +367,11 @@ def evaluate_run_reuse( def state_has_resumable_services(runtime: Any, state: RunState) -> bool: for service_name, service in state.services.items(): - project_name = runtime._project_name_from_service(service_name) + project_name = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) if not project_name: continue service_type = str(getattr(service, "type", "")).strip().lower() diff --git a/python/envctl_engine/startup/run_reuse_fresh_start.py b/python/envctl_engine/startup/run_reuse_fresh_start.py index 0f91949cf..fae3b4e76 100644 --- a/python/envctl_engine/startup/run_reuse_fresh_start.py +++ b/python/envctl_engine/startup/run_reuse_fresh_start.py @@ -5,6 +5,7 @@ from typing import Any from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.lifecycle_service_termination import failed_listener_pids, unconfirmed_service_names from envctl_engine.shared.services import service_project_name, service_slug_from_record from envctl_engine.startup.session import metadata_with_state_sources from envctl_engine.startup.startup_selection_support import _restart_service_types_for_project @@ -75,6 +76,29 @@ class FreshStartServiceReplacer: def replace(self) -> None: route = self.session.effective_route + previous_preserve_flag = bool(getattr(self.session, "preserve_existing_state_on_failure", False)) + source_services = { + **dict(self.candidate_state.services), + **dict(self.session.preserved_services), + } + source_requirements = { + **dict(self.candidate_state.requirements), + **dict(self.session.preserved_requirements), + } + source_metadata = metadata_with_state_sources( + { + **dict(self.candidate_state.metadata), + **dict(self.session.base_metadata), + }, + self.candidate_state, + ) + # Bind failure finalization to the existing run before resolving any + # legacy selector or emitting diagnostics. Those adapters are allowed + # to fail, but their failure must never supersede active authority. + self.session.preserved_services = source_services + self.session.preserved_requirements = source_requirements + self.session.base_metadata = source_metadata + self.session.preserve_existing_state_on_failure = True dependencies_only = route.flags.get("runtime_scope") == "dependencies" selected_services = ( set() @@ -112,58 +136,60 @@ def replace(self) -> None: else: preserved_requirements = dict(self.candidate_state.requirements) requirements_to_release = {} - if selected_services: - self.announce_session_identifiers(self.session) - progress_message = ( - f"Auto-resume disabled; replacing {len(selected_services)} existing service(s)..." - if self.reason == "explicit_no_resume" - else f"Startup selection changed; replacing {len(selected_services)} existing service(s)..." - ) - self.report_progress(route, progress_message) - self.runtime._emit( - "state.run_reuse.replace_existing_services", - run_id=self.candidate_state.run_id, - mode=self.session.runtime_mode, - reason=self.reason, - selected_services=sorted(selected_services), - ) - if selected_services: - termination_result = self.runtime._terminate_services_from_state( - self.candidate_state, - selected_services=selected_services, - aggressive=False, - verify_ownership=True, - ) - failed_services = _failed_service_names(termination_result) - if failed_services: - self.session.preserve_existing_state_on_failure = True - raise RuntimeError( - "Fresh start aborted because existing services could not be stopped: " - + ", ".join(sorted(failed_services)) + try: + if selected_services: + self.announce_session_identifiers(self.session) + progress_message = ( + f"Auto-resume disabled; replacing {len(selected_services)} existing service(s)..." + if self.reason == "explicit_no_resume" + else f"Startup selection changed; replacing {len(selected_services)} existing service(s)..." ) - failed_orphan_pids = self.terminate_restart_orphan_listeners( - state=self.candidate_state, - selected_services=selected_services, - aggressive=True, + self.report_progress(route, progress_message) + self.runtime._emit( + "state.run_reuse.replace_existing_services", + run_id=self.candidate_state.run_id, + mode=self.session.runtime_mode, + reason=self.reason, + selected_services=sorted(selected_services), ) - if failed_orphan_pids: - self.session.preserve_existing_state_on_failure = True - raise RuntimeError( - "Fresh start aborted because orphan listeners could not be stopped: " - + ", ".join(str(pid) for pid in sorted(failed_orphan_pids)) + if selected_services: + termination_result = self.runtime._terminate_services_from_state( + self.candidate_state, + selected_services=selected_services, + aggressive=False, + verify_ownership=True, + ) + failed_services = unconfirmed_service_names(termination_result, selected_services) + if failed_services: + raise RuntimeError( + "Fresh start aborted because existing services could not be stopped: " + + ", ".join(sorted(failed_services)) + ) + orphan_result = self.terminate_restart_orphan_listeners( + state=self.candidate_state, + selected_services=selected_services, + aggressive=True, ) - for requirements in requirements_to_release.values(): - self.runtime._release_requirement_ports(requirements) + failed_orphan_pids = failed_listener_pids(orphan_result) + if failed_orphan_pids is None: + raise RuntimeError("Fresh start aborted because orphan listener cleanup was not confirmed") + if failed_orphan_pids: + raise RuntimeError( + "Fresh start aborted because orphan listeners could not be stopped: " + + ", ".join(str(pid) for pid in sorted(failed_orphan_pids)) + ) + for requirements in requirements_to_release.values(): + self.runtime._release_requirement_ports(requirements) + self._rebind_replacement_ports(selected_services) + except Exception: + self.session.preserved_services = source_services + self.session.preserved_requirements = source_requirements + self.session.base_metadata = source_metadata + self.session.preserve_existing_state_on_failure = True + raise self.session.preserved_services = preserved_services self.session.preserved_requirements = preserved_requirements - self.session.base_metadata = metadata_with_state_sources( - { - **dict(self.candidate_state.metadata), - **dict(self.session.base_metadata), - }, - self.candidate_state, - ) - self._rebind_replacement_ports(selected_services) + self.session.preserve_existing_state_on_failure = previous_preserve_flag def _rebind_replacement_ports(self, selected_services: set[str]) -> None: port_planner = getattr(self.runtime, "port_planner", None) @@ -258,12 +284,6 @@ def _split_requirements( return preserved, replaced -def _failed_service_names(result: object) -> set[str]: - if not isinstance(result, (set, frozenset)): - return set() - return {str(name).strip() for name in result if str(name).strip()} - - def _route_has_app_service_scope(route: Route) -> bool: runtime_scope = str(route.flags.get("runtime_scope") or "").strip().lower() if runtime_scope in {"backend", "frontend", "fullstack", "dependencies"}: diff --git a/python/envctl_engine/startup/run_reuse_identity.py b/python/envctl_engine/startup/run_reuse_identity.py index 41d837485..8aa294248 100644 --- a/python/envctl_engine/startup/run_reuse_identity.py +++ b/python/envctl_engine/startup/run_reuse_identity.py @@ -9,6 +9,7 @@ from envctl_engine.requirements.core import dependency_ids from envctl_engine.requirements.external import dependency_external_mode from envctl_engine.runtime.command_router import Route +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.state.models import RunState @@ -101,13 +102,17 @@ def project_identities_from_state(runtime: Any, state: RunState) -> list[Project if not normalized_name: continue names[normalized_name] = normalize_project_root(root_value) - for project_name in state.requirements: - normalized_name = str(project_name).strip() + for storage_name, requirements in state.requirements.items(): + normalized_name = str(getattr(requirements, "project", "") or storage_name).strip() if not normalized_name: continue names.setdefault(normalized_name, normalize_project_root(roots.get(normalized_name))) - for service_name in state.services: - project_name = runtime._project_name_from_service(service_name) + for service_name, service in state.services.items(): + project_name = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) normalized_name = str(project_name).strip() if not normalized_name: continue diff --git a/python/envctl_engine/startup/run_reuse_resolution.py b/python/envctl_engine/startup/run_reuse_resolution.py index 7eb95778a..69b52a315 100644 --- a/python/envctl_engine/startup/run_reuse_resolution.py +++ b/python/envctl_engine/startup/run_reuse_resolution.py @@ -108,17 +108,19 @@ def _resolve_auto_reuse(self) -> int | None: def _emit_reuse_evaluation(self, decision: RunReuseDecision) -> None: candidate_state = decision.candidate_state - self.runtime._emit( - "state.run_reuse.evaluate", - run_id=candidate_state.run_id if candidate_state is not None else None, - mode=self.runtime_mode, - command=self.route.command, - decision_kind=decision.decision_kind, - reason=decision.reason, - selected_projects=decision.selected_projects, - state_projects=decision.state_projects, - weak_identity=decision.weak_identity, - mismatch_details=decision.mismatch_details, + self._best_effort_diagnostic( + lambda: self.runtime._emit( + "state.run_reuse.evaluate", + run_id=candidate_state.run_id if candidate_state is not None else None, + mode=self.runtime_mode, + command=self.route.command, + decision_kind=decision.decision_kind, + reason=decision.reason, + selected_projects=decision.selected_projects, + state_projects=decision.state_projects, + weak_identity=decision.weak_identity, + mismatch_details=decision.mismatch_details, + ) ) def _prepare_dashboard_stopped_restore(self, *, decision: RunReuseDecision, reuse_started: float) -> bool: @@ -152,53 +154,63 @@ def _apply_reuse_expand(self, *, decision: RunReuseDecision, candidate_state: An for context in self.session.selected_contexts if str(getattr(context, "name", "")).strip().casefold() not in state_project_names ] - self.emit_phase( - self.session, - "auto_resume_evaluate", - reuse_started, - status="expand", - match_mode="expand", - state_project_count=len(decision.state_projects), - selected_project_count=len(self.session.selected_contexts), + self._best_effort_diagnostic( + lambda: self.emit_phase( + self.session, + "auto_resume_evaluate", + reuse_started, + status="expand", + match_mode="expand", + state_project_count=len(decision.state_projects), + selected_project_count=len(self.session.selected_contexts), + ) ) - self.runtime._emit( - "state.run_reuse.applied", - run_id=candidate_state.run_id, - mode=self.runtime_mode, - command=self.route.command, - decision_kind=decision.decision_kind, - reason=decision.reason, - preserved_services=sorted(self.session.preserved_services), + self._best_effort_diagnostic( + lambda: self.runtime._emit( + "state.run_reuse.applied", + run_id=candidate_state.run_id, + mode=self.runtime_mode, + command=self.route.command, + decision_kind=decision.decision_kind, + reason=decision.reason, + preserved_services=sorted(self.session.preserved_services), + ) ) def _handle_no_reuse(self, *, decision: RunReuseDecision, reuse_started: float) -> None: candidate_state = decision.candidate_state - self.emit_phase( - self.session, - "auto_resume_evaluate", - reuse_started, - status="skipped" if candidate_state is not None else "none", - reason=decision.reason if candidate_state is not None else None, - state_project_count=len(decision.state_projects), - selected_project_count=len(self.session.selected_contexts), + self._best_effort_diagnostic( + lambda: self.emit_phase( + self.session, + "auto_resume_evaluate", + reuse_started, + status="skipped" if candidate_state is not None else "none", + reason=decision.reason if candidate_state is not None else None, + state_project_count=len(decision.state_projects), + selected_project_count=len(self.session.selected_contexts), + ) ) if candidate_state is None: return - self.runtime._emit( - "state.auto_resume.skipped", - reason=decision.reason, - state_projects=[project["name"] for project in decision.state_projects], - selected_projects=[project["name"] for project in decision.selected_projects], - mode=self.runtime_mode, - command=self.route.command, + self._best_effort_diagnostic( + lambda: self.runtime._emit( + "state.auto_resume.skipped", + reason=decision.reason, + state_projects=[project["name"] for project in decision.state_projects], + selected_projects=[project["name"] for project in decision.selected_projects], + mode=self.runtime_mode, + command=self.route.command, + ) ) - self.runtime._emit( - "state.run_reuse.skipped", - run_id=candidate_state.run_id, - reason=decision.reason, - mode=self.runtime_mode, - command=self.route.command, - mismatch_details=decision.mismatch_details, + self._best_effort_diagnostic( + lambda: self.runtime._emit( + "state.run_reuse.skipped", + run_id=candidate_state.run_id, + reason=decision.reason, + mode=self.runtime_mode, + command=self.route.command, + mismatch_details=decision.mismatch_details, + ) ) self.replace_existing_project_services_for_fresh_start( self.session, @@ -224,14 +236,23 @@ def _run_planning_prs_if_requested(self) -> int | None: return code def _emit_startup_branch_snapshot(self, debug_orch_groups: set[str]) -> None: - self.emit_snapshot( - self.session, - "startup_branch_enter", - command=self.requested_command, - mode=self.runtime_mode, - orch_group=sorted(debug_orch_groups) or None, + self._best_effort_diagnostic( + lambda: self.emit_snapshot( + self.session, + "startup_branch_enter", + command=self.requested_command, + mode=self.runtime_mode, + orch_group=sorted(debug_orch_groups) or None, + ) ) + @staticmethod + def _best_effort_diagnostic(callback: Callable[[], object]) -> None: + try: + callback() + except Exception: # noqa: BLE001 + pass + def _resume_matching_run( self, *, diff --git a/python/envctl_engine/startup/run_reuse_resume.py b/python/envctl_engine/startup/run_reuse_resume.py index 3790ba759..237bc755f 100644 --- a/python/envctl_engine/startup/run_reuse_resume.py +++ b/python/envctl_engine/startup/run_reuse_resume.py @@ -15,7 +15,7 @@ from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.startup.finalization import build_planning_dashboard_state from envctl_engine.startup.run_reuse_decision import RunReuseDecision -from envctl_engine.startup.session import StartupSession +from envctl_engine.startup.session import StartupSession, metadata_with_state_sources @dataclass(frozen=True, slots=True) @@ -40,25 +40,47 @@ class StartupRunReuseResumeHandler: def resume_matching_run(self) -> int | None: previous_run_id = self.session.run_id previous_identifiers_announced = self.session.identifiers_announced - self.session.run_id = self.candidate_state.run_id - self.announce_session_identifiers(self.session) - self._emit_resume_match_phase() - self._emit_resume_match_events() - - lifecycle_lease_was_active = lifecycle_operation_active(self.runtime) - resume_code = self.runtime._resume(self._resume_route()) - if int(resume_code) == 0: - return self._resume_success_result() - lifecycle_lease_was_released = lifecycle_lease_was_active and not lifecycle_operation_active(self.runtime) - self._handle_resume_failure( - resume_code=int(resume_code), - previous_run_id=previous_run_id, - previous_identifiers_announced=previous_identifiers_announced, - allow_fresh_fallback=not lifecycle_lease_was_released, - ) - return int(resume_code) if lifecycle_lease_was_released else None + previous_authority = self._session_authority() + try: + self.session.run_id = self.candidate_state.run_id + self.announce_session_identifiers(self.session) + self._emit_resume_match_phase() + self._emit_resume_match_events() + lifecycle_lease_was_active = lifecycle_operation_active(self.runtime) + resume_code = self.runtime._resume(self._resume_route()) + if int(resume_code) == 0: + result = self._resume_success_result() + self._restore_session_authority(previous_authority) + return result + lifecycle_lease_was_released = lifecycle_lease_was_active and not lifecycle_operation_active(self.runtime) + self._handle_resume_failure( + resume_code=int(resume_code), + previous_run_id=previous_run_id, + previous_identifiers_announced=previous_identifiers_announced, + allow_fresh_fallback=not lifecycle_lease_was_released, + ) + return int(resume_code) if lifecycle_lease_was_released else None + except Exception: + self.session.run_id = previous_run_id + self.session.identifiers_announced = previous_identifiers_announced + self._bind_candidate_failure_authority(previous_authority) + raise def resume_dashboard_run(self) -> int | None: + previous_run_id = self.session.run_id + previous_identifiers_announced = self.session.identifiers_announced + previous_authority = self._session_authority() + try: + result = self._resume_dashboard_run_guarded() + except Exception: + self.session.run_id = previous_run_id + self.session.identifiers_announced = previous_identifiers_announced + self._bind_candidate_failure_authority(previous_authority) + raise + self._restore_session_authority(previous_authority) + return result + + def _resume_dashboard_run_guarded(self) -> int | None: self.session.run_id = self.candidate_state.run_id self.announce_session_identifiers(self.session) self._persist_dashboard_resume_metadata() @@ -82,6 +104,48 @@ def resume_dashboard_run(self) -> int | None: return self.runtime._run_interactive_dashboard_loop(self.candidate_state) return 0 + def _session_authority( + self, + ) -> tuple[dict[str, object], dict[str, object], dict[str, object], bool]: + return ( + dict(self.session.preserved_services), + dict(self.session.preserved_requirements), + dict(self.session.base_metadata), + bool(self.session.preserve_existing_state_on_failure), + ) + + def _restore_session_authority( + self, + authority: tuple[dict[str, object], dict[str, object], dict[str, object], bool], + ) -> None: + services, requirements, metadata, preserve = authority + self.session.preserved_services = dict(services) # type: ignore[assignment] + self.session.preserved_requirements = dict(requirements) # type: ignore[assignment] + self.session.base_metadata = dict(metadata) + self.session.preserve_existing_state_on_failure = preserve + + def _bind_candidate_failure_authority( + self, + prior: tuple[dict[str, object], dict[str, object], dict[str, object], bool], + ) -> None: + prior_services, prior_requirements, prior_metadata, _prior_preserve = prior + self.session.preserved_services = { # type: ignore[assignment] + **dict(self.candidate_state.services), + **prior_services, + } + self.session.preserved_requirements = { # type: ignore[assignment] + **dict(self.candidate_state.requirements), + **prior_requirements, + } + self.session.base_metadata = metadata_with_state_sources( + { + **dict(self.candidate_state.metadata), + **prior_metadata, + }, + self.candidate_state, + ) + self.session.preserve_existing_state_on_failure = True + @property def _match_mode(self) -> str: return "exact" if self.decision.decision_kind == "resume_exact" else "subset" diff --git a/python/envctl_engine/startup/selected_context_startup.py b/python/envctl_engine/startup/selected_context_startup.py index b96fcf004..f30a0c411 100644 --- a/python/envctl_engine/startup/selected_context_startup.py +++ b/python/envctl_engine/startup/selected_context_startup.py @@ -2,6 +2,7 @@ import concurrent.futures from contextlib import nullcontext +from dataclasses import replace from typing import Any, Callable, cast from envctl_engine.runtime.command_router import Route @@ -12,6 +13,7 @@ StartupSession, track_startup_failure, ) +from envctl_engine.state.models import RequirementsResult, ServiceRecord from envctl_engine.startup.startup_progress import suppress_progress_output as default_suppress_progress_output @@ -27,14 +29,205 @@ def record_project_startup( context: ProjectContextLike, result: ProjectStartupResult, ) -> None: + existing_service_names = set(session.preserved_services) + existing_service_names.update(session.unterminated_services) + for project_services in session.services_by_project.values(): + existing_service_names.update(project_services) + collisions = sorted(existing_service_names.intersection(result.services)) + project_collision = ( + context.name in session.services_by_project + or context.name in session.requirements_by_project + ) + if collisions or project_collision: + # The replacement processes are already live at this point. Give every + # one a unique durable identity before raising so parallel projects and + # preserved state can never overwrite one another during cleanup. + preserved_collisions = set(session.preserved_services).intersection(collisions) + session.preserve_existing_state_on_failure = bool(preserved_collisions) + session.service_state_collisions.update(collisions) + occupied_names = existing_service_names.union(result.services) + for name, service in result.services.items(): + stored_name = name + stored_service = ( + service + if not isinstance(service, ServiceRecord) or service.project + else replace(service, project=context.name) + ) + if name in existing_service_names: + stored_name = _unique_collision_service_name(name, service, occupied_names) + occupied_names.add(stored_name) + stored_service = replace(stored_service, name=stored_name) + session.service_state_collision_rows.append( + { + "original_name": name, + "replacement_name": stored_name, + "replacement_pid": getattr(service, "pid", None), + "replacement_project": getattr(service, "project", None) or context.name, + } + ) + session.unterminated_services[stored_name] = stored_service + _record_collision_requirements(session, context.name, result.requirements) + collision_labels = collisions or [f"project:{context.name}"] + raise RuntimeError( + "Refusing to overwrite preserved service state or tracked startup state " + "with newly started services: " + + ", ".join(collision_labels) + ) session.requirements_by_project[context.name] = result.requirements - session.services_by_project[context.name] = result.services + session.services_by_project[context.name] = { + name: ( + service + if not isinstance(service, ServiceRecord) or service.project + else replace(service, project=context.name) + ) + for name, service in result.services.items() + } session.started_context_names.append(context.name) for warning in result.warnings: if "No local app system is configured" in warning: session.warnings.append(warning) +def _unique_collision_service_name(name: str, service: object, occupied: set[str]) -> str: + pid = getattr(service, "pid", None) + suffix = f"Restart Collision {pid}" if isinstance(pid, int) and pid > 0 else "Restart Collision" + base = f"{name} {suffix}" + candidate = base + index = 2 + while candidate in occupied: + candidate = f"{base} {index}" + index += 1 + return candidate + + +def _record_collision_requirements( + session: StartupSession, + project: str, + requirements: RequirementsResult, +) -> None: + # A preserved requirement with the same storage key is remapped by the + # failure finalizer, which can also record whether it remained managed. + if project in session.preserved_requirements: + session.requirements_by_project[project] = requirements + return + if project not in session.requirements_by_project: + session.requirements_by_project[project] = requirements + return + occupied = set(session.preserved_requirements).union(session.requirements_by_project) + base = f"{project} Restart Collision" + storage_key = base + index = 2 + while storage_key in occupied: + storage_key = f"{base} {index}" + index += 1 + session.requirements_by_project[storage_key] = requirements + session.requirement_state_collision_rows.append( + { + "original_project": project, + "replacement_project": storage_key, + "replacement_requirements_retained": True, + } + ) + + +class _UnrecordedProjectStartup(RuntimeError): + """Adapter used to feed a returned startup result into failure tracking.""" + + def __init__( + self, + *, + project: str, + requirements: RequirementsResult | None, + services: dict[str, ServiceRecord], + ) -> None: + super().__init__(f"startup result for {project} was not committed to the session") + self.project = project + self.requirements = requirements + self.unterminated_services = services + + +def _retain_unrecorded_project_startup( + session: StartupSession, + context: ProjectContextLike, + result: ProjectStartupResult, +) -> None: + """Retain every resource returned before a callback/cancellation failure. + + ``record_project_startup`` is normally the ownership handoff. A signal can + arrive after project startup returns but before (or during) that handoff. + Feed only resources that are not already represented in the session through + the same collision-aware failure tracker used by lower startup layers. + """ + + tracked_requirements = [ + *session.preserved_requirements.values(), + *session.requirements_by_project.values(), + ] + requirements = ( + None + if any(existing is result.requirements for existing in tracked_requirements) + else result.requirements + ) + tracked_services = [ + *session.preserved_services.values(), + *session.unterminated_services.values(), + *( + service + for project_services in session.services_by_project.values() + for service in project_services.values() + ), + ] + untracked_services = { + name: service + for name, service in result.services.items() + if isinstance(service, ServiceRecord) + and not any(_same_runtime_service(service, existing) for existing in tracked_services) + } + if requirements is None and not untracked_services: + return + track_startup_failure( + session, + _UnrecordedProjectStartup( + project=str(context.name), + requirements=requirements, + services=untracked_services, + ), + ) + + +def _same_runtime_service(left: ServiceRecord, right: ServiceRecord) -> bool: + if left is right: + return True + for attribute in ("container_launch_token", "container_id"): + left_value = str(getattr(left, attribute, "") or "").strip() + right_value = str(getattr(right, attribute, "") or "").strip() + if left_value and left_value == right_value: + return True + left_pid = getattr(left, "pid", None) + right_pid = getattr(right, "pid", None) + return ( + isinstance(left_pid, int) + and not isinstance(left_pid, bool) + and left_pid > 0 + and left_pid == right_pid + ) + + +def _retain_unrecorded_project_startup_safely( + session: StartupSession, + context: ProjectContextLike, + result: ProjectStartupResult, + triggering_error: BaseException, +) -> None: + try: + _retain_unrecorded_project_startup(session, context, result) + except BaseException as retention_error: # noqa: BLE001 - never replace the triggering cancellation + try: + triggering_error.add_note(f"startup result retention also failed: {retention_error}") + except Exception: # noqa: BLE001 + pass + + def start_selected_contexts( *, runtime: StartupRuntime, @@ -142,10 +335,10 @@ def start_selected_contexts( project_spinner_group=project_spinner_group, use_project_spinner_group=use_project_spinner_group, ) - except Exception: + except BaseException: try: _fail_single_spinner(runtime=runtime, active_spinner=active_spinner, enabled=use_single_spinner) - except Exception: # noqa: BLE001 + except BaseException: # noqa: BLE001 - presentation must not mask cancellation pass raise _succeed_single_spinner( @@ -236,86 +429,141 @@ def _start_contexts_parallel( completed: dict[str, ProjectStartupResult] = {} failures: list[str] = [] ui_failures: list[Exception] = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=parallel_workers) as executor: - future_map = { - executor.submit( - runtime._start_project_context, - context=context, - mode=session.runtime_mode, - route=route_for_execution, - run_id=resolved_run_id(session), - ): context - for context in session.contexts_to_start - } - for future in concurrent.futures.as_completed(future_map): - context = future_map[future] - try: - result = cast(ProjectStartupResult, future.result()) - except RuntimeError as exc: - track_startup_failure(session, exc) + future_map: dict[concurrent.futures.Future[Any], ProjectContextLike] = {} + try: + # Let the executor context finish its wait before handling a main-thread + # cancellation. Once control reaches the outer handler, every submitted + # worker has a terminal result that can be transferred to the session. + with concurrent.futures.ThreadPoolExecutor(max_workers=parallel_workers) as executor: + for context in session.contexts_to_start: + future = executor.submit( + runtime._start_project_context, + context=context, + mode=session.runtime_mode, + route=route_for_execution, + run_id=resolved_run_id(session), + ) + future_map[future] = context + for future in concurrent.futures.as_completed(future_map): + context = future_map[future] try: - degrade = should_degrade_to_plan_agent_handoff(session, str(exc)) - except Exception as callback_exc: # noqa: BLE001 - ui_failures.append(callback_exc) - degrade = False - if degrade: + result = cast(ProjectStartupResult, future.result()) + except RuntimeError as exc: + track_startup_failure(session, exc) try: - record_plan_agent_handoff_local_startup_failure( - session, - project_name=context.name, - error=str(exc), - ) - if use_project_spinner_group: - project_spinner_group.mark_success( - context.name, - "AI session running; local startup failed", + degrade = should_degrade_to_plan_agent_handoff(session, str(exc)) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + degrade = False + if degrade: + try: + record_plan_agent_handoff_local_startup_failure( + session, + project_name=context.name, + error=str(exc), ) + if use_project_spinner_group: + project_spinner_group.mark_success( + context.name, + "AI session running; local startup failed", + ) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + continue + failures.append(str(exc)) + try: + runtime._emit("startup.project.failed", project=context.name, error=str(exc)) + if use_project_spinner_group: + project_spinner_group.mark_failure(context.name, str(exc)) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + continue + except Exception as exc: # noqa: BLE001 + track_startup_failure(session, exc) + failures.append(str(exc)) + try: + runtime._emit("startup.project.failed", project=context.name, error=str(exc)) + if use_project_spinner_group: + project_spinner_group.mark_failure(context.name, str(exc)) except Exception as callback_exc: # noqa: BLE001 ui_failures.append(callback_exc) continue - failures.append(str(exc)) + try: - runtime._emit("startup.project.failed", project=context.name, error=str(exc)) - if use_project_spinner_group: - project_spinner_group.mark_failure(context.name, str(exc)) - except Exception as callback_exc: # noqa: BLE001 - ui_failures.append(callback_exc) - continue - except Exception as exc: # noqa: BLE001 - track_startup_failure(session, exc) - failures.append(str(exc)) + record_project_startup(session, context, result) + except Exception as exc: # noqa: BLE001 + _retain_unrecorded_project_startup_safely(session, context, result, exc) + track_startup_failure(session, exc) + failures.append(str(exc)) + try: + runtime._emit("startup.project.failed", project=context.name, error=str(exc)) + if use_project_spinner_group: + project_spinner_group.mark_failure(context.name, str(exc)) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + # Keep draining the executor. Other futures may already have + # launched resources that must be recorded for final cleanup. + continue + completed[context.name] = result try: - runtime._emit("startup.project.failed", project=context.name, error=str(exc)) - if use_project_spinner_group: - project_spinner_group.mark_failure(context.name, str(exc)) - except Exception as callback_exc: # noqa: BLE001 - ui_failures.append(callback_exc) - continue - - completed[context.name] = result - record_project_startup(session, context, result) - try: - _record_parallel_success( - runtime=runtime, - session=session, - context=context, - result=result, - route_for_execution=route_for_execution, - render_project_startup_warnings=render_project_startup_warnings, - active_spinner=active_spinner, - use_single_spinner=use_single_spinner, - project_spinner_group=project_spinner_group, - use_project_spinner_group=use_project_spinner_group, - completed_count=len(completed), - ) - except Exception as exc: # noqa: BLE001 - ui_failures.append(exc) + _record_parallel_success( + runtime=runtime, + session=session, + context=context, + result=result, + route_for_execution=route_for_execution, + render_project_startup_warnings=render_project_startup_warnings, + active_spinner=active_spinner, + use_single_spinner=use_single_spinner, + project_spinner_group=project_spinner_group, + use_project_spinner_group=use_project_spinner_group, + completed_count=len(completed), + ) + except Exception as exc: # noqa: BLE001 + ui_failures.append(exc) + except BaseException as exc: # noqa: BLE001 - cancellation requires a complete ownership drain + _drain_parallel_startup_futures( + session=session, + future_map=future_map, + triggering_error=exc, + ) + raise if failures: raise RuntimeError("; ".join(failures)) if ui_failures: raise ui_failures[0] +def _drain_parallel_startup_futures( + *, + session: StartupSession, + future_map: dict[concurrent.futures.Future[Any], ProjectContextLike], + triggering_error: BaseException, +) -> None: + """Transfer every completed worker result after a main-thread interruption.""" + + for future, context in future_map.items(): + try: + result = cast(ProjectStartupResult, future.result()) + except BaseException as worker_error: # noqa: BLE001 - retain any failure-owned resources + try: + track_startup_failure(session, worker_error) + except BaseException as retention_error: # noqa: BLE001 + try: + triggering_error.add_note( + f"startup failure retention also failed for {context.name}: {retention_error}" + ) + except Exception: # noqa: BLE001 + pass + continue + _retain_unrecorded_project_startup_safely( + session, + context, + result, + triggering_error, + ) + + def _record_parallel_success( *, runtime: StartupRuntime, @@ -383,7 +631,15 @@ def _start_contexts_sequential( project_spinner_group.mark_success(context.name, "AI session running; local startup failed") continue raise - record_project_startup(session, context, result) + except BaseException as exc: # noqa: BLE001 - persist failure-owned resources before propagation + track_startup_failure(session, exc) + raise + try: + record_project_startup(session, context, result) + except BaseException as exc: # noqa: BLE001 - result may already own live resources + _retain_unrecorded_project_startup_safely(session, context, result, exc) + track_startup_failure(session, exc) + raise render_project_startup_warnings( context=context, warnings=result.warnings, diff --git a/python/envctl_engine/startup/service_attach_execution.py b/python/envctl_engine/startup/service_attach_execution.py index 5e24e1a2f..bd3d3606e 100644 --- a/python/envctl_engine/startup/service_attach_execution.py +++ b/python/envctl_engine/startup/service_attach_execution.py @@ -8,6 +8,7 @@ from envctl_engine.runtime.docker_service_runtime import ( DockerServiceLaunch, + DockerServiceLaunchCleanupError, DockerServiceRuntime, docker_service_container_command_source, docker_service_mode_enabled, @@ -15,6 +16,8 @@ from envctl_engine.runtime.service_manager import ServiceCleanupError, ServiceStartDescriptor from envctl_engine.runtime.service_truth_diagnostics import service_listener_failure_class from envctl_engine.shared.reason_codes import ServiceFailureReason +from envctl_engine.shared.process_runner import ProcessHandoffCleanupError +from envctl_engine.shared.services import service_display_name from envctl_engine.startup.service_execution_policy import ordered_service_layers from envctl_engine.startup.service_execution_records import PreparedServiceLaunch from envctl_engine.startup.session import unconfirmed_service_names @@ -95,15 +98,26 @@ def release_failed_port_then_report( self._annotate_container_records(records) self._record_port_lock_session(records.values()) return records - except Exception as exc: - container_cleanup_error: Exception | None = None + except BaseException as exc: + container_cleanup_error: BaseException | None = None try: self._cleanup_container_launches() - except Exception as cleanup_exc: # noqa: BLE001 + except BaseException as cleanup_exc: # noqa: BLE001 - cancellation must not skip cleanup container_cleanup_error = cleanup_exc - self._record_port_lock_session(dict(getattr(exc, "unterminated_services", {}) or {}).values()) - self._release_failed_start_reservations(exc) + unterminated = dict(getattr(exc, "unterminated_services", {}) or {}) + if isinstance(container_cleanup_error, ServiceCleanupError): + unterminated.update(container_cleanup_error.unterminated_services) + tracked_error: BaseException = ( + ServiceCleanupError(str(exc), unterminated) if unterminated else exc + ) + self._record_port_lock_session(unterminated.values()) + self._release_failed_start_reservations(tracked_error) if container_cleanup_error is not None: + if unterminated: + raise ServiceCleanupError( + f"{exc}; Docker cleanup error: {container_cleanup_error}", + unterminated, + ) from exc raise RuntimeError(f"{exc}; Docker cleanup error: {container_cleanup_error}") from exc raise @@ -142,7 +156,7 @@ def _release_failed_start_reservations(self, error: BaseException) -> None: preserved_service_types = { str(getattr(service, "type", "") or "").strip().lower() for service in dict(getattr(error, "unterminated_services", {}) or {}).values() - if _service_has_recorded_pid(service) + if _service_has_recorded_runtime_identity(service) } for service_type, port in tuple(self._active_service_ports.items()): if service_type in preserved_service_types: @@ -174,7 +188,7 @@ def _start_with_generic_attach( parallel_start=attach_parallel and len(layer) > 1, ) records.update(layer_records) - except Exception as exc: + except BaseException as exc: unterminated = dict(getattr(exc, "unterminated_services", {}) or {}) terminator = getattr(self.runtime, "_terminate_started_services", None) if records: @@ -184,7 +198,7 @@ def _start_with_generic_attach( terminator(records), records, ) - except Exception as cleanup_exc: # noqa: BLE001 + except BaseException as cleanup_exc: # noqa: BLE001 - cancellation must not skip cleanup failed_names = set(records) exc = RuntimeError(f"{exc}; cleanup error: {cleanup_exc}") else: @@ -334,7 +348,24 @@ def _launch_prepared_process( container_port=launch_port, listener_expected=launch.listener_expected, log_path=launch.log_path, + on_confirmed=lambda confirmed: self._container_launches.__setitem__( + service_name, + confirmed, + ), + ) + except DockerServiceLaunchCleanupError as exc: + self._container_launches[service_name] = exc.launch + self._emit_attach_phase(service_name, "container_launch", launch_started) + detail = str(exc) + self.runtime._emit( + "service.failure", + project=self.project_name, + service=service_name, + failure_class="container_cleanup_unconfirmed", + requested_port=event_port, + detail=detail, ) + return False, detail, None except (OSError, RuntimeError) as exc: self._emit_attach_phase(service_name, "container_launch", launch_started) detail = str(exc) @@ -358,6 +389,17 @@ def _launch_prepared_process( env=service_env, log_path=launch.log_path, ) + except ProcessHandoffCleanupError as exc: + detail = str(exc) + self.runtime._emit( + "service.failure", + project=self.project_name, + service=service_name, + failure_class="process_cleanup_unconfirmed", + requested_port=event_port, + detail=detail, + ) + return False, detail, exc.pid except OSError as exc: executable = command[0] if command else "" detail = f"process spawn failed for {executable}: {exc}" @@ -423,8 +465,16 @@ def _remove_failed_container_launch( launch: DockerServiceLaunch, ) -> str | None: try: - removed = DockerServiceRuntime(self.runtime, self.process_runtime).stop(launch.container_id) - except Exception as exc: # noqa: BLE001 - preserve the launch in state for a later stop retry + stop_kwargs = ( + {"expected_launch_token": launch.launch_token} if launch.launch_token else {} + ) + if launch.cleanup_pending_since is not None: + stop_kwargs["pending_cleanup_since"] = launch.cleanup_pending_since + removed = DockerServiceRuntime(self.runtime, self.process_runtime).stop( + launch.container_id, + **stop_kwargs, + ) + except BaseException as exc: # noqa: BLE001 - preserve the launch in state for a later stop retry return f"failed to remove container: {exc}" if not removed: return "failed to remove container" @@ -445,12 +495,65 @@ def _annotate_container_records(self, records: dict[str, ServiceRecord]) -> None record.container_id = launch.container_id record.container_name = launch.container_name record.container_image = launch.image + record.container_launch_token = launch.launch_token or None + record.container_cleanup_pending_since = launch.cleanup_pending_since def _cleanup_container_launches(self) -> None: docker_runtime = DockerServiceRuntime(self.runtime, self.process_runtime) - for launch in tuple(self._container_launches.values()): - docker_runtime.stop(launch.container_id) - self._container_launches.clear() + unconfirmed: dict[str, ServiceRecord] = {} + details: list[str] = [] + for service_name, launch in tuple(self._container_launches.items()): + try: + stop_kwargs = ( + {"expected_launch_token": launch.launch_token} if launch.launch_token else {} + ) + if launch.cleanup_pending_since is not None: + stop_kwargs["pending_cleanup_since"] = launch.cleanup_pending_since + stopped = docker_runtime.stop(launch.container_id, **stop_kwargs) + except BaseException as exc: # noqa: BLE001 - cancellation must not abandon authority + stopped = False + details.append(f"{service_name}: {exc}") + if stopped: + self._container_launches.pop(service_name, None) + continue + details.append(f"{service_name}: removal unconfirmed") + record = self._container_service_record(service_name, launch) + unconfirmed[record.name] = record + if unconfirmed: + raise ServiceCleanupError( + "Docker cleanup could not confirm container removal: " + "; ".join(dict.fromkeys(details)), + unconfirmed, + ) + + def _container_service_record( + self, + service_name: str, + launch: DockerServiceLaunch, + ) -> ServiceRecord: + prepared = self.prepared_launches.get(service_name) + listener_expected = bool(getattr(prepared, "listener_expected", True)) + port = self._active_service_ports.get(service_name) + record = ServiceRecord( + name=f"{self.project_name} {service_display_name(service_name)}", + type=service_name, + cwd=str(getattr(prepared, "cwd", self.project_root)), + requested_port=port if listener_expected else None, + actual_port=port if listener_expected else None, + log_path=launch.log_path, + status="termination_failed", + listener_expected=listener_expected, + project=self.project_name, + service_slug=service_name, + failure_detail="Docker container removal was not confirmed", + degraded=True, + runtime_kind="docker", + container_id=launch.container_id, + container_name=launch.container_name, + container_image=launch.image, + container_launch_token=launch.launch_token or None, + container_cleanup_pending_since=launch.cleanup_pending_since, + ) + return record def start_backend(self, port: int) -> tuple[bool, str | None, int | None]: conflict_result = self._conflicting_start_result("backend", port) @@ -491,7 +594,7 @@ def start_frontend(self, port: int) -> tuple[bool, str | None, int | None]: event_port=port, env_extra=self.frontend_env_extra, ) - except Exception: + except BaseException: self._release_rebound_launch_port("frontend") raise if not isinstance(result[2], int) or isinstance(result[2], bool) or result[2] <= 0: @@ -743,7 +846,10 @@ def _emit_attach_phase(self, service_name: str, phase: str, started: float) -> N ) -def _service_has_recorded_pid(service: object) -> bool: +def _service_has_recorded_runtime_identity(service: object) -> bool: + if str(getattr(service, "runtime_kind", "process") or "process").strip().lower() == "docker": + if str(getattr(service, "container_id", "") or getattr(service, "container_name", "")).strip(): + return True listener_pids = getattr(service, "listener_pids", None) listeners = listener_pids if isinstance(listener_pids, (list, tuple, set, frozenset)) else () candidates = [getattr(service, "pid", None), *listeners] diff --git a/python/envctl_engine/startup/session.py b/python/envctl_engine/startup/session.py index 84d0367cf..d50043143 100644 --- a/python/envctl_engine/startup/session.py +++ b/python/envctl_engine/startup/session.py @@ -2,7 +2,7 @@ from collections.abc import Collection, Mapping import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Any, cast from envctl_engine.runtime.command_router import Route @@ -61,7 +61,11 @@ class StartupSession: disabled_startup_mode: bool = False used_project_spinner_group: bool = False strict_truth_failed: bool = False + authority_committed: bool = False preserve_existing_state_on_failure: bool = False + service_state_collisions: set[str] = field(default_factory=set) + service_state_collision_rows: list[dict[str, object]] = field(default_factory=list) + requirement_state_collision_rows: list[dict[str, object]] = field(default_factory=list) failure_message: str | None = None errors: list[str] = field(default_factory=list) warnings: list[str] = field(default_factory=list) @@ -82,6 +86,17 @@ class StartupSession: @property def merged_services(self) -> dict[str, ServiceRecord]: + replacement_names = set(self.unterminated_services) + for project_services in self.services_by_project.values(): + replacement_names.update(project_services) + collisions = sorted(set(self.preserved_services).intersection(replacement_names)) + if collisions: + self.service_state_collisions.update(collisions) + self.preserve_existing_state_on_failure = True + raise RuntimeError( + "Refusing to overwrite preserved service state with newly started services: " + + ", ".join(collisions) + ) services = dict(self.preserved_services) for project_services in self.services_by_project.values(): services.update(project_services) @@ -91,7 +106,41 @@ def merged_services(self) -> dict[str, ServiceRecord]: @property def merged_requirements(self) -> dict[str, RequirementsResult]: requirements = dict(self.preserved_requirements) - requirements.update(self.requirements_by_project) + collisions: list[str] = [] + for project, replacement in self.requirements_by_project.items(): + # App-only restart reuses the exact persisted RequirementsResult. + # Its durable storage key can be a collision alias, while project + # startup naturally reports the canonical project name. Retain the + # existing storage identity instead of serializing the same stack + # twice under two keys. + if any(existing is replacement for existing in requirements.values()): + continue + preserved_key = _preserved_requirement_key(requirements, project) + preserved = requirements.get(preserved_key) if preserved_key is not None else None + if preserved is not None and preserved == replacement: + # Preserve the existing storage key, including a collision + # alias, when deserialization produced an equal object. + continue + if ( + preserved is not None + and self.effective_route.flags.get("launch_dependencies") is False + and not _requirements_have_enabled_components(replacement) + ): + # App-only and no-infrastructure starts still return a + # RequirementsResult containing disabled port projections. + # Those projections do not own resources and must not collide + # with, duplicate, or replace the preserved dependency stack. + continue + if preserved is not None: + collisions.append(project) + continue + requirements[project] = replacement + if collisions: + self.preserve_existing_state_on_failure = True + raise RuntimeError( + "Refusing to overwrite preserved requirement state with newly started requirements: " + + ", ".join(sorted(collisions)) + ) return requirements @property @@ -135,7 +184,53 @@ def unconfirmed_service_names( (str, bytes, Mapping), ): return set(services) - return {name for raw_name in termination_result if (name := str(raw_name).strip()) and name in services} + raw_reported = list(termination_result) + if any(not str(raw_name).strip() for raw_name in raw_reported): + return set(services) + reported = {str(raw_name).strip() for raw_name in raw_reported} + if not reported.issubset(services): + # An adapter that reports an unknown identity has not positively + # confirmed the disposition of any tracked process. Fail closed so a + # stale/case-mismatched name cannot make every real service disappear + # from the durable recovery state. + return set(services) + return reported + + +def _preserved_requirement_key( + requirements: Mapping[str, RequirementsResult], + project: str, +) -> str | None: + target = str(project).strip().casefold() + exact = [key for key in requirements if str(key).strip().casefold() == target] + if len(exact) == 1: + return exact[0] + if len(exact) > 1: + raise RuntimeError( + f"Ambiguous preserved requirements authority for {project}: " + + ", ".join(sorted(exact)) + ) + aliases = [ + key + for key, value in requirements.items() + if str(getattr(value, "project", "") or "").strip().casefold() == target + ] + if len(aliases) == 1: + return aliases[0] + if len(aliases) > 1: + raise RuntimeError( + f"Ambiguous preserved requirements authority for {project}: " + + ", ".join(sorted(aliases)) + ) + return None + + +def _requirements_have_enabled_components(requirements: RequirementsResult) -> bool: + return any( + bool(component.get("enabled", False)) + for component in requirements.components.values() + if isinstance(component, Mapping) + ) def metadata_with_state_sources(metadata: dict[str, Any], state: RunState) -> dict[str, Any]: @@ -158,13 +253,68 @@ def track_startup_failure(session: StartupSession, failure: BaseException) -> No project = str(getattr(failure, "project", "") or "").strip() requirements = getattr(failure, "requirements", None) if project and isinstance(requirements, RequirementsResult): - session.requirements_by_project[project] = requirements + requirement_key = project + if project in session.requirements_by_project: + occupied_projects = set(session.preserved_requirements).union(session.requirements_by_project) + base = f"{project} Restart Collision" + requirement_key = base + index = 2 + while requirement_key in occupied_projects: + requirement_key = f"{base} {index}" + index += 1 + session.requirement_state_collision_rows.append( + { + "original_project": project, + "replacement_project": requirement_key, + "replacement_requirements_retained": True, + } + ) + session.requirements_by_project[requirement_key] = requirements services = getattr(failure, "unterminated_services", None) if not isinstance(services, dict): return + occupied_names = set(session.preserved_services).union(session.unterminated_services) + for project_services in session.services_by_project.values(): + occupied_names.update(project_services) + tracked_identities = { + id(service) + for service in session.unterminated_services.values() + } + tracked_identities.update( + id(service) + for project_services in session.services_by_project.values() + for service in project_services.values() + ) for name, service in services.items(): if isinstance(name, str) and isinstance(service, ServiceRecord): - session.unterminated_services[name] = service + if id(service) in tracked_identities: + continue + stored_name = name + stored_service = service if service.project or not project else replace(service, project=project) + if stored_name in occupied_names: + pid = getattr(service, "pid", None) + suffix = f"Restart Collision {pid}" if isinstance(pid, int) and pid > 0 else "Restart Collision" + base = f"{name} {suffix}" + stored_name = base + index = 2 + while stored_name in occupied_names or stored_name in services: + stored_name = f"{base} {index}" + index += 1 + stored_service = replace(stored_service, name=stored_name) + session.service_state_collisions.add(name) + session.service_state_collision_rows.append( + { + "original_name": name, + "replacement_name": stored_name, + "replacement_pid": service.pid, + "replacement_project": service.project or project or None, + } + ) + if name in session.preserved_services: + session.preserve_existing_state_on_failure = True + occupied_names.add(stored_name) + tracked_identities.add(id(service)) + session.unterminated_services[stored_name] = stored_service def track_unterminated_services(session: StartupSession, failure: BaseException) -> None: diff --git a/python/envctl_engine/startup/startup_execution_support.py b/python/envctl_engine/startup/startup_execution_support.py index 51989c568..7641298b9 100644 --- a/python/envctl_engine/startup/startup_execution_support.py +++ b/python/envctl_engine/startup/startup_execution_support.py @@ -18,6 +18,7 @@ ) from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_env import effective_dependency_scope +from envctl_engine.shared.services import service_slug_from_record from envctl_engine.startup.protocols import ProjectContextLike, StartupOrchestratorLike from envctl_engine.startup.requirements_execution import ( REQUIREMENTS_PROGRESS_PROJECT_FLAG, @@ -31,6 +32,7 @@ from envctl_engine.startup.service_execution import start_project_services as start_project_services_impl from envctl_engine.startup.session import ProjectStartupResult, unconfirmed_service_names from envctl_engine.state.models import PortPlan, RequirementsResult, ServiceRecord +from envctl_engine.state.project_runtime import requirements_for_project @dataclass(slots=True) @@ -99,13 +101,18 @@ def start_project_context( route=route, ) rt._assert_project_services_post_start_truth(context=context, services=project_services) + services_by_type = { + service_slug_from_record(service): service + for service in project_services.values() + if service_slug_from_record(service) + } report_progress_fn( route, f"Services ready for {context.name}: " + " ".join( - f"{service_type}={_service_ready_label(project_services.get(f'{context.name} {service_type.title()}'))}" + f"{service_type}={_service_ready_label(services_by_type.get(service_type))}" for service_type in ("backend", "frontend") - if f"{context.name} {service_type.title()}" in project_services + if service_type in services_by_type ), project=context.name, ) @@ -322,7 +329,7 @@ def _load_or_start_shared_main_requirements( rt = orchestrator.runtime existing = rt._try_load_existing_state(mode="main", strict_mode_match=True) if existing is not None: - requirements = existing.requirements.get("Main") + requirements = requirements_for_project(existing, "Main") if isinstance(requirements, RequirementsResult) and rt._requirements_ready(requirements): reconciled = _annotate_shared_main_requirements(orchestrator, requirements) if _shared_requirements_runtime_ready(reconciled): diff --git a/python/envctl_engine/startup/startup_selection_support.py b/python/envctl_engine/startup/startup_selection_support.py index 1f9ba6f06..e83872f50 100644 --- a/python/envctl_engine/startup/startup_selection_support.py +++ b/python/envctl_engine/startup/startup_selection_support.py @@ -1,13 +1,21 @@ from __future__ import annotations from collections import OrderedDict +from collections.abc import Mapping from pathlib import Path +from envctl_engine.dashboard_metadata import ( + DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY, + DASHBOARD_STOPPED_SERVICES_KEY, + dashboard_project_configured_services_from_metadata, + normalize_dashboard_service_types, +) from envctl_engine.planning import list_planning_files, planning_existing_counts, select_projects_for_plan_files from envctl_engine.runtime.command_router import MODE_TREE_TOKENS, Route from envctl_engine.runtime.runtime_context import resolve_port_allocator, resolve_process_runtime from envctl_engine.shared.protocols import PortAllocator, ProcessRuntime from envctl_engine.shared.services import ( + resolve_service_project_name, service_display_name, service_matches_selector, service_project_name, @@ -34,17 +42,39 @@ def project_app_ports_text(context: ProjectContextLike) -> str: def state_project_names(*, runtime: StartupRuntime, state: RunState) -> set[str]: - names = {str(name).strip() for name in state.requirements.keys() if str(name).strip()} - for service_name in state.services: - project_name = runtime._project_name_from_service(service_name) - if isinstance(project_name, str) and project_name.strip(): - names.add(project_name.strip()) - metadata_roots = state.metadata.get("project_roots") + requirements = getattr(state, "requirements", {}) + names = { + str(getattr(requirement, "project", "") or storage_name).strip() + for storage_name, requirement in requirements.items() + if str(getattr(requirement, "project", "") or storage_name).strip() + } + for service_name, service in getattr(state, "services", {}).items(): + normalized = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) + if normalized: + names.add(normalized) + metadata = getattr(state, "metadata", {}) + metadata_roots = metadata.get("project_roots") if isinstance(metadata, Mapping) else None if isinstance(metadata_roots, dict): for project_name in metadata_roots: normalized = str(project_name).strip() if normalized: names.add(normalized) + if isinstance(metadata, Mapping): + names.update(dashboard_project_configured_services_from_metadata(metadata)) + raw_stopped = metadata.get(DASHBOARD_STOPPED_SERVICES_KEY) + if isinstance(raw_stopped, list): + for item in raw_stopped: + if not isinstance(item, Mapping): + continue + project_name = str(item.get("project", "") or "").strip() + if project_name: + names.add(project_name) + if not names and str(getattr(state, "mode", "")).strip().lower() == "main": + names.add("Main") return names @@ -232,6 +262,9 @@ def restart_include_requirements(route: Route) -> bool: return False if runtime_scope in {"dependencies", "entire-system"}: return True + launch_dependencies = route.flags.get("launch_dependencies") + if launch_dependencies is not None: + return bool(launch_dependencies) if bool(route.flags.get("all")): return True services = route.flags.get("services") @@ -245,6 +278,18 @@ def restart_include_requirements(route: Route) -> bool: def _restart_selected_services(*, state: RunState, route: Route) -> set[str]: service_filters = route.flags.get("services") selected: set[str] = set() + project_keys = { + str(project).strip().casefold() + for project in route.projects + if str(project).strip() + } + + def in_selected_projects(name: str, service: object) -> bool: + if not project_keys: + return True + project = service_project_name(service) or _project_name_from_service_name(name) + return bool(project and project.casefold() in project_keys) + if isinstance(service_filters, list): filters = [str(value).strip() for value in service_filters if str(value).strip()] for selector in filters: @@ -252,7 +297,8 @@ def _restart_selected_services(*, state: RunState, route: Route) -> set[str]: { name for name, service in state.services.items() - if name == selector or service_matches_selector(service, selector) + if (name == selector or service_matches_selector(service, selector)) + and in_selected_projects(name, service) } ) return selected @@ -280,53 +326,313 @@ def _restart_selected_services(*, state: RunState, route: Route) -> set[str]: if selected: return selected + runtime_scope = route.flags.get("runtime_scope") + if runtime_scope in {"backend", "frontend", "fullstack", "dependencies"}: + selected_types = { + "backend", + "frontend", + } if runtime_scope == "fullstack" else ({str(runtime_scope)} if runtime_scope != "dependencies" else set()) + return { + name + for name, service in state.services.items() + if service_slug_from_record(service) in selected_types and in_selected_projects(name, service) + } + if route.projects: project_set = { project.strip().lower() for project in route.projects if isinstance(project, str) and project.strip() } - for name in state.services: - project = _project_name_from_service_name(name) + for name, service in state.services.items(): + project = service_project_name(service) or _project_name_from_service_name(name) if project and project.lower() in project_set: selected.add(name) - if selected: - return selected + return selected return set(state.services.keys()) def restart_target_projects(*, state: RunState, route: Route, runtime: StartupRuntime) -> set[str]: targets: set[str] = set() + known_projects = {name.casefold(): name for name in state_project_names(runtime=runtime, state=state)} for value in route.projects: if isinstance(value, str) and value.strip(): - targets.add(value.strip()) + normalized = value.strip() + targets.add(known_projects.get(normalized.casefold(), normalized)) if targets: return targets services = route.flags.get("services") if isinstance(services, list): - for service_name in services: - if not isinstance(service_name, str): + for selector in services: + if not isinstance(selector, str): continue - matched = False - for name, service in state.services.items(): - if name == service_name or service_matches_selector(service, service_name): - matched = True - project = service_project_name(service) or _project_name_from_service_name(name) - if project: - targets.add(project) - if matched: + observed = set( + _observed_service_types_for_selector( + state=state, + runtime=runtime, + selector=selector, + ) + ) + if observed: + targets.update(observed) continue - project = _project_name_from_service_name(service_name) - if project: - targets.add(project) - if targets: + configured_projects = _configured_service_projects_for_selector( + state=state, + runtime=runtime, + selector=selector, + ) + if len(configured_projects) == 1: + targets.update(configured_projects) return targets - for service_name in state.services: - project = runtime._project_name_from_service(service_name) - if project: - targets.add(project) - return targets + return state_project_names(runtime=runtime, state=state) + + +def _stopped_service_projects_for_selector(state: RunState, selector: str) -> set[str]: + target = str(selector).strip().lower().removeprefix("service:").strip() + if not target: + return set() + metadata = getattr(state, "metadata", {}) + raw_stopped = metadata.get(DASHBOARD_STOPPED_SERVICES_KEY) if isinstance(metadata, Mapping) else None + if not isinstance(raw_stopped, list): + return set() + projects: set[str] = set() + for item in raw_stopped: + if not isinstance(item, Mapping): + continue + project = str(item.get("project", "") or "").strip() + name = str(item.get("name", "") or "").strip() + service_type = str(item.get("type", "") or "").strip().lower() + candidates = {name.lower(), service_type, service_display_name(service_type).lower()} - {""} + if project and target in candidates: + projects.add(project) + return projects + + +def _stopped_service_entries(state: RunState) -> list[tuple[str, str, str]]: + metadata = getattr(state, "metadata", {}) + raw_stopped = metadata.get(DASHBOARD_STOPPED_SERVICES_KEY) if isinstance(metadata, Mapping) else None + if not isinstance(raw_stopped, list): + return [] + entries: list[tuple[str, str, str]] = [] + for item in raw_stopped: + if not isinstance(item, Mapping): + continue + project = str(item.get("project", "") or "").strip() + normalized_types = normalize_dashboard_service_types([item.get("type", "")]) + service_type = normalized_types[0] if normalized_types else "" + name = str(item.get("name", "") or "").strip() + if project and service_type: + entries.append((project, service_type, name)) + return entries + + +def _additional_service_enabled_for_project( + service: object, + *, + state: RunState, + project_name: str, +) -> bool: + mode = str(getattr(state, "mode", "") or "main").strip().lower() + metadata = getattr(state, "metadata", {}) + roots = metadata.get("project_roots") if isinstance(metadata, Mapping) else None + project_root = None + if isinstance(roots, Mapping): + for raw_project, raw_root in roots.items(): + if str(raw_project).strip().casefold() == project_name.casefold(): + project_root = Path(str(raw_root)) + break + enabled_for_project = getattr(service, "enabled_for_project_root", None) + if callable(enabled_for_project) and project_root is not None: + return bool(enabled_for_project(mode, project_root)) + enabled_for_mode = getattr(service, "enabled_for_mode", None) + if callable(enabled_for_mode): + return bool(enabled_for_mode(mode)) + return True + + +def configured_service_types_for_project( + *, + state: RunState, + runtime: StartupRuntime, + project_name: str, +) -> set[str]: + metadata = getattr(state, "metadata", {}) + configured_by_project = ( + dashboard_project_configured_services_from_metadata(metadata) + if isinstance(metadata, Mapping) + else {} + ) + if isinstance(metadata, Mapping) and DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY in metadata: + snapshot_types: set[str] = set() + for project, service_types in configured_by_project.items(): + if project.casefold() == project_name.casefold(): + snapshot_types = set(service_types) + break + service_enabled = getattr(runtime, "_service_enabled_for_mode", None) + mode = str(getattr(state, "mode", "") or "main").strip().lower() + if callable(service_enabled): + snapshot_types = { + service_type + for service_type in snapshot_types + if service_type not in {"backend", "frontend"} + or bool(service_enabled(mode, service_type)) + } + config = getattr(runtime, "config", None) + if config is not None and hasattr(config, "additional_services"): + enabled_additional = { + str(getattr(service, "name", "") or "").strip().lower() + for service in tuple(getattr(config, "additional_services", ()) or ()) + if str(getattr(service, "name", "") or "").strip() + and _additional_service_enabled_for_project( + service, + state=state, + project_name=project_name, + ) + } + snapshot_types = { + service_type + for service_type in snapshot_types + if service_type in {"backend", "frontend"} or service_type in enabled_additional + } + return snapshot_types + + configured: set[str] = set() + service_enabled = getattr(runtime, "_service_enabled_for_mode", None) + mode = str(getattr(state, "mode", "") or "main").strip().lower() + for service_type in ("backend", "frontend"): + if not callable(service_enabled) or bool(service_enabled(mode, service_type)): + configured.add(service_type) + for service in tuple(getattr(getattr(runtime, "config", None), "additional_services", ()) or ()): + service_type = str(getattr(service, "name", "") or "").strip().lower() + if service_type and _additional_service_enabled_for_project( + service, + state=state, + project_name=project_name, + ): + configured.add(service_type) + for service_name, service in getattr(state, "services", {}).items(): + project = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) + service_type = _service_type_from_record_name(service_name, service) + if project and project.casefold() == project_name.casefold() and service_type: + configured.add(service_type) + for project, service_type, _service_name in _stopped_service_entries(state): + if project.casefold() == project_name.casefold(): + configured.add(service_type) + return configured + + +def _selector_key(selector: str) -> str: + return str(selector).strip().casefold().removeprefix("service:").strip() + + +def _selector_matches_service_type( + selector: str, + *, + project_name: str, + service_type: str, + service_name: str = "", +) -> bool: + target = _selector_key(selector) + if not target: + return False + display = service_display_name(service_type) + candidates = { + str(service_name).strip().casefold(), + service_type.casefold(), + display.casefold(), + f"{project_name} {display}".strip().casefold(), + } + candidates.discard("") + return target in candidates + + +def _observed_service_types_for_selector( + *, + state: RunState, + runtime: StartupRuntime, + selector: str, +) -> dict[str, set[str]]: + matches: dict[str, set[str]] = {} + for service_name, service in getattr(state, "services", {}).items(): + project = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) + service_type = _service_type_from_record_name(service_name, service) + if not project or not service_type: + continue + configured = configured_service_types_for_project( + state=state, + runtime=runtime, + project_name=project, + ) + if service_type not in configured: + continue + if service_matches_selector(service, selector) or _selector_matches_service_type( + selector, + project_name=project, + service_type=service_type, + service_name=service_name, + ): + matches.setdefault(project, set()).add(service_type) + for project, service_type, service_name in _stopped_service_entries(state): + configured = configured_service_types_for_project( + state=state, + runtime=runtime, + project_name=project, + ) + if service_type in configured and _selector_matches_service_type( + selector, + project_name=project, + service_type=service_type, + service_name=service_name, + ): + matches.setdefault(project, set()).add(service_type) + return matches + + +def _configured_service_projects_for_selector( + *, + state: RunState, + runtime: StartupRuntime, + selector: str, +) -> dict[str, set[str]]: + matches: dict[str, set[str]] = {} + for project in sorted(state_project_names(runtime=runtime, state=state), key=str.casefold): + for service_type in configured_service_types_for_project( + state=state, + runtime=runtime, + project_name=project, + ): + if _selector_matches_service_type( + selector, + project_name=project, + service_type=service_type, + ): + matches.setdefault(project, set()).add(service_type) + return matches + + +def _configured_service_target_projects( + *, + state: RunState, + runtime: StartupRuntime, + selector: str, +) -> set[str]: + matching_projects = set( + _configured_service_projects_for_selector( + state=state, + runtime=runtime, + selector=selector, + ) + ) + return matching_projects if len(matching_projects) == 1 else set() def restart_target_projects_for_selected_services( @@ -334,16 +640,17 @@ def restart_target_projects_for_selected_services( ) -> set[str]: targets: set[str] = set() for service_name in selected_services: - project = runtime._project_name_from_service(service_name) + service = state.services.get(service_name) + project = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) if project: targets.add(project) if targets: return targets - for service_name in state.services: - project = runtime._project_name_from_service(service_name) - if project: - targets.add(project) - return targets + return state_project_names(runtime=runtime, state=state) def _project_name_from_service_name(name: str) -> str | None: @@ -358,6 +665,18 @@ def _project_name_from_service_name(name: str) -> str | None: return None +def _service_type_from_record_name(name: str, service: object) -> str: + service_type = service_slug_from_record(service) + if service_type: + return service_type + lowered = str(name).strip().lower() + if lowered.endswith(" backend"): + return "backend" + if lowered.endswith(" frontend"): + return "frontend" + return "" + + def _service_filter_types_for_project( *, route: Route, project_name: str, configured_service_types: set[str] ) -> set[str]: @@ -418,6 +737,105 @@ def _expand_service_dependencies( return expanded +def restart_service_types_by_project( + *, + state: RunState, + route: Route, + runtime: StartupRuntime, + target_projects: set[str], +) -> dict[str, set[str]]: + additional_services = tuple( + getattr(getattr(runtime, "config", None), "additional_services", ()) or () + ) + active_types: dict[str, set[str]] = {} + for service_name, service in getattr(state, "services", {}).items(): + project = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) + service_type = _service_type_from_record_name(service_name, service) + if project and service_type: + active_types.setdefault(project.casefold(), set()).add(service_type) + + selectors = route.flags.get("services") + explicit_types = route.flags.get("restart_service_types") + selected_by_project: dict[str, set[str]] = {} + for project in sorted(target_projects, key=str.casefold): + configured = configured_service_types_for_project( + state=state, + runtime=runtime, + project_name=project, + ) + selected: set[str] + if isinstance(selectors, list): + selected = set() + for selector in selectors: + if not isinstance(selector, str) or not selector.strip(): + continue + observed = _observed_service_types_for_selector( + state=state, + runtime=runtime, + selector=selector, + ) + configured_matches = _configured_service_projects_for_selector( + state=state, + runtime=runtime, + selector=selector, + ) + for candidate_project, service_types in (*observed.items(), *configured_matches.items()): + if candidate_project.casefold() == project.casefold(): + selected.update(service_types) + selected = _expand_service_dependencies( + selected, + route=route, + project_name=project, + additional_services=additional_services, + ) + elif isinstance(explicit_types, list): + selected = { + str(service_type).strip().lower() + for service_type in explicit_types + if str(service_type).strip() + } + else: + selected = set(active_types.get(project.casefold(), set())) + runtime_scope = str(route.flags.get("runtime_scope") or "").strip().lower() + if runtime_scope in {"backend", "frontend"}: + selected.intersection_update({runtime_scope}) + elif runtime_scope == "fullstack": + selected.intersection_update({"backend", "frontend"}) + elif runtime_scope == "dependencies": + selected.clear() + selected.intersection_update(configured) + selected_by_project[project] = _apply_startup_service_launch_flags(selected, route=route) + return selected_by_project + + +def restart_selected_services_for_type_map( + *, + state: RunState, + runtime: StartupRuntime, + service_types_by_project: Mapping[str, set[str]], +) -> set[str]: + selected: set[str] = set() + type_map = { + str(project).strip().casefold(): {str(value).strip().lower() for value in service_types} + for project, service_types in service_types_by_project.items() + if str(project).strip() + } + for service_name, service in getattr(state, "services", {}).items(): + project = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) + service_type = _service_type_from_record_name(service_name, service) + if project and service_type in type_map.get(project.casefold(), set()): + selected.add(service_name) + return selected + + def _restart_service_types_for_project( *, route: Route | None, @@ -429,6 +847,24 @@ def _restart_service_types_for_project( if route is None: return configured_service_types + type_map = route.flags.get("_restart_service_types_by_project") + if isinstance(type_map, Mapping): + selected: set[str] = set() + for raw_project, raw_types in type_map.items(): + if str(raw_project).strip().casefold() != project_name.casefold(): + continue + if isinstance(raw_types, (list, tuple, set, frozenset)): + selected.update( + str(service_type).strip().lower() + for service_type in raw_types + if str(service_type).strip() + ) + break + return _apply_startup_service_launch_flags( + selected.intersection(configured_service_types), + route=route, + ) + services_value = route.flags.get("services") if isinstance(services_value, list) and services_value: service_types = _service_filter_types_for_project( @@ -471,6 +907,15 @@ def _restart_service_types_for_project( normalized.intersection(configured_service_types or normalized), route=route, ) + selected_services = route.flags.get("_restart_selected_services") + if isinstance(selected_services, list): + selected_names = {str(value).strip().casefold() for value in selected_services if str(value).strip()} + selected_types = { + service_type + for service_type in configured_service_types + if f"{project_name} {service_display_name(service_type)}".casefold() in selected_names + } + return _apply_startup_service_launch_flags(selected_types, route=route) return _apply_startup_service_launch_flags(configured_service_types, route=route) diff --git a/python/envctl_engine/state/__init__.py b/python/envctl_engine/state/__init__.py index 23e9f890c..6120f032e 100644 --- a/python/envctl_engine/state/__init__.py +++ b/python/envctl_engine/state/__init__.py @@ -3,6 +3,7 @@ import json import re import shlex +from dataclasses import replace from pathlib import Path from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord @@ -52,6 +53,10 @@ def load_state(path: str, *, allowed_root: str | None = None) -> RunState: container_id=str(svc.get("container_id") or "") or None, container_name=str(svc.get("container_name") or "") or None, container_image=str(svc.get("container_image") or "") or None, + container_launch_token=str(svc.get("container_launch_token") or "") or None, + container_cleanup_pending_since=parse_float_or_none( + svc.get("container_cleanup_pending_since") + ), ) for name, svc in data.get("services", {}).items() } @@ -67,7 +72,7 @@ def load_state(path: str, *, allowed_root: str | None = None) -> RunState: "n8n": payload.get("n8n", {}), } requirements[name] = RequirementsResult( - project=name, + project=str(payload.get("project") or name), components=components, health=str(payload.get("health", "unknown") or "unknown"), failures=list(payload.get("failures", [])), @@ -140,8 +145,36 @@ def merge_states(states: list[RunState]) -> RunState: merged_pointers: dict[str, str] = {} merged_metadata: dict[str, object] = {} for state in states: - merged_services.update(state.services) - merged_requirements.update(state.requirements) + for storage_name, service in state.services.items(): + existing = merged_services.get(storage_name) + existing_project = str(getattr(existing, "project", "") or "").strip().casefold() + incoming_project = str(getattr(service, "project", "") or "").strip().casefold() + if existing is not None and existing_project and incoming_project and existing_project != incoming_project: + base = f"{storage_name} State Collision {service.project}" + collision_name = base + index = 2 + occupied = set(merged_services).union(state.services) + while collision_name in occupied: + collision_name = f"{base} {index}" + index += 1 + merged_services[collision_name] = replace(service, name=collision_name) + else: + merged_services[storage_name] = service + for storage_key, requirements in state.requirements.items(): + existing = merged_requirements.get(storage_key) + existing_project = str(getattr(existing, "project", "") or "").strip().casefold() + incoming_project = str(requirements.project or "").strip().casefold() + if existing is not None and existing_project and incoming_project and existing_project != incoming_project: + base = f"{storage_key} State Collision {requirements.project}" + collision_key = base + index = 2 + occupied = set(merged_requirements).union(state.requirements) + while collision_key in occupied: + collision_key = f"{base} {index}" + index += 1 + merged_requirements[collision_key] = requirements + else: + merged_requirements[storage_key] = requirements merged_pointers.update(state.pointers) merged_metadata.update(state.metadata) latest = states[-1] @@ -196,11 +229,14 @@ def state_to_dict(state: RunState) -> dict[str, object]: "container_id": svc.container_id, "container_name": svc.container_name, "container_image": svc.container_image, + "container_launch_token": svc.container_launch_token, + "container_cleanup_pending_since": svc.container_cleanup_pending_since, } for name, svc in state.services.items() }, "requirements": { project: { + "project": req.project, "components": {dependency_id: req.component(dependency_id) for dependency_id in dependency_ids()}, "db": req.db, "redis": req.redis, diff --git a/python/envctl_engine/state/action_health_support.py b/python/envctl_engine/state/action_health_support.py index fae8e9f05..d5048cc1a 100644 --- a/python/envctl_engine/state/action_health_support.py +++ b/python/envctl_engine/state/action_health_support.py @@ -155,7 +155,8 @@ def requirement_health_rows(self, state: RunState) -> list[dict[str, object]]: rows: list[dict[str, object]] = [] definitions = tuple(self._dependency_definitions()) component_order = {definition.id: index for index, definition in enumerate(definitions)} - for project, requirements in state.requirements.items(): + for storage_key, requirements in state.requirements.items(): + project = str(getattr(requirements, "project", "") or storage_key).strip() for definition in definitions: component = definition.id data = requirements.component(component) @@ -172,6 +173,7 @@ def requirement_health_rows(self, state: RunState) -> list[dict[str, object]]: rows.append( { "project": project, + "storage_key": storage_key, "component": component, "status": status, "port": dependency_display_port(component, data), @@ -209,6 +211,12 @@ def _service_row(service: ServiceRecord) -> dict[str, object]: "container_id": getattr(service, "container_id", None), "container_name": getattr(service, "container_name", None), "container_image": getattr(service, "container_image", None), + "container_launch_token": getattr(service, "container_launch_token", None), + "container_cleanup_pending_since": getattr( + service, + "container_cleanup_pending_since", + None, + ), } rows = self._parallel_service_map( diff --git a/python/envctl_engine/state/action_orchestrator.py b/python/envctl_engine/state/action_orchestrator.py index 1c65e2753..972c6ab6c 100644 --- a/python/envctl_engine/state/action_orchestrator.py +++ b/python/envctl_engine/state/action_orchestrator.py @@ -20,7 +20,7 @@ services_from_selection, ) from envctl_engine.ui.selection_types import TargetSelection -from envctl_engine.shared.services import service_matches_selector +from envctl_engine.shared.services import resolve_service_project_name, service_matches_selector class StateActionRuntimeFacade: @@ -230,8 +230,12 @@ def _resolve_selected_services(self, route: Route, state: RunState) -> set[str] service_selected.add(name) if project_filters: project_selected = set() - for name in state.services: - project = self.runtime.project_name_from_service(name).lower() + for name, service in state.services.items(): + project = resolve_service_project_name( + name, + service, + project_name_from_service=self.runtime.project_name_from_service, + ).lower() if project and project in project_filters: project_selected.add(name) @@ -255,8 +259,20 @@ def _filtered_state(self, state: RunState, selected_services: set[str]) -> RunSt if not selected_services: return RunState(run_id=state.run_id, mode=state.mode) services = {name: svc for name, svc in state.services.items() if name in selected_services} - project_names = {self.runtime.project_name_from_service(name) for name in services} - requirements = {project: req for project, req in state.requirements.items() if project in project_names} + project_names = { + resolve_service_project_name( + name, + service, + project_name_from_service=self.runtime.project_name_from_service, + ) + for name, service in services.items() + } + project_keys = {str(project).strip().casefold() for project in project_names if str(project).strip()} + requirements = { + storage_key: req + for storage_key, req in state.requirements.items() + if str(getattr(req, "project", "") or storage_key).strip().casefold() in project_keys + } return RunState( run_id=state.run_id, mode=state.mode, diff --git a/python/envctl_engine/state/models.py b/python/envctl_engine/state/models.py index 68f045a7e..c581dae51 100644 --- a/python/envctl_engine/state/models.py +++ b/python/envctl_engine/state/models.py @@ -42,6 +42,8 @@ class ServiceRecord: container_id: str | None = None container_name: str | None = None container_image: str | None = None + container_launch_token: str | None = None + container_cleanup_pending_since: float | None = None @dataclass(slots=True) diff --git a/python/envctl_engine/state/project_runtime.py b/python/envctl_engine/state/project_runtime.py index aa68f87df..1e89ec815 100644 --- a/python/envctl_engine/state/project_runtime.py +++ b/python/envctl_engine/state/project_runtime.py @@ -80,10 +80,54 @@ def payload(self) -> dict[str, object]: } +def requirement_keys_for_project(state: RunState, project: str) -> list[str]: + target = str(project).strip().casefold() + if not target: + return [] + exact = [key for key in state.requirements if str(key).strip().casefold() == target] + owner_aliases = [ + key + for key, requirements in state.requirements.items() + if key not in exact + and str(getattr(requirements, "project", "") or "").strip().casefold() == target + ] + return [*exact, *owner_aliases] + + +def requirement_key_for_project(state: RunState, project: str) -> str | None: + target = str(project).strip().casefold() + if not target: + return None + exact = [key for key in state.requirements if str(key).strip().casefold() == target] + if len(exact) == 1: + return exact[0] + if len(exact) > 1: + raise RuntimeError( + f"Ambiguous requirements authority for {project}: " + ", ".join(sorted(exact)) + ) + owner_aliases = [ + key + for key, requirements in state.requirements.items() + if str(getattr(requirements, "project", "") or "").strip().casefold() == target + ] + if len(owner_aliases) == 1: + return owner_aliases[0] + if len(owner_aliases) > 1: + raise RuntimeError( + f"Ambiguous requirements authority for {project}: " + ", ".join(sorted(owner_aliases)) + ) + return None + + +def requirements_for_project(state: RunState, project: str) -> RequirementsResult | None: + key = requirement_key_for_project(state, project) + return state.requirements.get(key) if key is not None else None + + def active_project_names(state: RunState, *, runtime: Any | None = None) -> list[str]: names: set[str] = set() - for name in getattr(state, "requirements", {}) or {}: - normalized = str(name).strip() + for storage_key, requirements in (getattr(state, "requirements", {}) or {}).items(): + normalized = str(getattr(requirements, "project", "") or storage_key).strip() if normalized: names.add(normalized) for service_name, service in (getattr(state, "services", {}) or {}).items(): @@ -309,7 +353,12 @@ def _filtered_requirements( requirements: Mapping[str, RequirementsResult], selected: set[str], ) -> dict[str, RequirementsResult]: - direct = {project: req for project, req in requirements.items() if project in selected} + selected_keys = {str(project).strip().casefold() for project in selected if str(project).strip()} + direct = { + storage_key: req + for storage_key, req in requirements.items() + if str(getattr(req, "project", "") or storage_key).strip().casefold() in selected_keys + } if direct: return direct # Shared tree dependency state often stores requirements under Main. When a @@ -425,8 +474,8 @@ def project_resolution_event_payload( def _runtime_truth_project_names(state: RunState, *, runtime: Any | None = None) -> list[str]: names: set[str] = set() - for name in getattr(state, "requirements", {}) or {}: - normalized = str(name).strip() + for storage_key, requirements in (getattr(state, "requirements", {}) or {}).items(): + normalized = str(getattr(requirements, "project", "") or storage_key).strip() if normalized: names.add(normalized) for service_name, service in (getattr(state, "services", {}) or {}).items(): diff --git a/python/envctl_engine/state/repository.py b/python/envctl_engine/state/repository.py index 408e863a1..0c0adbe14 100644 --- a/python/envctl_engine/state/repository.py +++ b/python/envctl_engine/state/repository.py @@ -162,6 +162,7 @@ def save_run( emit: Callable[..., None], runtime_map_builder: Callable[[RunState], dict[str, object]], write_runtime_readiness_report: Callable[[Path], None] | None = None, + on_commit: Callable[[], None] | None = None, ) -> StateRepositoryPaths: project_names = sorted( { @@ -199,6 +200,7 @@ def save_run( project_names=project_names, artifacts=artifacts, supersede_run_ids=self._source_run_ids(state), + on_commit=on_commit, ) run_dir = self.run_dir_path(state.run_id) if write_runtime_readiness_report is not None: @@ -272,14 +274,20 @@ def load_all(self, *, mode: str | None = None) -> list[RunState]: raise RuntimeError( "Active runtime state failed integrity validation for run(s): " + ", ".join(invalid_run_ids) ) - return [ - filter_state_to_owned_projects( - indexed_state.state, - frozenset(indexed_state.candidate.project_names), - service_project_name=self._service_project_name, + active_states: list[RunState] = [] + for indexed_state in indexed_states: + candidate_projects = normalized_project_names(indexed_state.candidate.project_names) + active_states.append( + filter_state_to_owned_projects( + indexed_state.state, + candidate_projects, + service_project_name=self._service_project_name, + fallback_project=( + next(iter(candidate_projects)) if len(candidate_projects) == 1 else None + ), + ) ) - for indexed_state in indexed_states - ] + return active_states def has_active_runs(self) -> bool: """Return registry activity without trusting or loading mutable run payloads.""" @@ -661,6 +669,7 @@ def _commit_revision_unlocked( project_names: Sequence[str], artifacts: Mapping[str, str], supersede_run_ids: Sequence[str] = (), + on_commit: Callable[[], None] | None = None, ) -> Path: superseded_run_ids = { require_path_component(run_id, label="superseded run_id") @@ -677,22 +686,37 @@ def _commit_revision_unlocked( revision_id = f"{datetime.now(tz=UTC).strftime('%Y%m%dT%H%M%S%fZ')}-{uuid.uuid4().hex[:12]}" revision_dir = self._contained_child(revisions_dir, revision_id, label="revision id") revision_state_path = revision_dir / "run_state.json" - committed = False + retain_revision = False try: durable_mkdir(revision_dir) for artifact_name, text in artifacts.items(): safe_name = require_path_component(artifact_name, label="artifact name") self._write_text(revision_dir / safe_name, text) - self.run_index.record( - state_path=revision_state_path, - run_id=state.run_id, - mode=state.mode, - project_names=project_names, - supersede_run_ids=supersede_run_ids, - ) - committed = True + try: + self.run_index.record( + state_path=revision_state_path, + run_id=state.run_id, + mode=state.mode, + project_names=project_names, + supersede_run_ids=supersede_run_ids, + ) + except BaseException: + indexed = self._revision_index_commit_state_unlocked( + state=state, + revision_state_path=revision_state_path, + ) + # An unreadable registry has an uncertain commit outcome. Keep + # the immutable revision so a visible primary/backup can never + # be left pointing at artifacts this writer deleted. + retain_revision = indexed is not False + if indexed and on_commit is not None: + on_commit() + raise + retain_revision = True + if on_commit is not None: + on_commit() finally: - if not committed: + if not retain_revision: shutil.rmtree(revision_dir, ignore_errors=True) try: @@ -713,6 +737,32 @@ def _commit_revision_unlocked( raise return revision_state_path + def _revision_index_commit_state_unlocked( + self, + *, + state: RunState, + revision_state_path: Path, + ) -> bool | None: + try: + expected_state_path = revision_state_path.resolve() + expected_state_fingerprint = file_fingerprint(expected_state_path) + runtime_map_path = expected_state_path.parent / "runtime_map.json" + expected_runtime_map_fingerprint = file_fingerprint(runtime_map_path) + candidates = self.run_index.candidates( + StateSelector(mode=None, project_names=()) + ) + except BaseException: # noqa: BLE001 - unreadable authority is uncertain, never negative + return None + expected_mode = str(state.mode or "").strip().casefold() + return any( + candidate.run_id == state.run_id + and candidate.mode == expected_mode + and candidate.state_path == expected_state_path + and candidate.state_fingerprint == expected_state_fingerprint + and candidate.runtime_map_fingerprint == expected_runtime_map_fingerprint + for candidate in candidates + ) + def _publish_revision_aliases( self, *, @@ -1761,8 +1811,8 @@ def _prepare_state_for_save( def _runtime_project_names_from_state(self, state: RunState) -> list[str]: names: set[str] = set() - for project in state.requirements: - normalized = str(project).strip() + for storage_name, requirements in state.requirements.items(): + normalized = str(getattr(requirements, "project", "") or storage_name).strip() if normalized: names.add(normalized) for service_name, service in state.services.items(): @@ -1804,8 +1854,8 @@ def _project_names_from_state(self, state: RunState) -> list[str]: names.add(normalized) if names: return sorted(names, key=str.casefold) - for project in state.requirements: - normalized = str(project).strip() + for storage_name, requirements in state.requirements.items(): + normalized = str(getattr(requirements, "project", "") or storage_name).strip() if normalized: names.add(normalized) for service_name, service in state.services.items(): diff --git a/python/envctl_engine/state/runtime_map.py b/python/envctl_engine/state/runtime_map.py index 218974297..41316ff3b 100644 --- a/python/envctl_engine/state/runtime_map.py +++ b/python/envctl_engine/state/runtime_map.py @@ -1,11 +1,16 @@ from __future__ import annotations import json +from collections.abc import Mapping from pathlib import Path -from typing import cast -from envctl_engine.shared.services import project_name_from_service_name, service_display_name -from envctl_engine.state.models import RunState +from envctl_engine.shared.services import ( + project_name_from_service_name, + service_display_name, + service_project_name, + service_slug_from_record, +) +from envctl_engine.state.models import RunState, ServiceRecord def build_runtime_map(state: RunState, *, host: str = "localhost") -> dict[str, object]: @@ -24,29 +29,22 @@ def write_runtime_map(path: str, state: RunState) -> None: def build_runtime_projection(state: RunState, *, host: str = "localhost") -> dict[str, dict[str, object]]: runtime_map = build_runtime_map_without_projection(state, host=host) projection: dict[str, dict[str, object]] = {} - projects = cast(dict[str, dict[str, object]], runtime_map["projects"]) - service_records = state.services - for project, ports in projects.items(): - backend_port = ports.get("backend_port") - frontend_port = ports.get("frontend_port") - backend_service = service_records.get(f"{project} Backend") - frontend_service = service_records.get(f"{project} Frontend") - backend_ready = _service_projection_ready(backend_service) - frontend_ready = _service_projection_ready(frontend_service) + projects_raw = runtime_map["projects"] + projects = projects_raw if isinstance(projects_raw, Mapping) else {} + for project, ports_raw in projects.items(): + ports = ports_raw if isinstance(ports_raw, Mapping) else {} + services_raw = ports.get("services") + services = services_raw if isinstance(services_raw, Mapping) else {} + backend_port, backend_url, backend_status = _projected_service_fields(services.get("backend")) + frontend_port, frontend_url, frontend_status = _projected_service_fields(services.get("frontend")) projection[project] = { "backend_port": backend_port, "frontend_port": frontend_port, - "backend_url": (f"http://{host}:{backend_port}" if backend_port is not None and backend_ready else None), - "frontend_url": ( - f"http://{host}:{frontend_port}" if frontend_port is not None and frontend_ready else None - ), - "backend_status": getattr(backend_service, "status", "unknown") - if backend_service is not None - else "unknown", - "frontend_status": getattr(frontend_service, "status", "unknown") - if frontend_service is not None - else "unknown", - "services": ports.get("services", {}), + "backend_url": backend_url, + "frontend_url": frontend_url, + "backend_status": backend_status, + "frontend_status": frontend_status, + "services": services, } return projection @@ -58,52 +56,63 @@ def build_runtime_map_without_projection(state: RunState, *, host: str = "localh service_to_url: dict[str, str] = {} service_to_public_url: dict[str, str] = {} service_to_health_url: dict[str, str] = {} - for service in state.services.values(): - project = str(getattr(service, "project", "") or "").strip() or _project_name_from_service_record(service) + + grouped = _services_by_project_and_slug(state) + reserved_service_keys_by_project: dict[str, set[str]] = {} + for project, service_slug in grouped: + reserved_service_keys_by_project.setdefault(project, set()).add(service_slug) + + for (project, service_slug), instances in sorted( + grouped.items(), + key=lambda item: (item[0][0].casefold(), item[0][1].casefold()), + ): project_entry = projects.setdefault(project, {"backend_port": None, "frontend_port": None, "services": {}}) - port = service.actual_port if service.actual_port is not None else service.requested_port - service_ready = _service_projection_ready(service) - url = f"http://{host}:{port}" if port is not None and service_ready else None - public_url = str(getattr(service, "public_url", "") or "").strip() or url - health_url = str(getattr(service, "health_url", "") or "").strip() or None - service_slug = str(getattr(service, "service_slug", "") or "").strip() or service.type services_entry = project_entry.setdefault("services", {}) - if isinstance(services_entry, dict): - services_entry[service_slug] = { - "name": service.name, - "type": service.type, - "service_slug": service_slug, - "port": port, - "url": url, - "public_url": public_url, - "health_url": health_url, - "status": service.status, - "cwd": service.cwd, - "listener_expected": service.listener_expected, - "requested_port": service.requested_port, - "actual_port": service.actual_port, - "log_path": service.log_path, - "failure_detail": getattr(service, "failure_detail", None), - "critical": getattr(service, "critical", True), - "degraded": getattr(service, "degraded", False), - "runtime_kind": getattr(service, "runtime_kind", "process"), - "container_id": getattr(service, "container_id", None), - "container_name": getattr(service, "container_name", None), - "container_image": getattr(service, "container_image", None), - } - if service.type == "backend": - project_entry["backend_port"] = port - elif service.type == "frontend": - project_entry["frontend_port"] = port - if port is not None: - port_to_service[port] = service.name - service_to_actual_port[service.name] = port + if not isinstance(services_entry, dict): + continue + + ordered_instances = sorted( + instances, + key=lambda item: _service_authority_sort_key(project=project, service_slug=service_slug, item=item), + ) + used_service_keys = set(services_entry) + reserved_service_keys = reserved_service_keys_by_project.get(project, set()) + for index, (state_name, service) in enumerate(ordered_instances): + projection_key = ( + service_slug + if index == 0 + else _duplicate_service_projection_key( + service_slug=service_slug, + state_name=state_name, + used_keys=used_service_keys, + reserved_keys=reserved_service_keys, + ) + ) + used_service_keys.add(projection_key) + payload = _runtime_service_payload( + service=service, + service_slug=service_slug, + host=host, + ) + services_entry[projection_key] = payload + if index == 0 and service_slug in {"backend", "frontend"}: + project_entry[f"{service_slug}_port"] = payload["port"] + + port = payload["port"] + url = payload["url"] + public_url = payload["public_url"] + health_url = payload["health_url"] + if not isinstance(port, int): + continue + service_name = str(payload["name"] or "").strip() or state_name + port_to_service.setdefault(port, service_name) + service_to_actual_port[service_name] = port if url is not None: - service_to_url[service.name] = url + service_to_url[service_name] = str(url) if public_url: - service_to_public_url[service.name] = public_url + service_to_public_url[service_name] = str(public_url) if health_url: - service_to_health_url[service.name] = health_url + service_to_health_url[service_name] = str(health_url) return { "schema_version": "1.0", "backend_mode": "python", @@ -118,6 +127,125 @@ def build_runtime_map_without_projection(state: RunState, *, host: str = "localh } +def _services_by_project_and_slug( + state: RunState, +) -> dict[tuple[str, str], list[tuple[str, ServiceRecord]]]: + grouped: dict[tuple[str, str], list[tuple[str, ServiceRecord]]] = {} + for state_name, service in state.services.items(): + project = str(getattr(service, "project", "") or "").strip() or _project_name_from_service_record(service) + service_slug = service_slug_from_record(service) or "service" + grouped.setdefault((project, service_slug), []).append((state_name, service)) + return grouped + + +def _service_authority_sort_key( + *, + project: str, + service_slug: str, + item: tuple[str, ServiceRecord], +) -> tuple[object, ...]: + state_name, service = item + status = str(getattr(service, "status", "unknown") or "unknown").strip().lower() + status_priority = { + "healthy": 0, + "running": 1, + "starting": 2, + "unknown": 3, + "stale": 4, + "degraded": 5, + "stopped": 6, + "termination_failed": 7, + }.get(status, 3) + explicit_metadata_count = sum( + bool(str(value or "").strip()) + for value in (getattr(service, "project", ""), getattr(service, "service_slug", "")) + ) + canonical_name = f"{project} {service_display_name(service_slug)}".casefold() + service_name = str(getattr(service, "name", "") or "").strip() + actual_port = getattr(service, "actual_port", None) + return ( + status == "termination_failed", + not _service_projection_ready(service), + status_priority, + -explicit_metadata_count, + bool(getattr(service, "degraded", False)), + service_name.casefold() != canonical_name, + not isinstance(actual_port, int) or actual_port <= 0, + service_name.casefold(), + service_name, + state_name.casefold(), + state_name, + int(getattr(service, "pid", 0) or 0), + ) + + +def _duplicate_service_projection_key( + *, + service_slug: str, + state_name: str, + used_keys: set[str], + reserved_keys: set[str], +) -> str: + base = f"{service_slug}@{state_name}" + candidate = base + index = 2 + while candidate in used_keys or candidate in reserved_keys: + candidate = f"{base}#{index}" + index += 1 + return candidate + + +def _runtime_service_payload( + *, + service: ServiceRecord, + service_slug: str, + host: str, +) -> dict[str, object]: + port = service.actual_port if service.actual_port is not None else service.requested_port + url = f"http://{host}:{port}" if port is not None and _service_projection_ready(service) else None + public_url = str(getattr(service, "public_url", "") or "").strip() or url + health_url = str(getattr(service, "health_url", "") or "").strip() or None + return { + "name": service.name, + "type": service.type, + "service_slug": service_slug, + "port": port, + "url": url, + "public_url": public_url, + "health_url": health_url, + "status": service.status, + "cwd": service.cwd, + "listener_expected": service.listener_expected, + "requested_port": service.requested_port, + "actual_port": service.actual_port, + "log_path": service.log_path, + "failure_detail": getattr(service, "failure_detail", None), + "critical": getattr(service, "critical", True), + "degraded": getattr(service, "degraded", False), + "runtime_kind": getattr(service, "runtime_kind", "process"), + "container_id": getattr(service, "container_id", None), + "container_name": getattr(service, "container_name", None), + "container_image": getattr(service, "container_image", None), + "container_launch_token": getattr(service, "container_launch_token", None), + "container_cleanup_pending_since": getattr( + service, + "container_cleanup_pending_since", + None, + ), + } + + +def _projected_service_fields(raw: object) -> tuple[int | None, str | None, str]: + if not isinstance(raw, Mapping): + return None, None, "unknown" + port_raw = raw.get("port") + port = port_raw if isinstance(port_raw, int) else None + url_raw = raw.get("url") + url = str(url_raw) if isinstance(url_raw, str) and url_raw else None + status = str(raw.get("status", "unknown") or "unknown") + return port, url, status + + def _service_projection_ready(service: object | None) -> bool: if service is None: return False @@ -126,6 +254,9 @@ def _service_projection_ready(service: object | None) -> bool: def _project_name_from_service_record(service: object) -> str: + resolved_project = service_project_name(service) + if resolved_project: + return resolved_project name = str(getattr(service, "name", "") or "").strip() service_type = str(getattr(service, "type", "") or "").strip().lower() display_suffix = service_display_name(service_type) diff --git a/python/envctl_engine/state/state_aggregation.py b/python/envctl_engine/state/state_aggregation.py index bf63e9904..d0242985f 100644 --- a/python/envctl_engine/state/state_aggregation.py +++ b/python/envctl_engine/state/state_aggregation.py @@ -2,7 +2,8 @@ import json from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace +from pathlib import Path from envctl_engine.dashboard_metadata import DASHBOARD_STOPPED_SERVICES_KEY from envctl_engine.state.fingerprints import text_fingerprint @@ -57,10 +58,16 @@ def select_indexed_owners( owned_projects = frozenset( project_name for project_name, owner_run_id in owner_by_project.items() if owner_run_id == run_id ) + candidate_projects = normalized_project_names(indexed_state.candidate.project_names) filtered = filter_state_to_owned_projects( indexed_state.state, owned_projects, service_project_name=service_project_name, + fallback_project=( + next(iter(owned_projects)) + if len(owned_projects) == 1 and len(candidate_projects) == 1 + else None + ), ) chosen_states.append(IndexedState(candidate=indexed_state.candidate, state=filtered)) return chosen_states @@ -92,22 +99,49 @@ def filter_state_to_owned_projects( indexed_projects: frozenset[str], *, service_project_name: ServiceProjectResolver, + fallback_project: str | None = None, ) -> RunState: if not indexed_projects: return state - services = { - name: service - for name, service in state.services.items() - if service_project_name(name, service).casefold() in indexed_projects - } + fallback_key = str(fallback_project or "").strip().casefold() + fallback_display = _owned_project_display_name(state, fallback_key) if fallback_key else "" + known_project_keys = _known_state_project_keys(state).union(indexed_projects) + services: dict[str, ServiceRecord] = {} + ambiguous_services: list[str] = [] + for name, service in state.services.items(): + resolved_project = service_project_name(name, service).strip() + if resolved_project and resolved_project.casefold() not in known_project_keys: + resolved_project = "" + if not resolved_project: + resolved_project = _service_project_from_cwd_roots( + state, + service, + ) + if not resolved_project and fallback_key: + resolved_project = fallback_display or fallback_key + service = replace(service, project=resolved_project) + elif resolved_project and not str(getattr(service, "project", "") or "").strip(): + service = replace(service, project=resolved_project) + if not resolved_project: + ambiguous_services.append(name) + continue + if resolved_project.casefold() in indexed_projects: + services[name] = service + if ambiguous_services: + raise RuntimeError( + "Cannot safely determine project ownership for service(s): " + + ", ".join(sorted(ambiguous_services)) + + "; candidate projects: " + + ", ".join(sorted(indexed_projects)) + ) requirements = { name: requirement for name, requirement in state.requirements.items() - if str(name).strip().casefold() in indexed_projects + if str(getattr(requirement, "project", "") or name).strip().casefold() in indexed_projects or ( state.mode == "trees" - and str(name).strip().casefold() == "main" + and str(getattr(requirement, "project", "") or name).strip().casefold() == "main" and bool(state.metadata.get("shared_dependencies")) ) } @@ -134,6 +168,64 @@ def filter_state_to_owned_projects( ) +def _owned_project_display_name(state: RunState, project_key: str) -> str: + if not project_key: + return "" + metadata_names = state.metadata.get("project_names") + if isinstance(metadata_names, Sequence) and not isinstance(metadata_names, (str, bytes)): + for name in metadata_names: + rendered = str(name).strip() + if rendered.casefold() == project_key: + return rendered + for storage_key, requirements in state.requirements.items(): + rendered = str(getattr(requirements, "project", "") or storage_key).strip() + if rendered.casefold() == project_key: + return rendered + return project_key + + +def _known_state_project_keys(state: RunState) -> set[str]: + projects: set[str] = set() + metadata_names = state.metadata.get("project_names") + if isinstance(metadata_names, Sequence) and not isinstance(metadata_names, (str, bytes)): + projects.update(str(name).strip().casefold() for name in metadata_names if str(name).strip()) + metadata_roots = state.metadata.get("project_roots") + if isinstance(metadata_roots, Mapping): + projects.update(str(name).strip().casefold() for name in metadata_roots if str(name).strip()) + projects.update( + str(getattr(requirements, "project", "") or storage_key).strip().casefold() + for storage_key, requirements in state.requirements.items() + if str(getattr(requirements, "project", "") or storage_key).strip() + ) + return projects + + +def _service_project_from_cwd_roots( + state: RunState, + service: ServiceRecord, +) -> str: + cwd = str(getattr(service, "cwd", "") or "").strip() + roots = state.metadata.get("project_roots") + if not cwd or not isinstance(roots, Mapping): + return "" + try: + cwd_path = Path(cwd).expanduser().resolve(strict=False) + except (OSError, RuntimeError): + return "" + matches: list[str] = [] + for project, root in roots.items(): + project_name = str(project).strip() + if not project_name: + continue + try: + root_path = Path(str(root)).expanduser().resolve(strict=False) + cwd_path.relative_to(root_path) + except (OSError, RuntimeError, ValueError): + continue + matches.append(project_name) + return matches[0] if len(matches) == 1 else "" + + def _display_project_names( state: RunState, *, @@ -185,10 +277,30 @@ def merge_indexed_states( project_names: dict[str, str] = {} for indexed_state in ordered: state = indexed_state.state - services.update(state.services) - requirements.update(state.requirements) + state_project_names = [str(name).strip() for name in project_names_from_state(state) if str(name).strip()] + if not state_project_names: + state_project_names = [ + str(name).strip() + for name in indexed_state.candidate.project_names + if str(name).strip() + ] + fallback_project = state_project_names[0] if len(state_project_names) == 1 else "" + incoming_services = { + name: ( + service + if str(getattr(service, "project", "") or "").strip() or not fallback_project + else replace(service, project=fallback_project) + ) + for name, service in state.services.items() + } + _merge_indexed_services( + services, + incoming_services, + service_project_name=_service_project_from_record, + ) + _merge_indexed_requirements(requirements, state.requirements) pointers.update(state.pointers) - for name in project_names_from_state(state): + for name in state_project_names: project_names.setdefault(name.casefold(), name) roots = state.metadata.get("project_roots") if isinstance(roots, Mapping): @@ -225,6 +337,82 @@ def merge_indexed_states( ) +def _service_project_from_record(name: str, service: ServiceRecord) -> str: + project = str(getattr(service, "project", "") or "").strip() + if project: + return project + rendered = str(name).strip() + service_type = str(getattr(service, "type", "") or "").strip() + suffix = f" {service_type}".casefold() + return rendered[: -len(suffix)].strip() if suffix and rendered.casefold().endswith(suffix) else "" + + +def _merge_indexed_services( + merged: dict[str, ServiceRecord], + incoming: Mapping[str, ServiceRecord], + *, + service_project_name: ServiceProjectResolver, +) -> None: + for storage_name, service in incoming.items(): + existing = merged.get(storage_name) + if existing is None: + merged[storage_name] = service + continue + existing_project = service_project_name(storage_name, existing).strip().casefold() + incoming_project = service_project_name(storage_name, service).strip().casefold() + if not existing_project or not incoming_project or existing_project == incoming_project: + merged[storage_name] = service + continue + replacement_name = _unique_aggregate_storage_name( + storage_name, + owner=str(getattr(service, "project", "") or incoming_project), + pid=getattr(service, "pid", None), + occupied=set(merged).union(incoming), + ) + merged[replacement_name] = replace(service, name=replacement_name) + + +def _merge_indexed_requirements( + merged: dict[str, RequirementsResult], + incoming: Mapping[str, RequirementsResult], +) -> None: + for storage_key, requirements in incoming.items(): + existing = merged.get(storage_key) + if existing is None: + merged[storage_key] = requirements + continue + existing_project = str(existing.project or storage_key).strip().casefold() + incoming_project = str(requirements.project or storage_key).strip().casefold() + if not existing_project or not incoming_project or existing_project == incoming_project: + merged[storage_key] = requirements + continue + replacement_key = _unique_aggregate_storage_name( + storage_key, + owner=str(requirements.project or incoming_project), + pid=None, + occupied=set(merged).union(incoming), + ) + merged[replacement_key] = requirements + + +def _unique_aggregate_storage_name( + original: str, + *, + owner: str, + pid: object, + occupied: set[str], +) -> str: + owner_label = str(owner).strip() or "unknown" + pid_label = f" {pid}" if isinstance(pid, int) and pid > 0 else "" + base = f"{original} State Collision {owner_label}{pid_label}" + candidate = base + index = 2 + while candidate in occupied: + candidate = f"{base} {index}" + index += 1 + return candidate + + def merge_project_scoped_metadata( latest_metadata: Mapping[str, object], indexed_states: Sequence[IndexedState], diff --git a/python/envctl_engine/ui/dashboard/dependency_rendering.py b/python/envctl_engine/ui/dashboard/dependency_rendering.py index 645128101..77444cd21 100644 --- a/python/envctl_engine/ui/dashboard/dependency_rendering.py +++ b/python/envctl_engine/ui/dashboard/dependency_rendering.py @@ -25,19 +25,23 @@ def print_dashboard_dependency_rows( severity_color_fn: SeverityColorFn, ) -> None: _ = label_color - requirements = state.requirements.get(project) - if requirements is None: - return - print_dashboard_requirement_component_rows( - self, - requirements=requirements, - ok_color=ok_color, - warn_color=warn_color, - bad_color=bad_color, - reset=reset, - visual_url_fn=visual_url_fn, - severity_color_fn=severity_color_fn, - ) + project_key = str(project).strip().casefold() + matching_requirements = [ + requirements + for storage_key, requirements in state.requirements.items() + if str(getattr(requirements, "project", "") or storage_key).strip().casefold() == project_key + ] + for requirements in matching_requirements: + print_dashboard_requirement_component_rows( + self, + requirements=requirements, + ok_color=ok_color, + warn_color=warn_color, + bad_color=bad_color, + reset=reset, + visual_url_fn=visual_url_fn, + severity_color_fn=severity_color_fn, + ) def print_dashboard_shared_dependency_rows( diff --git a/python/envctl_engine/ui/dashboard/pr_link_rendering.py b/python/envctl_engine/ui/dashboard/pr_link_rendering.py index ee66d0253..1e5e1271a 100644 --- a/python/envctl_engine/ui/dashboard/pr_link_rendering.py +++ b/python/envctl_engine/ui/dashboard/pr_link_rendering.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Any +from envctl_engine.shared.services import service_for_project_type from envctl_engine.state.models import RunState @@ -72,8 +73,8 @@ def dashboard_project_root(self: Any, *, state: RunState, project: str) -> Path if isinstance(root_raw, str) and root_raw.strip(): return Path(root_raw).expanduser() - for service_suffix in (" Backend", " Frontend"): - service = state.services.get(f"{project}{service_suffix}") + for service_type in ("backend", "frontend"): + service = service_for_project_type(state, project=project, service_type=service_type) cwd_raw = getattr(service, "cwd", None) if not isinstance(cwd_raw, str) or not cwd_raw.strip(): continue diff --git a/python/envctl_engine/ui/dashboard/restart_selection_support.py b/python/envctl_engine/ui/dashboard/restart_selection_support.py index d047092f2..53ce54d91 100644 --- a/python/envctl_engine/ui/dashboard/restart_selection_support.py +++ b/python/envctl_engine/ui/dashboard/restart_selection_support.py @@ -300,8 +300,6 @@ def restart_services_by_project(state: RunState, runtime: Any) -> dict[str, list project_name = service_project_name(service) if not project_name: project_name = str(runtime._project_name_from_service(service_name) or "").strip() - if not project_name: - project_name = str(service_project_name(service_name)).strip() if callable(service_project_name) else "" if not project_name: project_name = "Services" service_type = stop_service_type(service_name, service) @@ -344,6 +342,8 @@ def restart_services_by_project(state: RunState, runtime: Any) -> dict[str, list def dashboard_stopped_services_by_project(state: RunState) -> dict[str, dict[str, str]]: + from envctl_engine.dashboard_metadata import normalize_dashboard_service_types + metadata = state.metadata if isinstance(state.metadata, dict) else {} raw_stopped = metadata.get("dashboard_stopped_services") stopped: dict[str, dict[str, str]] = {} @@ -353,11 +353,12 @@ def dashboard_stopped_services_by_project(state: RunState) -> dict[str, dict[str if not isinstance(item, dict): continue project = str(item.get("project", "") or "").strip() - service_type = str(item.get("type", "") or "").strip().lower() + normalized_types = normalize_dashboard_service_types([item.get("type", "")]) + service_type = normalized_types[0] if normalized_types else "" name = str(item.get("name", "") or "").strip() - if not project or service_type not in {"backend", "frontend"}: + if not project or not service_type: continue - stopped.setdefault(project, {})[service_type] = name or f"{project} {service_type.title()}" + stopped.setdefault(project, {})[service_type] = name or f"{project} {service_display_name(service_type)}" return stopped diff --git a/python/envctl_engine/ui/dashboard/snapshot_rendering.py b/python/envctl_engine/ui/dashboard/snapshot_rendering.py index 5f3cab187..31c008e2a 100644 --- a/python/envctl_engine/ui/dashboard/snapshot_rendering.py +++ b/python/envctl_engine/ui/dashboard/snapshot_rendering.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from typing import Any +from envctl_engine.shared.services import service_for_project_type from envctl_engine.state.models import RunState from envctl_engine.ui.dashboard.snapshot_support import build_dashboard_snapshot_model @@ -137,8 +138,8 @@ def _print_project( ) -> None: backend_url = project_item.get("backend_url") frontend_url = project_item.get("frontend_url") - backend_service = state.services.get(f"{project} Backend") - frontend_service = state.services.get(f"{project} Frontend") + backend_service = service_for_project_type(state, project=project, service_type="backend") + frontend_service = service_for_project_type(state, project=project, service_type="frontend") stopped_for_project = snapshot.stopped_services.get(project, {}) configured_missing_for_project = snapshot.configured_missing_services.get(project, set()) backend_stopped = backend_service is None and ( diff --git a/python/envctl_engine/ui/dashboard/stop_scope_support.py b/python/envctl_engine/ui/dashboard/stop_scope_support.py index b90935b90..c78ce7786 100644 --- a/python/envctl_engine/ui/dashboard/stop_scope_support.py +++ b/python/envctl_engine/ui/dashboard/stop_scope_support.py @@ -212,8 +212,8 @@ def stop_project_order(state: RunState, runtime: Any, *, project_names_from_stat if name and name.casefold() not in seen: seen.add(name.casefold()) names.append(name) - for project_name in state.requirements: - name = str(project_name).strip() + for storage_name, requirements in state.requirements.items(): + name = str(getattr(requirements, "project", "") or storage_name).strip() if name and name.casefold() not in seen: seen.add(name.casefold()) names.append(name) @@ -242,8 +242,8 @@ def stop_services_by_project(state: RunState, runtime: Any) -> dict[str, list[tu def stop_dependencies_by_project(state: RunState) -> dict[str, list[tuple[str, str]]]: labels = {definition.id: definition.display_name for definition in dependency_definitions()} dependencies_by_project: dict[str, list[tuple[str, str]]] = {} - for project_name, requirements in state.requirements.items(): - project = str(project_name).strip() + for storage_name, requirements in state.requirements.items(): + project = str(getattr(requirements, "project", "") or storage_name).strip() if not project: continue for definition in dependency_definitions(): diff --git a/python/envctl_engine/ui/selection_support.py b/python/envctl_engine/ui/selection_support.py index b46efe457..e9042dcb0 100644 --- a/python/envctl_engine/ui/selection_support.py +++ b/python/envctl_engine/ui/selection_support.py @@ -4,7 +4,7 @@ from typing import Any, Sequence from envctl_engine.runtime.command_router import Route -from envctl_engine.shared.services import project_name_from_service_name +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.state.models import RunState from envctl_engine.ui.dashboard.terminal_ui import RuntimeTerminalUI from envctl_engine.ui.selection_types import TargetSelection @@ -36,8 +36,8 @@ def interactive_selection_allowed( def project_names_from_state(runtime: Any, state: RunState) -> list[object]: names: list[str] = [] seen: set[str] = set() - for project in state.requirements: - project_name = str(project).strip() + for storage_name, requirements in state.requirements.items(): + project_name = str(getattr(requirements, "project", "") or storage_name).strip() if not project_name: continue lowered = project_name.lower() @@ -45,10 +45,12 @@ def project_names_from_state(runtime: Any, state: RunState) -> list[object]: continue seen.add(lowered) names.append(project_name) - for name in state.services: - project = runtime._project_name_from_service(name) # type: ignore[attr-defined] - if not project: - project = project_name_from_service_name(str(name)) + for name, service in state.services.items(): + project = resolve_service_project_name( + name, + service, + project_name_from_service=runtime._project_name_from_service, # type: ignore[attr-defined] + ) if not project: continue lowered = project.lower() @@ -77,10 +79,12 @@ def services_from_selection(runtime: Any, selection: TargetSelection, state: Run return {name for name in state.services if name in selection.service_names} if selection.project_names: selected: set[str] = set() - for name in state.services: - project = runtime._project_name_from_service(name) # type: ignore[attr-defined] - if not project: - project = project_name_from_service_name(str(name)) + for name, service in state.services.items(): + project = resolve_service_project_name( + name, + service, + project_name_from_service=runtime._project_name_from_service, # type: ignore[attr-defined] + ) if not project: continue for wanted in selection.project_names: diff --git a/tests/python/actions/test_action_target_support.py b/tests/python/actions/test_action_target_support.py index 627676982..6e60643ba 100644 --- a/tests/python/actions/test_action_target_support.py +++ b/tests/python/actions/test_action_target_support.py @@ -102,6 +102,27 @@ def project_name_from_service(name: str) -> str: self.assertEqual(resolved, ["feature-a-1", "feature-b-1"]) self.assertEqual(mapped, ["feature-a-1 Worker", "feature-b-1 Worker"]) + def test_projects_for_services_prefers_additional_service_project_metadata(self) -> None: + state = SimpleNamespace( + services={ + "My Project Voice Runtime": SimpleNamespace( + name="My Project Voice Runtime", + type="voice-runtime", + project="My Project", + ) + } + ) + mapped: list[str] = [] + runtime = SimpleNamespace( + load_existing_state=lambda mode: state if mode == "trees" else None, + project_name_from_service=lambda name: mapped.append(name) or "", + ) + + resolved = projects_for_services(runtime, ["voice-runtime"]) + + self.assertEqual(resolved, ["My Project"]) + self.assertEqual(mapped, []) + def test_resolve_action_targets_uses_single_candidate_without_prompt(self) -> None: target = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") runtime = SimpleNamespace( diff --git a/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py b/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py index 20e6c844e..003136731 100644 --- a/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py +++ b/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py @@ -163,11 +163,19 @@ def test_matching_and_port_helpers_cover_stale_or_partial_state(self) -> None: cwd="/tmp/repo/trees/feature-a-2/apps/api", requested_port=8000, ) + metadata_service = ServiceRecord( + name="Opaque API Process", + type="backend", + cwd="/tmp/outside-project-root", + requested_port=8100, + project="feature-a", + ) stale_service = ServiceRecord(name="feature-b backend", type="backend", cwd="", actual_port=9000) self.assertTrue(_state_project_matches_worktree("feature-a", worktree)) self.assertFalse(_state_project_matches_worktree("feature-b", worktree)) self.assertTrue(_state_service_matches_worktree(matching_service, worktree)) + self.assertTrue(_state_service_matches_worktree(metadata_service, worktree)) self.assertFalse(_state_service_matches_worktree(stale_service, worktree)) self.assertEqual(_component_port({"requested": "0", "resources": {"primary": "49152"}}), 49152) self.assertEqual( diff --git a/tests/python/runtime/lifecycle_parity_test_support.py b/tests/python/runtime/lifecycle_parity_test_support.py index 8126f913d..fc53df8da 100644 --- a/tests/python/runtime/lifecycle_parity_test_support.py +++ b/tests/python/runtime/lifecycle_parity_test_support.py @@ -103,12 +103,13 @@ def pid_owns_port(self, pid: int, port: int) -> bool: class _ResumeRestoreRunner: - def __init__(self) -> None: + def __init__(self, *, listeners_ready: bool = False) -> None: self.run_calls: list[tuple[str, ...]] = [] self.start_calls: list[tuple[str, ...]] = [] self.terminated: list[int] = [] self._next_pid = 42000 self._running: set[int] = set() + self.listeners_ready = listeners_ready self.fail_alembic = False def run(self, cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 @@ -167,7 +168,7 @@ def wait_for_pid_port( debug_pid_wait_group: str = "", ) -> bool: _ = host, timeout, debug_pid_wait_group - return False + return self.listeners_ready and self.pid_owns_port(_pid, _port) def pid_owns_port(self, pid: int, _port: int) -> bool: return pid in self._running diff --git a/tests/python/runtime/test_command_flag_storage.py b/tests/python/runtime/test_command_flag_storage.py index 9c05e94a4..d4d93398d 100644 --- a/tests/python/runtime/test_command_flag_storage.py +++ b/tests/python/runtime/test_command_flag_storage.py @@ -19,6 +19,7 @@ def test_parse_projects_splits_comma_separated_targets(self) -> None: def test_boolean_flag_name_maps_known_runtime_flags(self) -> None: self.assertEqual(boolean_flag_name("--headless"), "batch") self.assertEqual(boolean_flag_name("--cmux"), "cmux") + self.assertEqual(boolean_flag_name("--ignore-service-deps"), "ignore_service_deps") def test_store_value_flag_extends_repeatable_values(self) -> None: flags: dict[str, object] = {"services": ["backend"]} diff --git a/tests/python/runtime/test_docker_service_runtime.py b/tests/python/runtime/test_docker_service_runtime.py index 84ed397aa..22dd0ddd6 100644 --- a/tests/python/runtime/test_docker_service_runtime.py +++ b/tests/python/runtime/test_docker_service_runtime.py @@ -1,10 +1,12 @@ from __future__ import annotations import json +import os from pathlib import Path from types import SimpleNamespace from envctl_engine.runtime.docker_service_runtime import ( + DockerServiceLaunchCleanupError, DockerServiceRuntime, docker_service_container_name, refresh_container_log_snapshots, @@ -24,6 +26,7 @@ def __init__(self) -> None: self.terminated: list[int] = [] self.build_env: dict[str, str] = {} self.existing_container: tuple[bool, str] | None = None + self.launched_container: dict[str, str] | None = None self.local_images: set[str] = set() self.remote_exposed_ports: dict[str, dict[str, object]] = {} @@ -36,15 +39,52 @@ def run(self, command, **_kwargs): if argv[:2] == ["docker", "run"]: env_file = Path(argv[argv.index("--env-file") + 1]) self.env_file_payload = env_file.read_text(encoding="utf-8") - return SimpleNamespace(returncode=0, stdout="container-id\n", stderr="") + labels = [argv[index + 1] for index, value in enumerate(argv) if value == "--label"] + self.launched_container = { + "id": "d" * 64, + "name": argv[argv.index("--name") + 1], + "scope": next(value.split("=", 1)[1] for value in labels if value.startswith("io.envctl.runtime-scope=")), + "token": next(value.split("=", 1)[1] for value in labels if value.startswith("io.envctl.launch-token=")), + } + return SimpleNamespace(returncode=0, stdout=f"{'d' * 64}\n", stderr="") if argv[:3] == ["docker", "container", "inspect"]: + reference = argv[-1] + launched = self.launched_container + if launched is not None and reference in {launched["id"], launched["name"]}: + format_value = argv[argv.index("--format") + 1] + if ".State.Running" in format_value: + return SimpleNamespace( + returncode=0, + stdout=f"{launched['id']}|true|{launched['scope']}\n", + stderr="", + ) + label_value = launched["token"] if "io.envctl.launch-token" in format_value else launched["scope"] + return SimpleNamespace( + returncode=0, + stdout=f"{launched['id']}|{label_value}\n", + stderr="", + ) if self.existing_container is None: return SimpleNamespace(returncode=1, stdout="", stderr="No such container") running, owner = self.existing_container format_value = argv[argv.index("--format") + 1] if ".State.Running" not in format_value: - return SimpleNamespace(returncode=0, stdout=f"{owner}\n", stderr="") - return SimpleNamespace(returncode=0, stdout=f"{str(running).lower()}|{owner}\n", stderr="") + return SimpleNamespace(returncode=0, stdout=f"{'c' * 64}|{owner}\n", stderr="") + return SimpleNamespace( + returncode=0, + stdout=f"{'c' * 64}|{str(running).lower()}|{owner}\n", + stderr="", + ) + if argv[:3] == ["docker", "container", "ls"]: + launched = self.launched_container + token_filter = argv[argv.index("--filter") + 1] + expected_token = token_filter.rpartition("=")[2] + stdout = ( + f"{launched['id']}\n" + if launched is not None and launched["token"] == expected_token + else "" + ) + return SimpleNamespace(returncode=0, stdout=stdout, stderr="") if argv[:3] == ["docker", "inspect", "--format"]: return SimpleNamespace(returncode=0 if self.running else 1, stdout="true\n" if self.running else "", stderr="") if argv[:3] == ["docker", "image", "inspect"]: @@ -59,6 +99,11 @@ def run(self, command, **_kwargs): self.local_images.add(argv[-1]) return SimpleNamespace(returncode=0, stdout="pulled\n", stderr="") if argv[:3] == ["docker", "rm", "--force"]: + if self.launched_container is not None and argv[-1] in { + self.launched_container["id"], + self.launched_container["name"], + }: + self.launched_container = None return SimpleNamespace(returncode=0, stdout="", stderr="") if argv[:2] == ["docker", "logs"]: return SimpleNamespace(returncode=0, stdout="container log\n", stderr="") @@ -147,10 +192,839 @@ def test_builds_with_docker_layer_cache_and_runs_managed_container(tmp_path: Pat assert docker_run[runtime_label_index].startswith("io.envctl.runtime-scope=") assert "PATH=" not in runner.env_file_payload assert "DATABASE_URL=postgres://u:p@host.docker.internal:5432/db" in runner.env_file_payload - assert launch.container_id == "container-id" + assert launch.container_id == "d" * 64 assert any(event == "service.container.build" and payload["cache"] == "docker_layer_cache" for event, payload in events) +def test_docker_env_files_are_unique_private_and_independently_scoped(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner) + docker_runtime = DockerServiceRuntime(runtime, runner) + + first = docker_runtime._write_env_file(project_name="Main", service_name="backend", values={"SECRET": "one"}) + second = docker_runtime._write_env_file(project_name="Main", service_name="backend", values={"SECRET": "two"}) + try: + assert first != second + assert first.read_text(encoding="utf-8") == "SECRET=one\n" + assert second.read_text(encoding="utf-8") == "SECRET=two\n" + assert first.stat().st_mode & 0o777 == 0o600 + assert second.stat().st_mode & 0o777 == 0o600 + finally: + assert docker_runtime._scrub_env_file(first) is None + assert docker_runtime._scrub_env_file(second) is None + + +def test_stale_env_file_from_dead_process_is_scrubbed_before_new_write(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner) + docker_runtime = DockerServiceRuntime(runtime, runner) + run_dir = tmp_path / "runs" / "docker" + run_dir.mkdir(parents=True) + stale = run_dir / ".docker-env-aaaaaaaaaaaa-p99999999-stale123" + stale.write_text("SECRET=stale\n", encoding="utf-8") + monkeypatch.setattr(DockerServiceRuntime, "_pid_is_running", staticmethod(lambda _pid: False)) + + current = docker_runtime._write_env_file( + project_name="Main", + service_name="backend", + values={"SECRET": "current"}, + ) + try: + assert not stale.exists() + assert current.read_text(encoding="utf-8") == "SECRET=current\n" + finally: + assert docker_runtime._scrub_env_file(current) is None + + +def test_old_env_file_from_live_process_is_never_scrubbed(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_DOCKER_ENV_STALE_SECONDS": "0"}) + docker_runtime = DockerServiceRuntime(runtime, runner) + run_dir = tmp_path / "runs" / "docker" + run_dir.mkdir(parents=True) + active = run_dir / f".docker-env-aaaaaaaaaaaa-p{os.getpid()}-active123" + active.write_text("SECRET=active\n", encoding="utf-8") + os.utime(active, (1, 1)) + + current = docker_runtime._write_env_file( + project_name="Main", + service_name="backend", + values={"SECRET": "current"}, + ) + try: + assert active.read_text(encoding="utf-8") == "SECRET=active\n" + finally: + assert docker_runtime._scrub_env_file(active) is None + assert docker_runtime._scrub_env_file(current) is None + + +def test_old_legacy_env_file_is_scrubbed_after_grace_period(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner) + docker_runtime = DockerServiceRuntime(runtime, runner) + run_dir = tmp_path / "runs" / "docker" + run_dir.mkdir(parents=True) + stale = run_dir / ".docker-env-legacy" + stale.write_text("SECRET=legacy\n", encoding="utf-8") + os.utime(stale, (1, 1)) + + current = docker_runtime._write_env_file( + project_name="Main", + service_name="backend", + values={"SECRET": "current"}, + ) + try: + assert not stale.exists() + finally: + assert docker_runtime._scrub_env_file(current) is None + + +def test_unscrubbable_stale_env_file_blocks_new_secret_write(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner) + docker_runtime = DockerServiceRuntime(runtime, runner) + run_dir = tmp_path / "runs" / "docker" + run_dir.mkdir(parents=True) + stale = run_dir / ".docker-env-aaaaaaaaaaaa-p99999999-stale123" + stale.write_text("SECRET=stale\n", encoding="utf-8") + original_unlink = Path.unlink + original_open = os.open + + def fail_env_unlink(path: Path, *args, **kwargs): # noqa: ANN002, ANN003, ANN202 + if path == stale: + raise OSError("unlink unavailable") + return original_unlink(path, *args, **kwargs) + + def fail_env_truncate(path, flags, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + if Path(path) == stale and flags & os.O_TRUNC: + raise OSError("truncate unavailable") + return original_open(path, flags, *args, **kwargs) + + with monkeypatch.context() as scoped: + scoped.setattr(DockerServiceRuntime, "_pid_is_running", staticmethod(lambda _pid: False)) + scoped.setattr(Path, "unlink", fail_env_unlink) + scoped.setattr(os, "open", fail_env_truncate) + try: + docker_runtime._write_env_file( + project_name="Main", + service_name="backend", + values={"SECRET": "current"}, + ) + except RuntimeError as exc: + assert "stale docker env file cleanup failed" in str(exc).lower() + else: + raise AssertionError("an unscrubbable stale secret must block a new write") + + assert list(run_dir.glob(".docker-env-*")) == [stale] + assert docker_runtime._scrub_env_file(stale) is None + assert list(run_dir.glob(".docker-env-*")) == [] + + +def test_partial_env_write_failure_cannot_mask_unscrubbable_secret_file(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + runner = _Runner() + runtime, events = _runtime(tmp_path, runner) + docker_runtime = DockerServiceRuntime(runtime, runner) + original_unlink = Path.unlink + original_open = os.open + original_fdopen = os.fdopen + + class PartialWriteFailure: + def __init__(self, handle) -> None: # noqa: ANN001 + self.handle = handle + + def __enter__(self): # noqa: ANN204 + return self + + def write(self, value: str) -> None: + self.handle.write(value[:10]) + self.handle.flush() + raise OSError("partial write failed") + + def __exit__(self, *_args) -> None: # noqa: ANN002 + self.handle.close() + + def partial_fdopen(*args, **kwargs): # noqa: ANN002, ANN003, ANN202 + return PartialWriteFailure(original_fdopen(*args, **kwargs)) + + def fail_env_unlink(path: Path, *args, **kwargs): # noqa: ANN002, ANN003, ANN202 + if path.name.startswith(".docker-env-"): + raise OSError("unlink unavailable") + return original_unlink(path, *args, **kwargs) + + def fail_env_truncate(path, flags, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + if Path(path).name.startswith(".docker-env-") and flags & os.O_TRUNC: + raise OSError("truncate unavailable") + return original_open(path, flags, *args, **kwargs) + + with monkeypatch.context() as scoped: + scoped.setattr(os, "fdopen", partial_fdopen) + scoped.setattr(Path, "unlink", fail_env_unlink) + scoped.setattr(os, "open", fail_env_truncate) + try: + docker_runtime._write_env_file( + project_name="Main", + service_name="backend", + values={"SECRET": "do-not-leak"}, + ) + except RuntimeError as exc: + detail = str(exc).lower() + assert "partial write failed" in detail + assert "secret scrub failed" in detail + else: + raise AssertionError("the partial write must report the secret cleanup failure") + + env_files = list((tmp_path / "runs" / "docker").glob(".docker-env-*")) + assert len(env_files) == 1 + assert env_files[0].read_text(encoding="utf-8") + assert any(event == "service.container.env_file_cleanup_warning" for event, _payload in events) + assert docker_runtime._scrub_env_file(env_files[0]) is None + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_interrupt_during_env_write_scrubs_partial_secret_before_reraising(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner) + docker_runtime = DockerServiceRuntime(runtime, runner) + original_fdopen = os.fdopen + + class InterruptedWrite: + def __init__(self, handle) -> None: # noqa: ANN001 + self.handle = handle + + def __enter__(self): # noqa: ANN204 + return self + + def write(self, value: str) -> None: + self.handle.write(value) + self.handle.flush() + raise KeyboardInterrupt + + def __exit__(self, *_args) -> None: # noqa: ANN002 + self.handle.close() + + monkeypatch.setattr( + os, + "fdopen", + lambda *args, **kwargs: InterruptedWrite(original_fdopen(*args, **kwargs)), + ) + + try: + docker_runtime._write_env_file( + project_name="Main", + service_name="backend", + values={"SECRET": "must-not-remain"}, + ) + except KeyboardInterrupt: + pass + else: + raise AssertionError("the write interrupt must be restored after secret cleanup") + + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_successful_docker_run_with_empty_stdout_uses_owned_container_name(tmp_path: Path) -> None: + class EmptyOutputRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + result = super().run(command, **kwargs) + if [str(part) for part in command][:2] == ["docker", "run"]: + return SimpleNamespace(returncode=0, stdout="", stderr="") + return result + + runner = EmptyOutputRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + launch = DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + + assert launch.container_id == "d" * 64 + assert launch.launch_token + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + + +def test_successful_docker_run_with_malformed_stdout_resolves_owned_immutable_id(tmp_path: Path) -> None: + class DiagnosticOutputRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + result = super().run(command, **kwargs) + if [str(part) for part in command][:2] == ["docker", "run"]: + return SimpleNamespace(returncode=0, stdout="unexpected diagnostic line\n", stderr="") + return result + + runner = DiagnosticOutputRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + launch = DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + + assert launch.container_id == "d" * 64 + assert launch.container_id != "unexpected diagnostic line" + + +def test_env_file_unlink_failure_scrubs_secrets_without_masking_success(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + runner = _Runner() + runtime, events = _runtime( + tmp_path, + runner, + env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}, + ) + original_unlink = Path.unlink + + def fail_env_unlink(path: Path, *args, **kwargs): # noqa: ANN002, ANN003, ANN202 + if path.name.startswith(".docker-env-"): + raise OSError("unlink unavailable") + return original_unlink(path, *args, **kwargs) + + monkeypatch.setattr(Path, "unlink", fail_env_unlink) + + launch = DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={"SECRET": "do-not-leak"}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + + env_files = list((tmp_path / "runs" / "docker").glob(".docker-env-*")) + assert launch.container_id == "d" * 64 + assert len(env_files) == 1 + assert env_files[0].read_text(encoding="utf-8") == "" + assert any(event == "service.container.env_file_cleanup_warning" for event, _payload in events) + + +def test_unscrubbable_env_file_fails_launch_and_removes_owned_container(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + runner = _Runner() + runtime, events = _runtime( + tmp_path, + runner, + env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}, + ) + docker_runtime = DockerServiceRuntime(runtime, runner) + original_unlink = Path.unlink + original_open = os.open + + def fail_env_unlink(path: Path, *args, **kwargs): # noqa: ANN002, ANN003, ANN202 + if path.name.startswith(".docker-env-"): + raise OSError("unlink unavailable") + return original_unlink(path, *args, **kwargs) + + def fail_env_truncate(path, flags, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + if Path(path).name.startswith(".docker-env-") and flags & os.O_TRUNC: + raise OSError("truncate unavailable") + return original_open(path, flags, *args, **kwargs) + + with monkeypatch.context() as scoped: + scoped.setattr(Path, "unlink", fail_env_unlink) + scoped.setattr(os, "open", fail_env_truncate) + try: + docker_runtime.launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={"SECRET": "do-not-leak"}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except RuntimeError as exc: + assert "env file cleanup failed" in str(exc).lower() + else: + raise AssertionError("an unscrubbable secret file must fail the launch") + + env_files = list((tmp_path / "runs" / "docker").glob(".docker-env-*")) + assert runner.launched_container is None + assert ["docker", "rm", "--force", "d" * 64] in runner.calls + assert any(event == "service.container.env_file_cleanup_warning" for event, _payload in events) + assert len(env_files) == 1 + assert docker_runtime._scrub_env_file(env_files[0]) is None + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_pre_run_command_error_cannot_mask_unscrubbable_secret_file(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + runner = _Runner() + runtime, events = _runtime( + tmp_path, + runner, + env={ + "ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev", + "ENVCTL_BACKEND_DOCKER_COMMAND_MODE": "invalid", + }, + ) + docker_runtime = DockerServiceRuntime(runtime, runner) + original_unlink = Path.unlink + original_open = os.open + + def fail_env_unlink(path: Path, *args, **kwargs): # noqa: ANN002, ANN003, ANN202 + if path.name.startswith(".docker-env-"): + raise OSError("unlink unavailable") + return original_unlink(path, *args, **kwargs) + + def fail_env_truncate(path, flags, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + if Path(path).name.startswith(".docker-env-") and flags & os.O_TRUNC: + raise OSError("truncate unavailable") + return original_open(path, flags, *args, **kwargs) + + with monkeypatch.context() as scoped: + scoped.setattr(Path, "unlink", fail_env_unlink) + scoped.setattr(os, "open", fail_env_truncate) + try: + docker_runtime.launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=["python", "app.py"], + env={"SECRET": "do-not-leak"}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except RuntimeError as exc: + detail = str(exc).lower() + assert "invalid docker command mode" in detail + assert "env file cleanup failed" in detail + assert "secret scrub failed" in detail + else: + raise AssertionError("the pre-run error must report the secret cleanup failure") + + env_files = list((tmp_path / "runs" / "docker").glob(".docker-env-*")) + assert not any(call[:2] == ["docker", "run"] for call in runner.calls) + assert any(event == "service.container.env_file_cleanup_warning" for event, _payload in events) + assert len(env_files) == 1 + assert docker_runtime._scrub_env_file(env_files[0]) is None + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_tokenized_stop_treats_already_absent_container_as_confirmed(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner) + + stopped = DockerServiceRuntime(runtime, runner).stop( + "envctl-app-main-backend", + expected_launch_token="launch-token", + ) + + assert stopped + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + + +def test_tokenized_stop_removes_inspected_immutable_id_not_reused_name(tmp_path: Path) -> None: + container_id = "a" * 64 + + class NameReuseRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:3] == ["docker", "container", "inspect"]: + self.calls.append(argv) + return SimpleNamespace( + returncode=0, + stdout=f"{container_id}|launch-token\n", + stderr="", + ) + return super().run(command, **kwargs) + + runner = NameReuseRunner() + runtime, _events = _runtime(tmp_path, runner) + + stopped = DockerServiceRuntime(runtime, runner).stop( + "envctl-app-main-backend", + expected_launch_token="launch-token", + ) + + assert stopped + remove = next(call for call in runner.calls if call[:3] == ["docker", "rm", "--force"]) + assert remove == ["docker", "rm", "--force", container_id] + assert "envctl-app-main-backend" not in remove + + +def test_tokenized_stop_refuses_foreign_launch_token_without_deleting(tmp_path: Path) -> None: + class ForeignTokenRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:3] == ["docker", "container", "inspect"]: + self.calls.append(argv) + return SimpleNamespace( + returncode=0, + stdout=f"{'b' * 64}|foreign-token\n", + stderr="", + ) + return super().run(command, **kwargs) + + runner = ForeignTokenRunner() + runtime, _events = _runtime(tmp_path, runner) + + stopped = DockerServiceRuntime(runtime, runner).stop( + "envctl-app-main-backend", + expected_launch_token="expected-token", + ) + + assert not stopped + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + + +def test_tokenized_stop_rejects_short_container_id_without_deleting(tmp_path: Path) -> None: + class ShortIdentityRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:3] == ["docker", "container", "inspect"]: + self.calls.append(argv) + return SimpleNamespace(returncode=0, stdout="abcdef123456|launch-token\n", stderr="") + return super().run(command, **kwargs) + + runner = ShortIdentityRunner() + runtime, _events = _runtime(tmp_path, runner) + + stopped = DockerServiceRuntime(runtime, runner).stop( + "envctl-app-main-backend", + expected_launch_token="launch-token", + ) + + assert not stopped + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + + +def test_launch_transport_and_ownership_probe_failure_preserves_durable_identity(tmp_path: Path) -> None: + class InspectFailureRunner(_Runner): + def __init__(self) -> None: + super().__init__() + self.container_inspects = 0 + + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:3] == ["docker", "container", "inspect"]: + self.container_inspects += 1 + if self.container_inspects > 1: + raise OSError("inspect transport failed") + if argv[:2] == ["docker", "run"]: + env_file = Path(argv[argv.index("--env-file") + 1]) + self.env_file_payload = env_file.read_text(encoding="utf-8") + raise OSError("docker run transport failed") + return super().run(command, **kwargs) + + runner = InspectFailureRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + try: + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={"SECRET": "scrub-me"}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except DockerServiceLaunchCleanupError as exc: + assert "inspect transport failed" not in str(exc) + assert "cleanup could not confirm" in str(exc).lower() + assert exc.launch.container_id == exc.launch.container_name + assert exc.launch.container_name.startswith("envctl-app-main-backend") + assert exc.launch.launch_token + else: + raise AssertionError("unconfirmed launch cleanup must preserve durable identity") + + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_timeout_accepted_container_is_found_by_token_after_delayed_visibility(tmp_path: Path) -> None: + class DelayedVisibilityRunner(_Runner): + def __init__(self) -> None: + super().__init__() + self.pending_container: dict[str, str] | None = None + self.token_queries = 0 + + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:2] == ["docker", "run"]: + self.calls.append(argv) + env_file = Path(argv[argv.index("--env-file") + 1]) + self.env_file_payload = env_file.read_text(encoding="utf-8") + labels = [argv[index + 1] for index, value in enumerate(argv) if value == "--label"] + self.pending_container = { + "id": "f" * 64, + "name": argv[argv.index("--name") + 1], + "scope": next( + value.split("=", 1)[1] + for value in labels + if value.startswith("io.envctl.runtime-scope=") + ), + "token": next( + value.split("=", 1)[1] + for value in labels + if value.startswith("io.envctl.launch-token=") + ), + } + return SimpleNamespace(returncode=124, stdout="", stderr="Command timed out") + if argv[:3] == ["docker", "container", "ls"]: + self.token_queries += 1 + if self.token_queries == 2: + self.launched_container = self.pending_container + return super().run(command, **kwargs) + return super().run(command, **kwargs) + + runner = DelayedVisibilityRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + try: + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={"SECRET": "scrub-me"}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except RuntimeError as exc: + assert "timed out" in str(exc).lower() + assert not isinstance(exc, DockerServiceLaunchCleanupError) + else: + raise AssertionError("a timed-out Docker launch must fail after cleanup") + + assert runner.token_queries >= 2 + assert runner.launched_container is None + assert ["docker", "rm", "--force", "f" * 64] in runner.calls + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_timeout_without_visible_container_preserves_pending_cleanup_authority(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + class TimeoutRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:2] == ["docker", "run"]: + self.calls.append(argv) + env_file = Path(argv[argv.index("--env-file") + 1]) + self.env_file_payload = env_file.read_text(encoding="utf-8") + return SimpleNamespace(returncode=124, stdout="", stderr="Command timed out") + return super().run(command, **kwargs) + + clock = [0.0] + monkeypatch.setattr( + "envctl_engine.runtime.docker_service_runtime.time.monotonic", + lambda: clock[0], + ) + monkeypatch.setattr( + "envctl_engine.runtime.docker_service_runtime.time.sleep", + lambda seconds: clock.__setitem__(0, clock[0] + seconds), + ) + runner = TimeoutRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + docker_runtime = DockerServiceRuntime(runtime, runner) + + try: + docker_runtime.launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={"SECRET": "scrub-me"}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except DockerServiceLaunchCleanupError as exc: + pending = exc.launch + else: + raise AssertionError("an uncertain timed-out launch must retain cleanup authority") + + assert pending.cleanup_pending_since is not None + assert pending.container_id == pending.container_name + assert not docker_runtime.stop( + pending.container_id, + expected_launch_token=pending.launch_token, + pending_cleanup_since=pending.cleanup_pending_since, + ) + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_interrupt_after_daemon_accepts_run_removes_tokenized_container(tmp_path: Path) -> None: + class InterruptAfterCreateRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + result = super().run(command, **kwargs) + if [str(part) for part in command][:2] == ["docker", "run"]: + raise KeyboardInterrupt + return result + + runner = InterruptAfterCreateRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + try: + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={"SECRET": "scrub-me"}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except KeyboardInterrupt: + pass + else: + raise AssertionError("the interrupt must be restored after confirmed cleanup") + + assert runner.launched_container is None + assert ["docker", "rm", "--force", "d" * 64] in runner.calls + assert any(call[:3] == ["docker", "container", "ls"] for call in runner.calls) + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_interrupt_during_cleanup_discovery_preserves_pending_authority(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + class TimeoutRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:2] == ["docker", "run"]: + self.calls.append(argv) + return SimpleNamespace(returncode=124, stdout="", stderr="Command timed out") + return super().run(command, **kwargs) + + monkeypatch.setattr( + "envctl_engine.runtime.docker_service_runtime.time.sleep", + lambda _seconds: (_ for _ in ()).throw(KeyboardInterrupt()), + ) + runner = TimeoutRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + try: + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except DockerServiceLaunchCleanupError as exc: + pending = exc.launch + else: + raise AssertionError("cleanup cancellation must surface durable launch authority") + + assert pending.cleanup_pending_since is not None + assert pending.launch_token + assert pending.container_id == pending.container_name + + +def test_interrupt_during_post_run_identity_probe_cannot_orphan_container(tmp_path: Path) -> None: + class IdentityProbeInterruptRunner(_Runner): + def __init__(self) -> None: + super().__init__() + self.interrupted = False + + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if ( + argv[:3] == ["docker", "container", "inspect"] + and self.launched_container is not None + and not self.interrupted + ): + self.calls.append(argv) + self.interrupted = True + raise KeyboardInterrupt + return super().run(command, **kwargs) + + runner = IdentityProbeInterruptRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + try: + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except KeyboardInterrupt: + pass + else: + raise AssertionError("the identity-probe interrupt must be restored after cleanup") + + assert runner.launched_container is None + assert ["docker", "rm", "--force", "d" * 64] in runner.calls + + +def test_interrupting_best_effort_start_event_cannot_preempt_authority_handoff(tmp_path: Path) -> None: + runner = _Runner() + runner.local_images.add("example/backend:dev") + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + def interrupt_start_event(event: str, **_payload: object) -> None: + if event == "service.container.start": + raise KeyboardInterrupt + + runtime._emit = interrupt_start_event + confirmed: list[object] = [] + docker_runtime = DockerServiceRuntime(runtime, runner) + + launch = docker_runtime.launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + on_confirmed=confirmed.append, + ) + + assert confirmed == [launch] + assert runner.launched_container is not None + assert docker_runtime.stop( + launch.container_id, + expected_launch_token=launch.launch_token, + ) + assert runner.launched_container is None + + def test_running_untracked_container_is_not_replaced_or_removed(tmp_path: Path) -> None: runner = _Runner() runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) @@ -205,6 +1079,47 @@ def test_container_owned_by_another_runtime_scope_is_not_replaced(tmp_path: Path assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) +def test_stopped_container_preflight_rejects_invalid_running_state_and_missing_owner(tmp_path: Path) -> None: + cases = (("not-a-bool", True, "invalid running state"), ("false", False, "another envctl runtime scope")) + for running, include_owner, expected_error in cases: + class InvalidPreflightRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:3] == ["docker", "container", "inspect"]: + self.calls.append(argv) + owner = DockerServiceRuntime(runtime, self)._runtime_scope_identity() if include_owner else "" + return SimpleNamespace(returncode=0, stdout=f"{'e' * 64}|{running}|{owner}\n", stderr="") + return super().run(command, **kwargs) + + runner = InvalidPreflightRunner() + runtime, _events = _runtime( + tmp_path, + runner, + env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}, + ) + + try: + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except RuntimeError as exc: + assert expected_error in str(exc) + else: + raise AssertionError("invalid preflight identity must fail closed") + + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + assert not any(call[:2] == ["docker", "run"] for call in runner.calls) + + def test_stopped_container_from_same_runtime_scope_is_removed(tmp_path: Path) -> None: runner = _Runner() runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) @@ -228,6 +1143,7 @@ def test_stopped_container_from_same_runtime_scope_is_removed(tmp_path: Path) -> remove_index = next(index for index, call in enumerate(runner.calls) if call[:3] == ["docker", "rm", "--force"]) run_index = next(index for index, call in enumerate(runner.calls) if call[:2] == ["docker", "run"]) assert inspect_index < remove_index < run_index + assert runner.calls[remove_index][-1] == "c" * 64 def test_configured_image_runs_without_build_and_preserves_browser_public_url(tmp_path: Path) -> None: @@ -354,6 +1270,10 @@ def test_verified_stop_refuses_container_from_another_runtime_scope(tmp_path: Pa def test_container_state_drives_health_and_stop(tmp_path: Path) -> None: runner = _Runner() runtime, events = _runtime(tmp_path, runner) + runner.existing_container = ( + True, + DockerServiceRuntime(runtime, runner)._runtime_scope_identity(), + ) service = ServiceRecord( name="Main Backend", type="backend", @@ -372,7 +1292,7 @@ def test_container_state_drives_health_and_stop(tmp_path: Path) -> None: runner.port_ready = False assert service_truth_status(runtime, service) == "unreachable" assert terminate_service_record(runtime, service, aggressive=False, verify_ownership=True) - assert ["docker", "rm", "--force", "container-id"] in runner.calls + assert ["docker", "rm", "--force", "c" * 64] in runner.calls assert runner.terminated == [4321] assert any(event == "service.container.stop" for event, _payload in events) diff --git a/tests/python/runtime/test_engine_runtime_artifacts.py b/tests/python/runtime/test_engine_runtime_artifacts.py index d858f314a..f36d0920e 100644 --- a/tests/python/runtime/test_engine_runtime_artifacts.py +++ b/tests/python/runtime/test_engine_runtime_artifacts.py @@ -94,6 +94,7 @@ def test_print_summary_prefers_preserved_service_truth_over_unused_planned_port( def test_write_artifacts_forwards_expected_state_repository_payload(self) -> None: captured: dict[str, object] = {} + committed: list[bool] = [] def save_run(**kwargs): # noqa: ANN003 captured.update(kwargs) @@ -109,13 +110,22 @@ def save_run(**kwargs): # noqa: ANN003 ) state = RunState(run_id="run-1", mode="main") - write_artifacts(runtime, state, [SimpleNamespace(name="Main")], errors=["boom"]) + write_artifacts( + runtime, + state, + [SimpleNamespace(name="Main")], + errors=["boom"], + on_commit=lambda: committed.append(True), + ) self.assertIs(captured["state"], state) self.assertEqual(captured["errors"], ["boom"]) self.assertEqual(captured["events"], [{"event": "x"}]) self.assertTrue(callable(captured["runtime_map_builder"])) self.assertTrue(callable(captured["write_runtime_readiness_report"])) + self.assertTrue(callable(captured["on_commit"])) + captured["on_commit"]() + self.assertEqual(committed, [True]) def test_persist_events_snapshot_updates_bound_run_directory(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: diff --git a/tests/python/runtime/test_engine_runtime_command_parity_state_config.py b/tests/python/runtime/test_engine_runtime_command_parity_state_config.py index 374fcbed2..b830b37cb 100644 --- a/tests/python/runtime/test_engine_runtime_command_parity_state_config.py +++ b/tests/python/runtime/test_engine_runtime_command_parity_state_config.py @@ -2,10 +2,30 @@ from __future__ import annotations from tests.python.runtime.engine_runtime_command_parity_test_support import * +from envctl_engine.runtime.inspection_support import _running_tree_project_names from envctl_engine.state.run_index import StateSelector class EngineRuntimeCommandParityStateConfigTests(EngineRuntimeCommandParityTestCase): + def test_running_tree_projects_prefers_additional_service_project_metadata(self) -> None: + state = RunState( + run_id="run-1", + mode="trees", + services={ + "Customer Platform Voice Runtime": ServiceRecord( + name="Customer Platform Voice Runtime", + type="voice-runtime", + cwd=".", + status="running", + project="Customer Platform", + service_slug="voice-runtime", + ) + }, + ) + runtime = SimpleNamespace(_project_name_from_service=lambda _name: "") + + self.assertEqual(_running_tree_project_names(runtime=runtime, state=state), {"Customer Platform"}) + def test_list_targets_discovers_projects_in_main_mode(self) -> None: tmpdir = tempfile.TemporaryDirectory() self.addCleanup(tmpdir.cleanup) diff --git a/tests/python/runtime/test_engine_runtime_hooks.py b/tests/python/runtime/test_engine_runtime_hooks.py index d0af728d2..a6bd1a68f 100644 --- a/tests/python/runtime/test_engine_runtime_hooks.py +++ b/tests/python/runtime/test_engine_runtime_hooks.py @@ -126,13 +126,24 @@ def test_services_from_hook_payload_builds_records_and_updates_service_ports(sel "port": 9010, "status": "healthy", }, + { + "name": "Opaque Worker Process", + "type": "voice-runtime", + "pid": 103, + "port": 9020, + }, ] }, ) - self.assertEqual(sorted(records.keys()), ["feature/a-1 Backend", "feature/a-1 Frontend"]) + self.assertEqual( + sorted(records.keys()), + ["Opaque Worker Process", "feature/a-1 Backend", "feature/a-1 Frontend"], + ) self.assertEqual(records["feature/a-1 Backend"].actual_port, 8010) self.assertEqual(records["feature/a-1 Frontend"].requested_port, 9010) + self.assertEqual(records["Opaque Worker Process"].project, "feature/a-1") + self.assertEqual(records["Opaque Worker Process"].service_slug, "voice-runtime") self.assertEqual(context.ports["backend"].final, 8010) self.assertEqual(context.ports["frontend"].final, 9010) diff --git a/tests/python/runtime/test_engine_runtime_lifecycle_support.py b/tests/python/runtime/test_engine_runtime_lifecycle_support.py index 6ca190ecc..9ac92e916 100644 --- a/tests/python/runtime/test_engine_runtime_lifecycle_support.py +++ b/tests/python/runtime/test_engine_runtime_lifecycle_support.py @@ -190,7 +190,7 @@ def test_blast_worktree_before_delete_updates_state_and_emits_finish(self) -> No _emit=lambda event, **payload: events.append((event, payload)), _try_load_existing_state=lambda **kwargs: state if kwargs["mode"] == "trees" else None, _project_name_from_service=lambda name: "Feature" if "Feature" in name else "", - _terminate_services_from_state=lambda *args, **kwargs: None, + _terminate_services_from_state=lambda *args, **kwargs: set(), port_planner=SimpleNamespace(release=lambda port: released.append(port)), state_repository=SimpleNamespace( save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) @@ -299,7 +299,7 @@ def run(self, cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001, ARG002 _emit=lambda event, **payload: events.append((event, payload)), _try_load_existing_state=lambda **kwargs: state if kwargs["mode"] == "trees" else None, _project_name_from_service=lambda name: "Feature" if "Feature" in name else "", - _terminate_services_from_state=lambda *args, **kwargs: None, + _terminate_services_from_state=lambda *args, **kwargs: set(), _blast_all_process_command=lambda pid: f"python service {pid}", _blast_all_kill_pid_tree=lambda pid: killed_pids.append(pid), _collect_container_volume_candidates=lambda cid, volumes: volumes.append(f"vol-{cid}"), diff --git a/tests/python/runtime/test_engine_runtime_real_startup.py b/tests/python/runtime/test_engine_runtime_real_startup.py index 00310d1a9..b985eae31 100644 --- a/tests/python/runtime/test_engine_runtime_real_startup.py +++ b/tests/python/runtime/test_engine_runtime_real_startup.py @@ -72,7 +72,7 @@ def test_startup_emits_port_lock_reclaim_event_with_previous_owner_metadata(self lock_path.write_text( json.dumps( { - "owner": "old-worker", + "owner": "feature-a-1:backend", "session": "old-session", "pid": 999999, "created_at": "2000-01-01T00:00:00+00:00", @@ -95,7 +95,7 @@ def test_startup_emits_port_lock_reclaim_event_with_previous_owner_metadata(self self.assertTrue(reclaim_events) latest = reclaim_events[-1] self.assertEqual(latest.get("port"), planned_backend_port) - self.assertEqual(latest.get("reclaimed_owner"), "old-worker") + self.assertEqual(latest.get("reclaimed_owner"), "feature-a-1:backend") self.assertEqual(latest.get("reclaimed_session"), "old-session") self.assertEqual(latest.get("reclaimed_pid"), 999999) diff --git a/tests/python/runtime/test_engine_runtime_startup_support.py b/tests/python/runtime/test_engine_runtime_startup_support.py index 05a5cb0e9..68f53ffcc 100644 --- a/tests/python/runtime/test_engine_runtime_startup_support.py +++ b/tests/python/runtime/test_engine_runtime_startup_support.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json +import tempfile import unittest from pathlib import Path from types import SimpleNamespace @@ -659,6 +661,60 @@ def test_discover_projects_and_reserve_project_ports(self) -> None: reserve_project_ports(runtime, context) self.assertEqual(events[-1][0], "port.reserved") + def test_reserving_selected_project_reclaims_only_its_candidate_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_dir = Path(tmpdir) + foreign_payload = json.dumps( + { + "owner": "Other:backend", + "session": "historical-other", + "pid": 99999999, + "created_at": 0, + }, + sort_keys=True, + ) + selected_payload = json.dumps( + { + "owner": "Selected:backend", + "session": "historical-selected", + "pid": 99999998, + "created_at": 0, + }, + sort_keys=True, + ) + (lock_dir / "18000.lock").write_text(foreign_payload, encoding="utf-8") + (lock_dir / "18001.lock").write_text(selected_payload, encoding="utf-8") + planner = PortPlanner( + lock_dir=lock_dir, + session_id="selected-start", + availability_checker=lambda _port: True, + ) + runtime = SimpleNamespace( + port_planner=planner, + _emit=lambda *_args, **_kwargs: None, + _lock_inventory=lambda: [], + ) + context = ProjectContext( + name="Selected", + root=Path("/repo/trees/selected"), + ports={ + "backend": PortPlan( + project="Selected", + requested=18001, + assigned=18001, + final=18001, + source="fixed", + ) + }, + ) + + reserve_project_ports(runtime, context) + + self.assertEqual((lock_dir / "18000.lock").read_text(encoding="utf-8"), foreign_payload) + selected_lock = json.loads((lock_dir / "18001.lock").read_text(encoding="utf-8")) + self.assertEqual(selected_lock["owner"], "Selected:backend") + self.assertEqual(selected_lock["session"], "selected-start") + def test_no_infra_reserves_no_application_or_dependency_ports(self) -> None: reserve_calls: list[tuple[int, str]] = [] events: list[tuple[str, dict[str, object]]] = [] diff --git a/tests/python/runtime/test_engine_runtime_state_truth.py b/tests/python/runtime/test_engine_runtime_state_truth.py index b4f6e3d72..0af60b107 100644 --- a/tests/python/runtime/test_engine_runtime_state_truth.py +++ b/tests/python/runtime/test_engine_runtime_state_truth.py @@ -18,10 +18,25 @@ requirement_truth_issues, state_fingerprint, ) +from envctl_engine.runtime.requirement_reconcile_truth import requirement_truth_identity # noqa: E402 from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord # noqa: E402 class EngineRuntimeStateTruthTests(unittest.TestCase): + def test_collision_requirement_truth_uses_authoritative_project_and_root(self) -> None: + requirements = RequirementsResult(project="Main", redis={"enabled": True, "final": 6381}) + state = RunState( + run_id="run-collision", + mode="main", + requirements={"Main Restart Collision": requirements}, + metadata={"project_roots": {"Main": "/repo/main"}}, + ) + + project, root = requirement_truth_identity(state, "Main Restart Collision", requirements) + + self.assertEqual(project, "Main") + self.assertEqual(root, Path("/repo/main")) + def test_state_fingerprint_changes_when_state_changes(self) -> None: first = RunState(run_id="run-1", mode="main", services={}) second = RunState( diff --git a/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py b/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py index 9332c314f..69c412d74 100644 --- a/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py +++ b/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py @@ -88,7 +88,7 @@ def _state_lookup_strict_mode_match(_route): # noqa: ANN001 return False def _terminate_services_from_state(self, *args, **kwargs): # noqa: ANN001, ARG002 - return None + return set() def _release_port_session(self) -> None: return None @@ -142,11 +142,11 @@ def _terminate_services_from_state(self, *args, **kwargs): # noqa: ANN001, ARG0 class LifecycleCleanupSpinnerIntegrationTests(unittest.TestCase): - def test_clear_runtime_state_purges_after_dependency_container_cleanup_failure(self) -> None: + def test_full_cleanup_retains_dependency_state_after_container_cleanup_failure(self) -> None: for command, aggressive in (("stop-all", False), ("blast-all", True)): with self.subTest(command=command): runtime = _RuntimeStub() - runtime._try_load_existing_state = lambda *args, **kwargs: RunState( # type: ignore[method-assign] + state = RunState( run_id="run-1", mode="main", requirements={ @@ -160,6 +160,11 @@ def test_clear_runtime_state_purges_after_dependency_container_cleanup_failure(s ) }, ) + runtime.state_repository.active_states = [state] + released_requirements: list[str] = [] + runtime._release_requirement_ports = ( # type: ignore[attr-defined] + lambda requirements: released_requirements.append(requirements.project) + ) orchestrator = LifecycleCleanupOrchestrator(runtime) orchestrator.blast_all_ecosystem_enabled = lambda: False # type: ignore[method-assign] orchestrator.blast_all_purge_legacy_state_artifacts = lambda: None # type: ignore[method-assign] @@ -172,9 +177,15 @@ def test_clear_runtime_state_purges_after_dependency_container_cleanup_failure(s ), redirect_stdout(output), ): - orchestrator.clear_runtime_state(command=command, aggressive=aggressive) - - self.assertEqual(runtime.state_repository.purge_calls, [aggressive]) + code = orchestrator.execute(Route(command=command, mode="main")) + + self.assertEqual(code, 1) + self.assertEqual(runtime.state_repository.purge_calls, []) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(runtime.state_repository.active_states, [state]) + self.assertEqual(runtime.state_repository.saved_states, [state]) + self.assertEqual(set(state.requirements), {"Main"}) + self.assertEqual(released_requirements, []) self.assertIn("could not stop redis container", output.getvalue()) warnings = [ event @@ -191,6 +202,17 @@ def test_clear_runtime_state_purges_after_dependency_container_cleanup_failure(s } ], ) + incomplete = [event for event in runtime.events if event.get("event") == "cleanup.incomplete"] + self.assertEqual(incomplete[-1].get("failed_services"), {}) + self.assertEqual(incomplete[-1].get("failed_requirements"), {"run-1": ["Main"]}) + + retry_code = orchestrator.execute(Route(command=command, mode="main")) + + self.assertEqual(retry_code, 0) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-1"]) + self.assertEqual(runtime.state_repository.active_states, []) + self.assertEqual(released_requirements, ["Main"]) + self.assertEqual(runtime.state_repository.purge_calls, [aggressive]) def test_targeted_runtime_cleanup_remains_fail_closed(self) -> None: runtime = _RuntimeStub() @@ -221,6 +243,48 @@ def test_targeted_runtime_cleanup_remains_fail_closed(self) -> None: self.assertEqual(runtime.state_repository.purge_calls, []) + def test_full_cleanup_retains_state_when_requirement_port_release_fails(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-port-release", + mode="main", + requirements={ + "Main": RequirementsResult( + project="Main", + redis={"id": "redis", "enabled": True, "external": True, "final": 6379}, + ) + }, + ) + runtime.state_repository.active_states = [state] + release_attempts = 0 + + def release_ports(_requirements: RequirementsResult) -> None: + nonlocal release_attempts + release_attempts += 1 + if release_attempts == 1: + raise OSError("lock unlink failed") + + runtime._release_requirement_ports = release_ports # type: ignore[attr-defined] + orchestrator = LifecycleCleanupOrchestrator(runtime) + + first_code = orchestrator.execute(Route(command="stop-all", mode="main")) + + self.assertEqual(first_code, 1) + self.assertEqual(runtime.state_repository.active_states, [state]) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(runtime.state_repository.purge_calls, []) + self.assertEqual(set(state.requirements), {"Main"}) + warning = next(event for event in runtime.events if event.get("event") == "cleanup.requirement_ports.warning") + self.assertEqual(warning.get("detail"), "lock unlink failed") + + retry_code = orchestrator.execute(Route(command="stop-all", mode="main")) + + self.assertEqual(retry_code, 0) + self.assertEqual(release_attempts, 2) + self.assertEqual(runtime.state_repository.active_states, []) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-port-release"]) + self.assertEqual(runtime.state_repository.purge_calls, [False]) + def test_stop_all_emits_spinner_policy_and_lifecycle(self) -> None: runtime = _RuntimeStub() orchestrator = LifecycleCleanupOrchestrator(runtime) @@ -327,7 +391,7 @@ def test_stop_all_terminates_every_independent_active_run_once(self) -> None: ] terminated: list[str] = [] runtime._terminate_services_from_state = ( # type: ignore[method-assign] - lambda state, **_kwargs: terminated.append(state.run_id) + lambda state, **_kwargs: terminated.append(state.run_id) or set() ) code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="trees")) @@ -624,7 +688,7 @@ def test_targeted_stop_terminates_and_deactivates_every_shadowed_run_for_project ] terminated: list[str] = [] runtime._terminate_services_from_state = ( # type: ignore[method-assign] - lambda state, **_kwargs: terminated.append(state.run_id) + lambda state, **_kwargs: terminated.append(state.run_id) or set() ) code = LifecycleCleanupOrchestrator(runtime).execute( @@ -682,6 +746,62 @@ def test_targeted_stop_failure_preserves_service_and_run_tracking(self) -> None: self.assertEqual(runtime.state_repository.deactivated_run_ids, []) self.assertEqual(runtime.state_repository.purge_calls, []) + def test_targeted_stop_missing_termination_result_preserves_service_and_requirements(self) -> None: + runtime = _RuntimeStub() + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + pid=101, + ) + requirements = RequirementsResult(project="Main", redis={"enabled": True, "final": 6379}) + state = RunState( + run_id="run-main", + mode="main", + services={service.name: service}, + requirements={"Main": requirements}, + ) + runtime._try_load_existing_state = lambda *_args, **_kwargs: state # type: ignore[method-assign] + runtime._terminate_services_from_state = lambda *_args, **_kwargs: None # type: ignore[method-assign] + runtime._release_requirement_ports = lambda _requirements: self.fail( # type: ignore[attr-defined] + "unconfirmed service must retain requirements" + ) + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="main", + flags={"services": [service.name], "batch": True}, + ) + ) + + self.assertEqual(code, 1) + self.assertIs(state.services[service.name], service) + self.assertIs(state.requirements["Main"], requirements) + + def test_stop_all_missing_termination_result_retains_active_run(self) -> None: + runtime = _RuntimeStub() + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + pid=101, + ) + state = RunState(run_id="run-main", mode="main", services={service.name: service}) + runtime.state_repository.active_states = [state] + runtime._terminate_services_from_state = lambda *_args, **_kwargs: None # type: ignore[method-assign] + + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(code, 1) + self.assertEqual(runtime.state_repository.active_states, [state]) + self.assertEqual(set(state.services), {service.name}) + self.assertEqual(runtime.state_repository.saved_states, [state]) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(runtime.state_repository.purge_calls, []) + def test_implicit_mode_targeted_stop_cleans_every_tree_shadow_with_duplicate_pid(self) -> None: runtime = _RuntimeStub() main_state = RunState( @@ -956,7 +1076,7 @@ def test_stop_dependencies_scope_releases_requirements_without_terminating_servi runtime.port_planner = SimpleNamespace(release=lambda port: released.append(port)) runtime._terminate_services_from_state = lambda _state, **kwargs: terminated.append( # type: ignore[method-assign] kwargs.get("selected_services") - ) + ) or set() state = RunState( run_id="run-1", mode="main", @@ -980,6 +1100,184 @@ def test_stop_dependencies_scope_releases_requirements_without_terminating_servi self.assertEqual(state.requirements, {}) self.assertEqual(runtime.state_repository.saved_states[0].services.keys(), {"Main Backend"}) + def test_targeted_dependencies_scope_preserves_other_project_requirements_and_state(self) -> None: + runtime = _RuntimeStub() + released: list[str] = [] + alpha = RequirementsResult(project="Alpha", redis={"enabled": True, "final": 6379}) + beta = RequirementsResult(project="Beta", redis={"enabled": True, "final": 6389}) + state = RunState( + run_id="run-multi", + mode="trees", + services={ + "Beta Backend": ServiceRecord( + name="Beta Backend", + type="backend", + cwd="/beta", + project="Beta", + pid=2, + ) + }, + requirements={"Alpha": alpha, "Beta": beta}, + metadata={"project_names": ["Alpha", "Beta"]}, + ) + runtime.state_repository.active_states = [state] + runtime._try_load_existing_state = lambda *_args, **_kwargs: state # type: ignore[method-assign] + runtime._release_requirement_ports = ( # type: ignore[attr-defined] + lambda requirements: released.append(requirements.project) + ) + orchestrator = LifecycleCleanupOrchestrator(runtime) + + code = orchestrator.execute( + Route( + command="stop", + mode="trees", + projects=["Alpha"], + flags={"runtime_scope": "dependencies", "batch": True}, + ) + ) + + self.assertEqual(code, 0) + self.assertEqual(released, ["Alpha"]) + self.assertEqual(set(state.requirements), {"Beta"}) + self.assertIn("Beta Backend", state.services) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(runtime.state_repository.saved_states[-1].metadata["project_names"], ["Beta"]) + + def test_dependency_scope_normalizes_positional_and_service_selectors(self) -> None: + routes = ( + Route( + command="stop", + mode="trees", + passthrough_args=["Alpha"], + flags={"runtime_scope": "dependencies", "batch": True}, + ), + Route( + command="stop", + mode="trees", + flags={ + "runtime_scope": "dependencies", + "services": ["Alpha Backend"], + "batch": True, + }, + ), + ) + for route in routes: + with self.subTest(route=route): + runtime = _RuntimeStub() + state = RunState( + run_id="run-selector", + mode="trees", + services={ + "Alpha Backend": ServiceRecord( + name="Alpha Backend", type="backend", project="Alpha", cwd="/alpha", pid=1 + ), + "Beta Backend": ServiceRecord( + name="Beta Backend", type="backend", project="Beta", cwd="/beta", pid=2 + ), + }, + requirements={ + "alpha-alias": RequirementsResult(project="Alpha", redis={"enabled": True}), + "beta-alias": RequirementsResult(project="Beta", redis={"enabled": True}), + }, + metadata={"project_names": ["Alpha", "Beta"]}, + ) + runtime.state_repository.active_states = [state] + runtime._try_load_existing_state = lambda *_args, **_kwargs: state # type: ignore[method-assign] + released: list[str] = [] + runtime._release_requirement_ports = ( # type: ignore[attr-defined] + lambda requirements: released.append(requirements.project) + ) + + code = LifecycleCleanupOrchestrator(runtime).execute(route) + + self.assertEqual(code, 0) + self.assertEqual(released, ["Alpha"]) + self.assertEqual(set(state.requirements), {"beta-alias"}) + self.assertEqual(set(state.services), {"Alpha Backend", "Beta Backend"}) + + def test_unmatched_dependency_service_selector_fails_without_broadening_scope(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-selector-miss", + mode="trees", + services={ + "Alpha Backend": ServiceRecord( + name="Alpha Backend", type="backend", project="Alpha", cwd="/alpha", pid=1 + ) + }, + requirements={"Alpha": RequirementsResult(project="Alpha", redis={"enabled": True})}, + ) + runtime.state_repository.active_states = [state] + runtime._try_load_existing_state = lambda *_args, **_kwargs: state # type: ignore[method-assign] + runtime._release_requirement_ports = lambda _requirements: self.fail("selector miss released ports") # type: ignore[attr-defined] + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + flags={"runtime_scope": "dependencies", "services": ["Missing Worker"], "batch": True}, + ) + ) + + self.assertEqual(code, 1) + self.assertEqual(set(state.requirements), {"Alpha"}) + self.assertEqual(runtime.state_repository.saved_states, []) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + + def test_entire_system_selector_does_not_remove_sibling_project_dependencies(self) -> None: + routes = ( + Route( + command="stop", + mode="trees", + passthrough_args=["Alpha"], + flags={"runtime_scope": "entire-system", "batch": True}, + ), + Route( + command="stop", + mode="trees", + flags={ + "runtime_scope": "entire-system", + "services": ["Alpha Backend"], + "batch": True, + }, + ), + ) + for route in routes: + with self.subTest(route=route): + runtime = _RuntimeStub() + released: list[int] = [] + runtime.port_planner = SimpleNamespace(release=lambda port: released.append(port)) + state = RunState( + run_id="run-entire-selector", + mode="trees", + services={ + "Alpha Backend": ServiceRecord( + name="Alpha Backend", type="backend", project="Alpha", cwd="/alpha", pid=1 + ), + "Beta Backend": ServiceRecord( + name="Beta Backend", type="backend", project="Beta", cwd="/beta", pid=2 + ), + }, + requirements={ + "alpha-alias": RequirementsResult( + project="Alpha", redis={"id": "redis", "enabled": True, "final": 6379} + ), + "beta-alias": RequirementsResult( + project="Beta", redis={"id": "redis", "enabled": True, "final": 6389} + ), + }, + metadata={"project_names": ["Alpha", "Beta"]}, + ) + runtime.state_repository.active_states = [state] + runtime._try_load_existing_state = lambda *_args, **_kwargs: state # type: ignore[method-assign] + + code = LifecycleCleanupOrchestrator(runtime).execute(route) + + self.assertEqual(code, 0) + self.assertEqual(released, [6379]) + self.assertEqual(set(state.services), {"Beta Backend"}) + self.assertEqual(set(state.requirements), {"beta-alias"}) + def test_stop_selected_dependency_component_preserves_services_and_other_dependencies(self) -> None: runtime = _RuntimeStub() released: list[int] = [] @@ -987,7 +1285,7 @@ def test_stop_selected_dependency_component_preserves_services_and_other_depende runtime.port_planner = SimpleNamespace(release=lambda port: released.append(port)) runtime._terminate_services_from_state = lambda _state, **kwargs: terminated.append( # type: ignore[method-assign] kwargs.get("selected_services") - ) + ) or set() state = RunState( run_id="run-1", mode="main", @@ -1032,7 +1330,7 @@ def test_stop_selected_services_can_leave_dependencies_running(self) -> None: runtime.port_planner = SimpleNamespace(release=lambda port: released.append(port)) runtime._terminate_services_from_state = lambda _state, **kwargs: terminated.append( # type: ignore[method-assign] kwargs.get("selected_services") - ) + ) or set() state = RunState( run_id="run-1", mode="main", @@ -1074,7 +1372,7 @@ def test_interactive_stop_all_services_preserves_stopped_dashboard_state(self) - terminated: list[set[str] | None] = [] runtime._terminate_services_from_state = lambda _state, **kwargs: terminated.append( # type: ignore[method-assign] kwargs.get("selected_services") - ) + ) or set() state = RunState( run_id="run-1", mode="main", diff --git a/tests/python/runtime/test_lifecycle_dependency_stop.py b/tests/python/runtime/test_lifecycle_dependency_stop.py index 590f941a6..ac2dcdb91 100644 --- a/tests/python/runtime/test_lifecycle_dependency_stop.py +++ b/tests/python/runtime/test_lifecycle_dependency_stop.py @@ -32,6 +32,26 @@ def test_entire_system_selects_all_enabled_dependencies_for_requested_project(se self.assertEqual(select_dependency_components_for_stop(state, route), {"Alpha": {"redis", "n8n"}}) + def test_entire_system_selects_every_storage_record_by_authoritative_project(self) -> None: + state = RunState( + run_id="run-collision", + mode="main", + requirements={ + "Main": RequirementsResult(project="Main", redis={"enabled": True}), + "Main Restart Collision": RequirementsResult(project="Main", redis={"enabled": True}), + "Other": RequirementsResult(project="Other", redis={"enabled": True}), + }, + ) + route = Route(command="stop", mode="main", projects=["Main"], flags={"runtime_scope": "entire-system"}) + + selected = select_dependency_components_for_stop(state, route) + + self.assertEqual(selected, {"Main": {"redis"}, "Main Restart Collision": {"redis"}}) + self.assertEqual( + requirement_component_port_owners(state.requirements["Main Restart Collision"], "redis"), + ("Main:redis", "Main:requirements"), + ) + def test_stop_requirement_component_removes_direct_and_compose_containers(self) -> None: calls: list[list[str]] = [] diff --git a/tests/python/runtime/test_lifecycle_parity_restore_startup.py b/tests/python/runtime/test_lifecycle_parity_restore_startup.py index 4d7517a51..aeeeee829 100644 --- a/tests/python/runtime/test_lifecycle_parity_restore_startup.py +++ b/tests/python/runtime/test_lifecycle_parity_restore_startup.py @@ -167,7 +167,7 @@ def test_resume_restore_does_not_reuse_requirements_when_project_root_reveals_ow "ENVCTL_DEBUG_RESTORE_TIMING": "true", }, ) - restore_runner = _ResumeRestoreRunner() + restore_runner = _ResumeRestoreRunner(listeners_ready=True) engine.process_runner = restore_runner # type: ignore[assignment] def fake_reconcile(project: str, requirements: RequirementsResult, *, project_root: Path | None = None): # noqa: ANN001 diff --git a/tests/python/runtime/test_lifecycle_parity_resume_policy.py b/tests/python/runtime/test_lifecycle_parity_resume_policy.py index ef6e7bd13..f08802cd1 100644 --- a/tests/python/runtime/test_lifecycle_parity_resume_policy.py +++ b/tests/python/runtime/test_lifecycle_parity_resume_policy.py @@ -174,6 +174,8 @@ def test_resume_reconciles_missing_service_status(self) -> None: runtime = Path(tmpdir) / "runtime" run_dir = runtime / "python-engine" (repo / ".git").mkdir(parents=True, exist_ok=True) + (repo / "backend").mkdir(parents=True, exist_ok=True) + (repo / "frontend").mkdir(parents=True, exist_ok=True) run_dir.mkdir(parents=True, exist_ok=True) state = RunState( @@ -199,7 +201,15 @@ def test_resume_reconciles_missing_service_status(self) -> None: "RUN_SH_RUNTIME_DIR": str(runtime), } ) - engine = PythonEngineRuntime(config, env={}) + engine = PythonEngineRuntime( + config, + env={ + "ENVCTL_BACKEND_START_CMD": "sh -lc 'sleep 1'", + "ENVCTL_FRONTEND_START_CMD": "sh -lc 'sleep 1'", + }, + ) + restore_runner = _ResumeRestoreRunner(listeners_ready=True) + engine.process_runner = restore_runner # type: ignore[assignment] code = engine.dispatch(parse_route(["--resume"], env={})) self.assertEqual(code, 0) diff --git a/tests/python/runtime/test_lifecycle_requirement_ports.py b/tests/python/runtime/test_lifecycle_requirement_ports.py index 34a67a0a7..c4977906a 100644 --- a/tests/python/runtime/test_lifecycle_requirement_ports.py +++ b/tests/python/runtime/test_lifecycle_requirement_ports.py @@ -91,6 +91,48 @@ def test_release_requirement_ports_can_release_verified_prior_session_owners(sel self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + def test_collision_storage_records_release_ports_with_shared_authoritative_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(6380, owner="Main:requirements") + prior.reserve_next(6381, owner="Main:requirements") + state = RunState( + run_id="run-collision", + mode="main", + requirements={ + "Main": RequirementsResult( + project="Main", + redis={ + "enabled": True, + "final": 6380, + "port_lock_session": "prior-session", + }, + ), + "Main Restart Collision": RequirementsResult( + project="Main", + redis={ + "enabled": True, + "final": 6381, + "port_lock_session": "prior-session", + }, + ), + }, + ) + + for requirements in state.requirements.values(): + release_requirement_ports(SimpleNamespace(port_planner=cleanup), requirements) + + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + def test_stale_requirement_state_cannot_release_newer_live_same_owner_lock(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: newer = PortPlanner( diff --git a/tests/python/runtime/test_lifecycle_service_stop_selection.py b/tests/python/runtime/test_lifecycle_service_stop_selection.py index 2e0154388..28db7929f 100644 --- a/tests/python/runtime/test_lifecycle_service_stop_selection.py +++ b/tests/python/runtime/test_lifecycle_service_stop_selection.py @@ -72,6 +72,45 @@ def test_runtime_scope_intersects_explicit_project_selector(self) -> None: self.assertEqual(selected, {"FeatureA Backend"}) + def test_entire_system_project_stop_includes_additional_services(self) -> None: + state = RunState( + run_id="run-1", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + pid=1, + project="Main", + ), + "Main Optional Bad": ServiceRecord( + name="Main Optional Bad", + type="optional-bad", + cwd=".", + pid=2, + project="Main", + service_slug="optional-bad", + ), + }, + ) + + selected = select_services_for_stop( + state, + Route( + command="stop", + mode="main", + projects=["Main"], + flags={"runtime_scope": "entire-system"}, + ), + project_name_from_service_fn=lambda name: name.rsplit(" ", 1)[0], + selectors_from_passthrough_fn=lambda _args: set(), + interactive_stop_selection_fn=lambda _route, _state: self.fail("interactive prompt should not run"), + services_from_selection_fn=lambda _selection, _state: self.fail("selection projection should not run"), + ) + + self.assertEqual(selected, {"Main Backend", "Main Optional Bad"}) + def test_service_and_project_selectors_match_slug_display_name_and_passthrough_project(self) -> None: state = RunState( run_id="run-1", diff --git a/tests/python/runtime/test_lifecycle_service_termination.py b/tests/python/runtime/test_lifecycle_service_termination.py index 6438d2d2d..19c2f24f7 100644 --- a/tests/python/runtime/test_lifecycle_service_termination.py +++ b/tests/python/runtime/test_lifecycle_service_termination.py @@ -12,16 +12,69 @@ from envctl_engine.runtime.lifecycle_service_termination import ( _wait_for_pid_exit, + failed_listener_pids, service_port, terminate_service_record, terminate_services_from_state, terminate_started_services, + unconfirmed_service_names, ) from envctl_engine.state.models import RunState, ServiceRecord from envctl_engine.shared.ports import PortPlanner class LifecycleServiceTerminationTests(unittest.TestCase): + def test_pending_docker_cleanup_timestamp_reaches_tokenized_stop(self) -> None: + runtime = SimpleNamespace( + process_runner=SimpleNamespace(), + _emit=lambda *_args, **_kwargs: None, + ) + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + runtime_kind="docker", + container_id="envctl-app-main-backend", + container_name="envctl-app-main-backend", + container_launch_token="pending-token", + container_cleanup_pending_since=1_700_000_000.25, + ) + + with patch( + "envctl_engine.runtime.docker_service_runtime.DockerServiceRuntime.stop", + return_value=False, + ) as stop: + self.assertFalse( + terminate_service_record( + runtime, + service, + aggressive=False, + verify_ownership=True, + ) + ) + + stop.assert_called_once_with( + "envctl-app-main-backend", + verify_ownership=True, + expected_launch_token="pending-token", + pending_cleanup_since=1_700_000_000.25, + ) + + def test_termination_result_normalization_fails_closed_for_missing_or_unknown_identities(self) -> None: + requested = {"Main Backend", "Main Frontend"} + + self.assertEqual(unconfirmed_service_names(None, requested), requested) + self.assertEqual(unconfirmed_service_names("Main Backend", requested), requested) + self.assertEqual(unconfirmed_service_names({"unknown"}, requested), requested) + self.assertEqual(unconfirmed_service_names([" "], requested), requested) + self.assertEqual(unconfirmed_service_names(["", "Main Backend"], requested), requested) + self.assertEqual(unconfirmed_service_names({"Main Backend"}, requested), {"Main Backend"}) + self.assertEqual(unconfirmed_service_names([], requested), set()) + self.assertEqual(failed_listener_pids(set()), set()) + self.assertEqual(failed_listener_pids({4242}), {4242}) + self.assertIsNone(failed_listener_pids(None)) + self.assertIsNone(failed_listener_pids({"4242"})) + def test_terminate_started_services_uses_non_aggressive_unverified_shutdown(self) -> None: calls: list[tuple[int, bool, bool]] = [] runtime = SimpleNamespace( @@ -158,6 +211,52 @@ def test_terminated_service_releases_verified_prior_session_port_owner(self) -> self.assertEqual(failed, set()) self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + def test_legacy_collision_name_releases_original_project_owner_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(8001, owner="Main:backend") + collision_name = "Main Backend Restart Collision 2" + state = RunState( + run_id="run-collision", + mode="main", + services={ + collision_name: ServiceRecord( + name=collision_name, + type="backend", + cwd=".", + pid=2, + actual_port=8001, + port_lock_session="prior-session", + project=None, + ) + }, + ) + runtime = SimpleNamespace( + port_planner=current, + _terminate_service_record=lambda _service, **_kwargs: True, + _project_name_from_service=lambda _name: "Main Backend Restart Collision", + ) + + failed = terminate_services_from_state( + runtime, + state, + selected_services=None, + aggressive=False, + verify_ownership=True, + ) + + self.assertEqual(failed, set()) + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + def test_terminated_rebound_frontend_releases_launch_owner_lock(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: prior = PortPlanner( diff --git a/tests/python/runtime/test_lifecycle_worktree_cleanup.py b/tests/python/runtime/test_lifecycle_worktree_cleanup.py index 8498d5f1e..ed7d6797a 100644 --- a/tests/python/runtime/test_lifecycle_worktree_cleanup.py +++ b/tests/python/runtime/test_lifecycle_worktree_cleanup.py @@ -155,7 +155,7 @@ def test_blast_worktree_before_delete_updates_matching_project_state(self) -> No _emit=lambda event, **payload: events.append((event, payload)), _try_load_existing_state=lambda **kwargs: state if kwargs["mode"] == "trees" else None, _project_name_from_service=lambda name: "Feature" if "Feature" in name else "", - _terminate_services_from_state=lambda *args, **kwargs: None, + _terminate_services_from_state=lambda *args, **kwargs: set(), port_planner=SimpleNamespace(release=lambda port: released.append(port)), state_repository=SimpleNamespace( save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) @@ -177,6 +177,143 @@ def test_blast_worktree_before_delete_updates_matching_project_state(self) -> No self.assertEqual(saved_states[0].requirements, {}) self.assertEqual(events[-1][0], "cleanup.worktree.finish") + def test_worktree_cleanup_releases_every_requirement_storage_row_for_project(self) -> None: + saved_states: list[RunState] = [] + released: list[int] = [] + state = RunState( + run_id="run-collision", + mode="trees", + requirements={ + "Feature": RequirementsResult(project="Feature", redis={"enabled": True, "final": 6379}), + "Feature Restart Collision": RequirementsResult( + project="Feature", + redis={"enabled": True, "final": 6381}, + ), + "Sibling": RequirementsResult(project="Sibling", redis={"enabled": True, "final": 6391}), + }, + metadata={"project_names": ["Feature", "Sibling"]}, + ) + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _try_load_existing_state=lambda **kwargs: state if kwargs["mode"] == "trees" else None, + _project_name_from_service=lambda _name: "", + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + port_planner=SimpleNamespace(release=lambda port: released.append(port)), + state_repository=SimpleNamespace( + save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) + ), + process_runner=SimpleNamespace(), + env={}, + ) + + warnings = blast_worktree_before_delete( + runtime, + project_name="Feature", + project_root=Path("/repo/feature"), + source_command="delete-worktree", + ) + + self.assertEqual(warnings, []) + self.assertEqual(released, [6379, 6381]) + self.assertEqual(set(saved_states[0].requirements), {"Sibling"}) + + def test_worktree_cleanup_stops_saved_custom_requirement_container_before_dropping_state(self) -> None: + calls: list[list[str]] = [] + + class Runner: + def run(self, command: list[str], **_kwargs: object) -> SimpleNamespace: + calls.append(command) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + state = RunState( + run_id="run-container-collision", + mode="trees", + requirements={ + "Feature Restart Collision": RequirementsResult( + project="Feature", + redis={ + "enabled": True, + "success": True, + "final": 6381, + "container_name": "custom-collision-redis", + }, + ) + }, + ) + saved_states: list[RunState] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _try_load_existing_state=lambda **kwargs: state if kwargs["mode"] == "trees" else None, + _project_name_from_service=lambda _name: "", + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + port_planner=SimpleNamespace(release=lambda _port: None), + state_repository=SimpleNamespace( + save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) + ), + process_runner=Runner(), + config=SimpleNamespace(additional_services=()), + env={}, + ) + + warnings = blast_worktree_before_delete( + runtime, + project_name="Feature", + project_root=Path("/repo/feature"), + source_command="delete-worktree", + ) + + self.assertEqual(warnings, []) + self.assertIn(["docker", "rm", "--force", "custom-collision-redis"], calls) + self.assertEqual(saved_states[0].requirements, {}) + + def test_worktree_cleanup_uses_multiword_additional_service_project_metadata(self) -> None: + terminated: list[set[str]] = [] + saved_states: list[RunState] = [] + service = ServiceRecord( + name="Opaque Runtime Process", + type="voice-runtime", + cwd="/repo/customer-platform/voice", + project="Customer Platform", + actual_port=8010, + ) + legacy_service = ServiceRecord( + name="Customer Platform Voice Runtime", + type="voice-runtime", + cwd="/repo/customer-platform/voice-legacy", + project=None, + service_slug="voice-runtime", + actual_port=8011, + ) + state = RunState( + run_id="run-metadata-project", + mode="trees", + services={service.name: service, legacy_service.name: legacy_service}, + ) + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _try_load_existing_state=lambda **kwargs: state if kwargs["mode"] == "trees" else None, + _project_name_from_service=lambda _name: "Opaque Runtime", + _terminate_services_from_state=lambda _state, *, selected_services, **_kwargs: ( + terminated.append(set(selected_services)) or set() + ), + state_repository=SimpleNamespace( + save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) + ), + process_runner=SimpleNamespace(), + env={}, + ) + + warnings = blast_worktree_before_delete( + runtime, + project_name="Customer Platform", + project_root=Path("/repo/customer-platform"), + source_command="delete-worktree", + ) + + self.assertEqual(warnings, []) + self.assertEqual(terminated, [{service.name, legacy_service.name}]) + self.assertEqual(saved_states[0].services, {}) + def test_blast_worktree_before_delete_uses_injected_cleanup_dependencies(self) -> None: calls: list[str] = [] state = RunState(run_id="run-1", mode="trees") diff --git a/tests/python/runtime/test_pr_preview_controller.py b/tests/python/runtime/test_pr_preview_controller.py index 9918d31fe..e617a6654 100644 --- a/tests/python/runtime/test_pr_preview_controller.py +++ b/tests/python/runtime/test_pr_preview_controller.py @@ -1,5 +1,6 @@ from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime import json from pathlib import Path @@ -2160,6 +2161,117 @@ def test_start_keeps_backend_venv_created_with_same_python(tmp_path): assert venv.exists() +def test_preview_python_command_shim_supports_legacy_python_token(tmp_path): + controller = load_controller() + env = { + "PYTHON_BIN": sys.executable, + "PATH": "/usr/bin:/bin", + } + + shim = controller.prepare_python_command_shim( + env, + runtime_root=tmp_path / "runtime", + ) + first_target = shim.resolve(strict=True) + second = controller.prepare_python_command_shim( + env, + runtime_root=tmp_path / "runtime", + ) + result = subprocess.run( + ["python", "-c", "print('legacy-python-ok')"], + check=False, + capture_output=True, + text=True, + env=env, + ) + + assert shim == second + assert first_target == Path(sys.executable).resolve() + assert second.resolve(strict=True) == Path(sys.executable).resolve() + assert Path(env["PATH"].split(controller.os.pathsep, 1)[0]) == shim.parent + assert env["ENVCTL_PYTHON_EXECUTABLE"] == str(Path(sys.executable).resolve()) + assert result.returncode == 0 + assert result.stdout.strip() == "legacy-python-ok" + + +def test_preview_python_command_shims_are_isolated_by_interpreter(tmp_path): + controller = load_controller() + + def make_interpreter(index: int) -> Path: + executable = tmp_path / f"python-{index}" + executable.write_text( + "#!/bin/sh\n" + "case \"$2\" in\n" + " *sys.version_info*) printf '3.13\\n' ;;\n" + f" *) printf 'interpreter-{index}\\n' ;;\n" + "esac\n", + encoding="utf-8", + ) + executable.chmod(0o755) + return executable + + interpreters = [make_interpreter(index) for index in range(16)] + + def prepare_and_run(interpreter: Path) -> tuple[Path, str]: + env = {"PYTHON_BIN": str(interpreter), "PATH": "/usr/bin:/bin"} + shim = controller.prepare_python_command_shim( + env, + runtime_root=tmp_path / "runtime", + ) + result = subprocess.run( + ["python", "-c", "print('selected')"], + check=False, + capture_output=True, + text=True, + env=env, + ) + assert result.returncode == 0 + return shim, result.stdout.strip() + + with ThreadPoolExecutor(max_workers=len(interpreters)) as executor: + results = list(executor.map(prepare_and_run, interpreters)) + + assert len({shim.parent for shim, _output in results}) == len(interpreters) + assert {output for _shim, output in results} == { + f"interpreter-{index}" for index in range(len(interpreters)) + } + assert list((tmp_path / "runtime").rglob(".python-*")) == [] + + reused_env = {"PYTHON_BIN": str(interpreters[0]), "PATH": "/usr/bin:/bin"} + controller.prepare_python_command_shim(reused_env, runtime_root=tmp_path / "runtime") + reused_env["PYTHON_BIN"] = str(interpreters[1]) + latest = controller.prepare_python_command_shim(reused_env, runtime_root=tmp_path / "runtime") + managed_path_entries = [ + Path(part) + for part in reused_env["PATH"].split(controller.os.pathsep) + if Path(part).parent == tmp_path / "runtime" / "python-command-shims" + ] + assert managed_path_entries == [latest.parent] + + +def test_preview_python_command_shim_rejects_symlinked_runtime_directory(tmp_path): + controller = load_controller() + runtime_root = tmp_path / "runtime" + runtime_root.mkdir() + external = tmp_path / "external" + external.mkdir() + (runtime_root / "python-command-shims").symlink_to(external, target_is_directory=True) + env = {"PYTHON_BIN": sys.executable, "PATH": "/usr/bin:/bin"} + + with pytest.raises(RuntimeError, match="not a real directory"): + controller.prepare_python_command_shim(env, runtime_root=runtime_root) + + assert list(external.iterdir()) == [] + + +def test_preview_python_command_shim_rejects_non_python_executable(tmp_path): + controller = load_controller() + env = {"PYTHON_BIN": "/bin/sh", "PATH": "/usr/bin:/bin"} + + with pytest.raises(RuntimeError, match="not Python 3"): + controller.prepare_python_command_shim(env, runtime_root=tmp_path / "runtime") + + def test_unlabeled_event_stops_tracked_preview(tmp_path): controller = load_controller() config = make_config(controller, tmp_path) diff --git a/tests/python/runtime/test_runtime_browser_diagnostics.py b/tests/python/runtime/test_runtime_browser_diagnostics.py index e001c78fe..7a80f5626 100644 --- a/tests/python/runtime/test_runtime_browser_diagnostics.py +++ b/tests/python/runtime/test_runtime_browser_diagnostics.py @@ -11,6 +11,7 @@ from envctl_engine.config import DependencyEnvTemplateEntry, load_config from envctl_engine.runtime.command_router import parse_route from envctl_engine.runtime.endpoints_command_support import build_endpoints_payload +from envctl_engine.runtime.browser_runtime_diagnostics import project_names from envctl_engine.runtime.engine_runtime import PythonEngineRuntime from envctl_engine.startup.finalization import build_success_run_state from envctl_engine.startup.session import StartupSession @@ -43,6 +44,18 @@ def pid_owns_port(self, _pid: int, _port: int) -> bool: class RuntimeBrowserDiagnosticsTests(unittest.TestCase): + def test_requirement_collision_storage_alias_is_not_exposed_as_browser_project(self) -> None: + state = RunState( + run_id="run-collision", + mode="main", + requirements={ + "Main": RequirementsResult(project="Main"), + "Main Restart Collision": RequirementsResult(project="Main"), + }, + ) + + self.assertEqual(project_names(state, runtime=None), ["Main"]) + def _runtime_with_state(self, state: RunState, *, env: dict[str, str] | None = None) -> PythonEngineRuntime: tmpdir = tempfile.TemporaryDirectory() self.addCleanup(tmpdir.cleanup) diff --git a/tests/python/shared/test_ports_lock_reclamation.py b/tests/python/shared/test_ports_lock_reclamation.py index af200b89f..00e9a605e 100644 --- a/tests/python/shared/test_ports_lock_reclamation.py +++ b/tests/python/shared/test_ports_lock_reclamation.py @@ -39,7 +39,7 @@ def test_stale_lock_is_reclaimed(self) -> None: lock_path.write_text( json.dumps( { - "owner": "old-worker", + "owner": "worker", "session": "old-session", "pid": 999999, "created_at": "2000-01-01T00:00:00+00:00", @@ -54,11 +54,11 @@ def test_stale_lock_is_reclaimed(self) -> None: session_id="new-session", availability_checker=lambda _port: True, ) - reserved = planner.reserve_next(5432, owner="new-worker") + reserved = planner.reserve_next(5432, owner="worker") self.assertEqual(reserved, 5432) payload = json.loads(lock_path.read_text(encoding="utf-8")) - self.assertEqual(payload["owner"], "new-worker") + self.assertEqual(payload["owner"], "worker") self.assertEqual(payload["session"], "new-session") def test_stale_lock_reclaim_event_includes_previous_owner_metadata(self) -> None: @@ -67,7 +67,7 @@ def test_stale_lock_reclaim_event_includes_previous_owner_metadata(self) -> None lock_path.write_text( json.dumps( { - "owner": "old-worker", + "owner": "worker", "session": "old-session", "pid": 999999, "created_at": "2000-01-01T00:00:00+00:00", @@ -88,18 +88,66 @@ def on_event(name: str, payload: dict[str, object]) -> None: availability_checker=lambda _port: True, event_handler=on_event, ) - reserved = planner.reserve_next(5432, owner="new-worker") + reserved = planner.reserve_next(5432, owner="worker") self.assertEqual(reserved, 5432) reclaim_events = [payload for name, payload in events if name == "port.lock.reclaim"] self.assertTrue(reclaim_events) latest = reclaim_events[-1] self.assertEqual(latest.get("port"), 5432) - self.assertEqual(latest.get("owner"), "new-worker") - self.assertEqual(latest.get("reclaimed_owner"), "old-worker") + self.assertEqual(latest.get("owner"), "worker") + self.assertEqual(latest.get("reclaimed_owner"), "worker") self.assertEqual(latest.get("reclaimed_session"), "old-session") self.assertEqual(latest.get("reclaimed_pid"), 999999) + def test_dead_foreign_owner_lock_is_preserved_and_reservation_rebounds(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "5432.lock" + original = json.dumps( + { + "owner": "Other:backend", + "session": "historical-session", + "pid": 99999999, + "created_at": 0, + }, + sort_keys=True, + ) + lock_path.write_text(original, encoding="utf-8") + planner = PortPlanner( + lock_dir=tmpdir, + session_id="selected-session", + availability_checker=lambda _port: True, + ) + + reserved = planner.reserve_next(5432, owner="Selected:backend") + + self.assertEqual(reserved, 5433) + self.assertEqual(lock_path.read_text(encoding="utf-8"), original) + + def test_dead_allocator_lock_is_preserved_when_same_owner_service_still_listens(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "5432.lock" + original = json.dumps( + { + "owner": "Main:backend", + "session": "historical-session", + "pid": 99999999, + "created_at": 0, + }, + sort_keys=True, + ) + lock_path.write_text(original, encoding="utf-8") + planner = PortPlanner( + lock_dir=tmpdir, + session_id="replacement-session", + availability_checker=lambda port: port != 5432, + ) + + reserved = planner.reserve_next(5432, owner="Main:backend") + + self.assertEqual(reserved, 5433) + self.assertEqual(lock_path.read_text(encoding="utf-8"), original) + def test_active_pid_lock_is_not_reclaimed_by_age_alone(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: lock_path = Path(tmpdir) / "5432.lock" @@ -206,7 +254,7 @@ def test_concurrent_stale_reclaim_is_serialized(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: lock_path = Path(tmpdir) / "13000.lock" lock_path.write_text( - json.dumps({"owner": "dead", "session": "dead", "pid": 99999999, "created_at": 0}), + json.dumps({"owner": "worker-0", "session": "dead", "pid": 99999999, "created_at": 0}), encoding="utf-8", ) worker_count = 32 @@ -257,7 +305,7 @@ def inspected_link(source: str | Path, destination: str | Path) -> None: self.assertEqual(list((Path(tmpdir) / ".port-guards").glob("*.pending")), []) fsync_directory.assert_called_once_with(Path(tmpdir)) - def test_corrupt_lock_gets_a_grace_period_then_becomes_stale(self) -> None: + def test_corrupt_ownerless_lock_requires_explicit_reap_after_grace_period(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: now = [1000.0] lock_path = Path(tmpdir) / "15000.lock" @@ -274,6 +322,9 @@ def test_corrupt_lock_gets_a_grace_period_then_becomes_stale(self) -> None: self.assertEqual(planner.reserve_next(15000, owner="new"), 15001) planner.release_session() now[0] += 3.0 + self.assertEqual(planner.reserve_next(15000, owner="new"), 15001) + planner.release_session() + self.assertTrue(planner.reap_stale(15000)) self.assertEqual(planner.reserve_next(15000, owner="new"), 15000) def test_permission_error_means_lock_owner_is_still_live(self) -> None: @@ -308,7 +359,7 @@ def test_boolean_pid_is_corrupt_data_not_a_live_pid_one_reservation(self) -> Non corrupt_lock_grace_seconds=0.0, ) - self.assertEqual(planner.reserve_next(16010, owner="new"), 16010) + self.assertEqual(planner.reserve_next(16010, owner="broken"), 16010) def test_failed_pid_probe_preserves_a_foreign_lock(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -342,7 +393,7 @@ def test_non_finite_corrupt_timestamp_cannot_pin_a_port_forever(self) -> None: corrupt_lock_grace_seconds=0.0, ) - self.assertEqual(planner.reserve_next(16030, owner="new"), 16030) + self.assertEqual(planner.reserve_next(16030, owner="broken"), 16030) def test_normal_release_is_session_qualified_but_release_all_is_explicitly_scoped(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: diff --git a/tests/python/shared/test_process_runner_launch_policy.py b/tests/python/shared/test_process_runner_launch_policy.py index 216cd46b9..85c50ca03 100644 --- a/tests/python/shared/test_process_runner_launch_policy.py +++ b/tests/python/shared/test_process_runner_launch_policy.py @@ -1,7 +1,11 @@ from __future__ import annotations +import os +import signal import subprocess +import sys import tempfile +import time import unittest from pathlib import Path from types import SimpleNamespace @@ -10,10 +14,313 @@ REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" -from envctl_engine.shared.process_runner import ProcessRunner +from envctl_engine.shared.process_runner import ProcessHandoffCleanupError, ProcessRunner + + +def _force_process_cleanup(process: subprocess.Popen[str] | None) -> None: + if process is None: + return + if process.poll() is None: + try: + os.killpg(process.pid, signal.SIGKILL) + except OSError: + try: + process.kill() + except OSError: + pass + try: + process.wait(timeout=5.0) + except (ChildProcessError, ProcessLookupError, subprocess.TimeoutExpired): + pass + + +def _wait_for_path(path: Path, *, timeout: float = 5.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if path.is_file(): + return + time.sleep(0.01) + raise AssertionError(f"child readiness marker was not created: {path}") + + +def _sigterm_ignoring_command(ready_path: Path) -> list[str]: + source = ( + "import pathlib, signal, time; " + "signal.signal(signal.SIGTERM, signal.SIG_IGN); " + f"pathlib.Path({str(ready_path)!r}).write_text('ready', encoding='utf-8'); " + "time.sleep(60)" + ) + return [sys.executable, "-c", source] class ProcessRunnerLaunchPolicyTests(unittest.TestCase): + def test_unconfirmed_background_cleanup_preserves_pid_authority(self) -> None: + class UnkillableProcess: + pid = 987_654 + returncode = None + + def poll(self) -> None: + return None + + def wait(self, timeout: float | None = None) -> int: + raise subprocess.TimeoutExpired(cmd=["fake-service"], timeout=timeout or 0.0) + + def terminate(self) -> None: + raise PermissionError("terminate denied") + + def kill(self) -> None: + raise PermissionError("kill denied") + + process = UnkillableProcess() + runner = ProcessRunner(emit=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("emit failed"))) + + with ( + patch( + "envctl_engine.shared.process_launch_support.subprocess.Popen", + return_value=process, + ), + patch( + "envctl_engine.shared.process_launch_support.os.killpg", + side_effect=PermissionError("killpg denied"), + ), + self.assertRaises(ProcessHandoffCleanupError) as raised, + ): + runner.start_background(["fake-service"]) + + self.assertIs(raised.exception.process, process) + self.assertEqual(raised.exception.pid, process.pid) + self.assertIsInstance(raised.exception.__cause__, RuntimeError) + self.assertTrue(runner._launch_records[-1].active) # noqa: SLF001 + with patch.object(runner, "is_pid_running", return_value=True): + self.assertEqual(runner.launch_diagnostics_summary()["active_launch_count"], 1) + + def test_unconfirmed_run_cleanup_preserves_pid_authority(self) -> None: + class UnkillableProcess: + pid = 987_655 + returncode = None + + def communicate(self, timeout: float | None = None) -> tuple[str, str]: + _ = timeout + raise KeyboardInterrupt + + def poll(self) -> None: + return None + + def wait(self, timeout: float | None = None) -> int: + raise subprocess.TimeoutExpired(cmd=["fake-command"], timeout=timeout or 0.0) + + def terminate(self) -> None: + raise PermissionError("terminate denied") + + def kill(self) -> None: + raise PermissionError("kill denied") + + process = UnkillableProcess() + runner = ProcessRunner() + + with ( + patch( + "envctl_engine.shared.process_runner.subprocess.Popen", + return_value=process, + ), + patch( + "envctl_engine.shared.process_launch_support.os.killpg", + side_effect=PermissionError("killpg denied"), + ), + self.assertRaises(ProcessHandoffCleanupError) as raised, + ): + runner.run(["fake-command"]) + + self.assertIs(raised.exception.process, process) + self.assertEqual(raised.exception.pid, process.pid) + self.assertIsInstance(raised.exception.__cause__, KeyboardInterrupt) + record = runner._launch_records[-1] # noqa: SLF001 + self.assertEqual(record.launch_intent, "foreground_cleanup_unconfirmed") + self.assertTrue(record.active) + with patch.object(runner, "is_pid_running", return_value=True): + self.assertEqual(runner.launch_diagnostics_summary()["active_launch_count"], 1) + + def test_unconfirmed_timeout_cleanup_never_returns_completed_result(self) -> None: + class UnkillableTimedOutProcess: + pid = 987_656 + returncode = None + + def communicate(self, timeout: float | None = None) -> tuple[str, str]: + raise subprocess.TimeoutExpired(cmd=["fake-command"], timeout=timeout or 0.0) + + def poll(self) -> None: + return None + + def wait(self, timeout: float | None = None) -> int: + raise subprocess.TimeoutExpired(cmd=["fake-command"], timeout=timeout or 0.0) + + def terminate(self) -> None: + raise PermissionError("terminate denied") + + def kill(self) -> None: + raise PermissionError("kill denied") + + process = UnkillableTimedOutProcess() + runner = ProcessRunner() + + with ( + patch( + "envctl_engine.shared.process_runner.subprocess.Popen", + return_value=process, + ), + patch( + "envctl_engine.shared.process_runner.os.killpg", + side_effect=PermissionError("killpg denied"), + ), + patch( + "envctl_engine.shared.process_launch_support.os.killpg", + side_effect=PermissionError("killpg denied"), + ), + self.assertRaises(ProcessHandoffCleanupError) as raised, + ): + runner.run(["fake-command"], timeout=0.01) + + self.assertIs(raised.exception.process, process) + self.assertEqual(raised.exception.pid, process.pid) + self.assertIsInstance(raised.exception.__cause__, subprocess.TimeoutExpired) + self.assertEqual( + runner._launch_records[-1].launch_intent, # noqa: SLF001 + "foreground_cleanup_unconfirmed", + ) + + @unittest.skipUnless(hasattr(os, "killpg"), "requires POSIX process groups") + def test_interactive_emit_failure_terminates_real_child_before_reraising(self) -> None: + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[str]] = [] + + with tempfile.TemporaryDirectory() as tmpdir: + ready_path = Path(tmpdir) / "ready" + command = _sigterm_ignoring_command(ready_path) + + def capture_popen(*args, **kwargs): # noqa: ANN002, ANN003, ANN202 + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + def fail_emit(_event: str, **_payload: object) -> None: + _wait_for_path(ready_path) + raise RuntimeError("emit interrupted") + + runner = ProcessRunner(emit=fail_emit) + try: + with ( + patch( + "envctl_engine.shared.process_launch_support.subprocess.Popen", + side_effect=capture_popen, + ), + self.assertRaisesRegex(RuntimeError, "emit interrupted"), + ): + runner.start_interactive_child(command) + + self.assertEqual(len(spawned), 1) + self.assertEqual(spawned[0].poll(), -signal.SIGKILL) + finally: + _force_process_cleanup(spawned[0] if spawned else None) + + @unittest.skipUnless(hasattr(os, "killpg"), "requires POSIX process groups") + def test_start_background_emit_failures_terminate_real_child_before_reraising(self) -> None: + real_popen = subprocess.Popen + + for failure_type in (RuntimeError, KeyboardInterrupt): + with self.subTest(failure_type=failure_type.__name__): + with tempfile.TemporaryDirectory() as tmpdir: + ready_path = Path(tmpdir) / "ready" + command = _sigterm_ignoring_command(ready_path) + spawned: list[subprocess.Popen[str]] = [] + + def capture_popen(*args, **kwargs): # noqa: ANN002, ANN003, ANN202 + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + def fail_emit(_event: str, **_payload: object) -> None: + _wait_for_path(ready_path) + raise failure_type("emit interrupted") + + runner = ProcessRunner(emit=fail_emit) + try: + with ( + patch( + "envctl_engine.shared.process_launch_support.subprocess.Popen", + side_effect=capture_popen, + ), + self.assertRaises(failure_type), + ): + runner.start_background(command) + + self.assertEqual(len(spawned), 1) + self.assertEqual(spawned[0].poll(), -signal.SIGKILL) + self.assertFalse(runner._launch_records[-1].active) # noqa: SLF001 + self.assertEqual(runner.launch_diagnostics_summary()["active_launch_count"], 0) + finally: + _force_process_cleanup(spawned[0] if spawned else None) + + @unittest.skipUnless(hasattr(os, "killpg"), "requires POSIX process groups") + def test_run_communicate_failures_terminate_real_child_before_reraising(self) -> None: + real_popen = subprocess.Popen + + for failure_type in (RuntimeError, KeyboardInterrupt): + with self.subTest(failure_type=failure_type.__name__): + with tempfile.TemporaryDirectory() as tmpdir: + ready_path = Path(tmpdir) / "ready" + command = _sigterm_ignoring_command(ready_path) + spawned: list[subprocess.Popen[str]] = [] + + class InterruptingProcess: + def __init__(self, process: subprocess.Popen[str]) -> None: + self.process = process + + @property + def pid(self) -> int: + return self.process.pid + + @property + def returncode(self) -> int | None: + return self.process.returncode + + def communicate(self, timeout: float | None = None) -> tuple[str, str]: + _ = timeout + _wait_for_path(ready_path) + raise failure_type("communicate interrupted") + + def poll(self) -> int | None: + return self.process.poll() + + def wait(self, timeout: float | None = None) -> int: + return self.process.wait(timeout=timeout) + + def terminate(self) -> None: + self.process.terminate() + + def kill(self) -> None: + self.process.kill() + + def capture_popen(*args, **kwargs): # noqa: ANN002, ANN003, ANN202 + process = real_popen(*args, **kwargs) + spawned.append(process) + return InterruptingProcess(process) + + runner = ProcessRunner() + try: + with ( + patch( + "envctl_engine.shared.process_runner.subprocess.Popen", + side_effect=capture_popen, + ), + self.assertRaises(failure_type), + ): + runner.run(command) + + self.assertEqual(len(spawned), 1) + self.assertEqual(spawned[0].poll(), -signal.SIGKILL) + finally: + _force_process_cleanup(spawned[0] if spawned else None) + def test_start_background_denies_controller_input_and_emits_launch_policy(self) -> None: events: list[tuple[str, dict[str, object]]] = [] runner = ProcessRunner(emit=lambda event, **payload: events.append((event, dict(payload)))) diff --git a/tests/python/shared/test_process_runner_listener_detection.py b/tests/python/shared/test_process_runner_listener_detection.py index 45ff0d0b7..3ba0f35c6 100644 --- a/tests/python/shared/test_process_runner_listener_detection.py +++ b/tests/python/shared/test_process_runner_listener_detection.py @@ -13,6 +13,11 @@ from envctl_engine.shared.process_runner import ProcessRunner +def _signal_then_report_process_group_gone(_pid: int, sig: int) -> None: + if sig == 0: + raise ProcessLookupError + + class ProcessRunnerListenerDetectionTests(unittest.TestCase): def _fake_run_for_ports(self, ports_by_pid: dict[int, list[int]]): def _run(cmd, **kwargs): # noqa: ANN001 @@ -164,9 +169,15 @@ class _FakeProcess: def communicate(self, timeout=None): # noqa: ANN001 raise subprocess.TimeoutExpired(cmd=["docker", "run"], timeout=120.0) + def wait(self, timeout=None): # noqa: ANN001 + return 0 + with ( patch("envctl_engine.shared.process_runner.subprocess.Popen", return_value=_FakeProcess()), - patch("envctl_engine.shared.process_runner.os.killpg"), + patch( + "envctl_engine.shared.process_runner.os.killpg", + side_effect=_signal_then_report_process_group_gone, + ), ): result = runner.run(["docker", "run"], timeout=120.0) @@ -193,9 +204,15 @@ def communicate(self, timeout=None): # noqa: ANN001 ) return ("tail stdout", "tail stderr") + def wait(self, timeout=None): # noqa: ANN001 + return 0 + with ( patch("envctl_engine.shared.process_runner.subprocess.Popen", return_value=_FakeProcess()), - patch("envctl_engine.shared.process_runner.os.killpg"), + patch( + "envctl_engine.shared.process_runner.os.killpg", + side_effect=_signal_then_report_process_group_gone, + ), ): result = runner.run(["docker", "run"], timeout=30.0) @@ -214,9 +231,15 @@ class _FakeProcess: def communicate(self, timeout=None): # noqa: ANN001 raise subprocess.TimeoutExpired(cmd=["docker", "run"], timeout=5.0) + def wait(self, timeout=None): # noqa: ANN001 + return 0 + with ( patch("envctl_engine.shared.process_runner.subprocess.Popen", return_value=_FakeProcess()), - patch("envctl_engine.shared.process_runner.os.killpg") as killpg, + patch( + "envctl_engine.shared.process_runner.os.killpg", + side_effect=_signal_then_report_process_group_gone, + ) as killpg, ): result = runner.run(["docker", "run"], timeout=5.0) diff --git a/tests/python/shared/test_structure_runtime_layout.py b/tests/python/shared/test_structure_runtime_layout.py index 6a62d805e..85ef02f2d 100644 --- a/tests/python/shared/test_structure_runtime_layout.py +++ b/tests/python/shared/test_structure_runtime_layout.py @@ -43,6 +43,16 @@ def test_runtime_lifecycle_cleanup_has_blast_owner(self) -> None: self.assertIn("LifecycleBlastCleanupSupport", orchestrator_text) self.assertLessEqual(len(orchestrator_text.splitlines()), 760) + def test_runtime_lifecycle_cleanup_has_full_cleanup_owner(self) -> None: + owner = REPO_ROOT / "python" / "envctl_engine" / "runtime" / "lifecycle_full_cleanup.py" + orchestrator = REPO_ROOT / "python" / "envctl_engine" / "runtime" / "lifecycle_cleanup_orchestrator.py" + + self.assertTrue(owner.is_file()) + owner_text = owner.read_text(encoding="utf-8") + orchestrator_text = orchestrator.read_text(encoding="utf-8") + self.assertIn("def clear_runtime_state", owner_text) + self.assertIn("from envctl_engine.runtime.lifecycle_full_cleanup import", orchestrator_text) + def test_runtime_command_parity_tests_are_split_by_owner(self) -> None: runtime_tests = REPO_ROOT / "tests" / "python" / "runtime" expected = [ diff --git a/tests/python/shared/test_utility_consolidation_contract.py b/tests/python/shared/test_utility_consolidation_contract.py index ad80b54ca..10c66f3eb 100644 --- a/tests/python/shared/test_utility_consolidation_contract.py +++ b/tests/python/shared/test_utility_consolidation_contract.py @@ -5,13 +5,19 @@ import tempfile import unittest from pathlib import Path +from types import SimpleNamespace from unittest.mock import patch REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" from envctl_engine.config import _default_port_value from envctl_engine.shared.node_tooling import detect_package_manager, detect_python_bin, load_package_json -from envctl_engine.shared.services import project_name_from_service_name, service_display_name +from envctl_engine.shared.services import ( + project_name_from_service_name, + resolve_service_project_name, + service_display_name, + service_project_name, +) from envctl_engine.shared.parsing import parse_bool, parse_float, parse_int, strip_quotes from envctl_engine.test_output.parser_base import strip_ansi from envctl_engine.ui.capabilities import ( @@ -87,6 +93,46 @@ def test_shared_domain_helpers_cover_services_ports_and_ansi(self) -> None: self.assertEqual(strip_ansi("\x1b]22;default\x07hello\x1b[15;72H"), "hello") self.assertEqual(strip_ansi("\\x1b[48;2;39;39;39m hello \\x1b[0m"), " hello ") + def test_service_project_resolution_prefers_record_metadata_then_legacy_parser(self) -> None: + legacy_calls: list[str] = [] + service = SimpleNamespace(name="Opaque Runtime Process", project="Customer Platform") + + self.assertEqual( + resolve_service_project_name( + service.name, + service, + project_name_from_service=lambda name: legacy_calls.append(name) or "Wrong Project", + ), + "Customer Platform", + ) + self.assertEqual(legacy_calls, []) + + legacy_additional = SimpleNamespace( + name="Customer Platform Voice Runtime", + project=None, + type="voice-runtime", + service_slug="voice-runtime", + ) + self.assertEqual(service_project_name(legacy_additional), "Customer Platform") + self.assertEqual( + resolve_service_project_name( + legacy_additional.name, + legacy_additional, + project_name_from_service=lambda _name: "Customer Platform Voice", + ), + "Customer Platform", + ) + + legacy = SimpleNamespace(name="Legacy Voice Runtime", project=None) + self.assertEqual( + resolve_service_project_name( + legacy.name, + legacy, + project_name_from_service=lambda _name: "Legacy", + ), + "Legacy", + ) + def test_service_display_name_has_one_shared_owner(self) -> None: for relative_path in ( "python/envctl_engine/dashboard_metadata.py", diff --git a/tests/python/startup/test_frontend_projection.py b/tests/python/startup/test_frontend_projection.py index fdaacf7c9..e051c73d3 100644 --- a/tests/python/startup/test_frontend_projection.py +++ b/tests/python/startup/test_frontend_projection.py @@ -62,6 +62,167 @@ def test_runtime_map_tracks_actual_frontend_rebound_port(self) -> None: self.assertEqual(runtime["service_to_actual_port"]["Tree Alpha Frontend"], 9122) self.assertEqual(runtime["port_to_service"][9122], "Tree Alpha Frontend") + def test_projection_uses_project_metadata_for_opaque_core_service_names(self) -> None: + state = RunState( + run_id="run-opaque", + mode="trees", + services={ + "Opaque API Process": ServiceRecord( + name="Opaque API Process", + type="backend", + cwd="/tmp/customer-platform/api", + actual_port=8181, + status="healthy", + project="Customer Platform", + service_slug="backend", + ), + "Opaque Web Process": ServiceRecord( + name="Opaque Web Process", + type="frontend", + cwd="/tmp/customer-platform/web", + actual_port=9191, + status="running", + project="Customer Platform", + service_slug="frontend", + ), + }, + ) + + runtime = build_runtime_map(state) + projection = runtime["projection"]["Customer Platform"] + + self.assertEqual(projection["backend_url"], "http://localhost:8181") + self.assertEqual(projection["frontend_url"], "http://localhost:9191") + self.assertEqual(projection["backend_status"], "healthy") + self.assertEqual(projection["frontend_status"], "running") + self.assertEqual( + frontend_env_for_project(state, "Customer Platform")["VITE_BACKEND_URL"], + "http://localhost:8181", + ) + + def test_collision_projection_keeps_every_instance_and_uses_one_authoritative_record(self) -> None: + collision_backend_name = "Main Backend Restart Collision 64001" + collision_frontend_name = "Main Frontend Restart Collision 64002" + services = { + collision_backend_name: ServiceRecord( + name=collision_backend_name, + type="backend", + cwd="/tmp/main/replacement-backend", + pid=64001, + actual_port=8251, + status="termination_failed", + project="Main", + service_slug="backend", + degraded=True, + ), + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + pid=63001, + actual_port=8000, + status="running", + project="Main", + service_slug="backend", + ), + collision_frontend_name: ServiceRecord( + name=collision_frontend_name, + type="frontend", + cwd="/tmp/main/replacement-frontend", + pid=64002, + actual_port=9251, + status="termination_failed", + project="Main", + service_slug="frontend", + degraded=True, + ), + "Main Frontend": ServiceRecord( + name="Main Frontend", + type="frontend", + cwd="/tmp/main/frontend", + pid=63002, + actual_port=9000, + status="healthy", + project="Main", + service_slug="frontend", + ), + } + state = RunState(run_id="run-collision", mode="main", services=services) + + runtime = build_runtime_map(state, host="runtime.test") + project = runtime["projects"]["Main"] + projected_services = project["services"] + projection = runtime["projection"]["Main"] + + self.assertEqual( + set(projected_services), + { + "backend", + f"backend@{collision_backend_name}", + "frontend", + f"frontend@{collision_frontend_name}", + }, + ) + self.assertEqual(projected_services["backend"]["name"], "Main Backend") + self.assertEqual(projected_services["backend"]["port"], 8000) + self.assertEqual(projected_services[f"backend@{collision_backend_name}"]["port"], 8251) + self.assertEqual(projected_services["frontend"]["name"], "Main Frontend") + self.assertEqual(projected_services["frontend"]["port"], 9000) + self.assertEqual(projected_services[f"frontend@{collision_frontend_name}"]["port"], 9251) + + self.assertEqual(project["backend_port"], 8000) + self.assertEqual(projection["backend_port"], 8000) + self.assertEqual(projection["backend_url"], "http://runtime.test:8000") + self.assertEqual(projection["backend_status"], "running") + self.assertEqual(project["frontend_port"], 9000) + self.assertEqual(projection["frontend_port"], 9000) + self.assertEqual(projection["frontend_url"], "http://runtime.test:9000") + self.assertEqual(projection["frontend_status"], "healthy") + + reversed_state = RunState( + run_id="run-collision-reversed", + mode="main", + services=dict(reversed(list(services.items()))), + ) + reversed_runtime = build_runtime_map(reversed_state, host="runtime.test") + + self.assertEqual( + reversed_runtime["projects"]["Main"]["services"], + projected_services, + ) + self.assertEqual(reversed_runtime["projection"]["Main"], projection) + + def test_legacy_collision_suffix_without_project_metadata_stays_under_original_project(self) -> None: + state = RunState( + run_id="run-legacy-collision", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/main", + actual_port=8000, + status="running", + ), + "Main Backend Restart Collision 2": ServiceRecord( + name="Main Backend Restart Collision 2", + type="backend", + cwd="/replacement", + actual_port=8001, + status="termination_failed", + ), + }, + ) + + runtime = build_runtime_map(state) + + self.assertEqual(set(runtime["projects"]), {"Main"}) + self.assertEqual( + set(runtime["projects"]["Main"]["services"]), + {"backend", "backend@Main Backend Restart Collision 2"}, + ) + self.assertEqual(runtime["projection"]["Main"]["backend_port"], 8000) + def test_multi_project_projection_uses_per_project_backend_port(self) -> None: state = RunState( run_id="run-2", diff --git a/tests/python/startup/test_restart_prestop_support.py b/tests/python/startup/test_restart_prestop_support.py index 173475120..ff706cafb 100644 --- a/tests/python/startup/test_restart_prestop_support.py +++ b/tests/python/startup/test_restart_prestop_support.py @@ -5,10 +5,12 @@ from pathlib import Path from types import SimpleNamespace -from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.command_router import Route, parse_route from envctl_engine.shared.ports import PortPlanner +from envctl_engine.state.models import RunState, ServiceRecord from envctl_engine.startup.restart_prestop_support import ( RestartOrphanListenerScan, + _validate_restart_selection, apply_restart_port_assignments, apply_restart_ports_to_contexts, handle_restart_prestop, @@ -22,6 +24,10 @@ restart_prestop_preservation, restart_start_route, ) +from envctl_engine.startup.startup_selection_support import ( + _restart_service_types_for_project, + trees_start_selection_required, +) class _NullContext: @@ -52,6 +58,389 @@ def fail(self, message: str) -> None: class RestartPrestopSupportTests(unittest.TestCase): + def test_restart_cli_parses_ignore_service_dependencies(self) -> None: + route = parse_route( + ["restart", "--service", "voice-runtime", "--ignore-service-deps"], + env={}, + ) + + self.assertEqual(route.flags["services"], ["voice-runtime"]) + self.assertTrue(route.flags["ignore_service_deps"]) + + def test_restart_prestop_respects_runtime_scope_and_project_intersection(self) -> None: + services = { + f"{project} {display}": ServiceRecord( + name=f"{project} {display}", + type=service_type, + cwd=".", + project=project, + service_slug=service_type, + ) + for project in ("Alpha", "Beta") + for service_type, display in ( + ("backend", "Backend"), + ("frontend", "Frontend"), + ("voice-runtime", "Voice Runtime"), + ) + } + state = SimpleNamespace(services=services) + + cases = ( + ({"runtime_scope": "backend"}, [], {"Alpha Backend", "Beta Backend"}), + ( + {"runtime_scope": "fullstack"}, + [], + {"Alpha Backend", "Alpha Frontend", "Beta Backend", "Beta Frontend"}, + ), + ({"runtime_scope": "dependencies"}, [], set()), + ( + {"services": ["voice-runtime"], "ignore_service_deps": True}, + ["Alpha"], + {"Alpha Voice Runtime"}, + ), + ) + for flags, projects, expected in cases: + with self.subTest(flags=flags, projects=projects): + selection = restart_prestop_selection( + state=state, + route=Route(command="restart", mode="trees", projects=projects, flags=flags), + runtime=SimpleNamespace(_project_name_from_service=lambda _name: "wrong"), + ) + self.assertEqual(selection.selected_services, expected) + + def test_restart_only_and_no_modifiers_keep_prestop_and_launch_in_lockstep(self) -> None: + services = { + f"Main {display}": ServiceRecord( + name=f"Main {display}", + type=service_type, + cwd=".", + project="Main", + service_slug=service_type, + ) + for service_type, display in ( + ("backend", "Backend"), + ("frontend", "Frontend"), + ("voice-runtime", "Voice Runtime"), + ) + } + state = RunState( + run_id="run-1", + mode="main", + services=services, + requirements={"Main": object()}, # type: ignore[dict-item] + metadata={ + "project_roots": {"Main": "/repo"}, + "dashboard_project_configured_services": { + "Main": ["backend", "frontend", "voice-runtime"] + }, + }, + ) + additional = (SimpleNamespace(name="voice-runtime", depends_on=("backend",)),) + runtime = SimpleNamespace( + config=SimpleNamespace(additional_services=additional), + _project_name_from_service=lambda _name: "Main", + ) + cases = ( + ("--only-backend", {"backend"}), + ("--only-frontend", {"frontend"}), + ("--no-infra", set()), + ("--no-dependencies", {"backend", "frontend", "voice-runtime"}), + ) + + for modifier, expected_types in cases: + with self.subTest(modifier=modifier): + route = parse_route(["restart", modifier], env={}) + selection = restart_prestop_selection(state=state, route=route, runtime=runtime) + expected_names = { + name for name, service in services.items() if service.type in expected_types + } + self.assertEqual(selection.selected_services, expected_names) + self.assertEqual(selection.service_types_by_project, {"Main": expected_types}) + self.assertFalse(selection.include_requirements) + + start_route = restart_start_route( + route, + restart_lookup_mode="main", + selected_services=selection.selected_services, + target_projects=selection.target_projects, + include_requirements=selection.include_requirements, + service_types_by_project=selection.service_types_by_project, + ) + self.assertEqual( + _restart_service_types_for_project( + route=start_route, + project_name="Main", + default_service_types={"backend", "frontend", "voice-runtime"}, + additional_services=additional, + ), + expected_types, + ) + + def test_restart_selection_preserves_heterogeneous_opaque_service_targets(self) -> None: + services = { + "alpha-api": ServiceRecord( + name="alpha-api", type="backend", service_slug="backend", project="Alpha", cwd="." + ), + "alpha-agent": ServiceRecord( + name="alpha-agent", + type="voice-runtime", + service_slug="voice-runtime", + project="Alpha", + cwd=".", + ), + "beta-api": ServiceRecord( + name="beta-api", type="backend", service_slug="backend", project="Beta", cwd="." + ), + "beta-agent": ServiceRecord( + name="beta-agent", + type="voice-runtime", + service_slug="voice-runtime", + project="Beta", + cwd=".", + ), + } + state = RunState( + run_id="run-1", + mode="trees", + services=services, + metadata={ + "project_roots": {"Alpha": "/alpha", "Beta": "/beta"}, + "dashboard_project_configured_services": { + "Alpha": ["backend", "voice-runtime"], + "Beta": ["backend", "voice-runtime", "worker"], + }, + "dashboard_stopped_services": [ + {"project": "Beta", "type": "worker", "name": "beta-job"} + ], + }, + ) + additional = ( + SimpleNamespace(name="voice-runtime", depends_on=("backend",)), + SimpleNamespace(name="worker", depends_on=("voice-runtime",)), + ) + runtime = SimpleNamespace( + config=SimpleNamespace(additional_services=additional), + _project_name_from_service=lambda _name: "wrong", + ) + route = Route( + command="restart", + mode="trees", + flags={"services": ["alpha-agent", "beta-job"]}, + ) + + selection = restart_prestop_selection(state=state, route=route, runtime=runtime) + _validate_restart_selection( + state=state, + route=route, + runtime=runtime, + target_projects=selection.target_projects, + service_types_by_project=selection.service_types_by_project, + ) + + self.assertEqual(selection.target_projects, {"Alpha", "Beta"}) + self.assertEqual( + selection.service_types_by_project, + { + "Alpha": {"backend", "voice-runtime"}, + "Beta": {"backend", "voice-runtime", "worker"}, + }, + ) + self.assertEqual(selection.selected_services, set(services)) + + def test_restart_validation_rejects_cross_project_unconfigured_and_empty_tree_targets(self) -> None: + state = RunState( + run_id="run-1", + mode="trees", + services={ + "alpha-agent": ServiceRecord( + name="alpha-agent", + type="voice-runtime", + service_slug="voice-runtime", + project="Alpha", + cwd=".", + ) + }, + metadata={ + "project_roots": {"Alpha": "/alpha", "Beta": "/beta"}, + "dashboard_project_configured_services": { + "Alpha": ["backend", "voice-runtime"], + "Beta": ["frontend"], + }, + }, + ) + runtime = SimpleNamespace( + config=SimpleNamespace( + additional_services=(SimpleNamespace(name="voice-runtime", depends_on=("backend",)),) + ), + _project_name_from_service=lambda _name: "wrong", + ) + for selector in ("alpha-agent", "backend"): + with self.subTest(selector=selector): + route = Route( + command="restart", + mode="trees", + projects=["Beta"], + flags={"services": [selector]}, + ) + selection = restart_prestop_selection(state=state, route=route, runtime=runtime) + with self.assertRaisesRegex(RuntimeError, "No matching services"): + _validate_restart_selection( + state=state, + route=route, + runtime=runtime, + target_projects=selection.target_projects, + service_types_by_project=selection.service_types_by_project, + ) + + conflicting_route = Route( + command="restart", + mode="trees", + projects=["Alpha"], + flags={ + "services": ["alpha-agent"], + "launch_backend": True, + "launch_frontend": False, + "launch_dependencies": False, + }, + ) + conflicting_selection = restart_prestop_selection( + state=state, + route=conflicting_route, + runtime=runtime, + ) + with self.assertRaisesRegex(RuntimeError, "conflicts with the requested launch scope"): + _validate_restart_selection( + state=state, + route=conflicting_route, + runtime=runtime, + target_projects=conflicting_selection.target_projects, + service_types_by_project=conflicting_selection.service_types_by_project, + ) + + empty_state = RunState(run_id="run-empty", mode="trees") + empty_route = Route( + command="restart", + mode="trees", + flags={"services": ["backend"], "batch": True}, + ) + empty_selection = restart_prestop_selection( + state=empty_state, + route=empty_route, + runtime=runtime, + ) + with self.assertRaisesRegex(RuntimeError, "No matching services|No active restart target"): + _validate_restart_selection( + state=empty_state, + route=empty_route, + runtime=runtime, + target_projects=empty_selection.target_projects, + service_types_by_project=empty_selection.service_types_by_project, + ) + + def test_global_dependency_restart_targets_requirements_only_projects(self) -> None: + state = RunState( + run_id="run-1", + mode="trees", + services={ + "Alpha Backend": ServiceRecord( + name="Alpha Backend", type="backend", project="Alpha", cwd="." + ) + }, + requirements={"Alpha": object(), "Beta": object()}, # type: ignore[dict-item] + metadata={"project_roots": {"Alpha": "/alpha", "Beta": "/beta"}}, + ) + runtime = SimpleNamespace( + config=SimpleNamespace(additional_services=()), + _project_name_from_service=lambda name: str(name).removesuffix(" Backend"), + ) + route = parse_route(["restart", "--trees", "--dependencies"], env={}) + + selection = restart_prestop_selection(state=state, route=route, runtime=runtime) + preservation = restart_prestop_preservation( + state, + selected_services=selection.selected_services, + include_requirements=selection.include_requirements, + target_projects=selection.target_projects, + ) + + self.assertEqual(selection.target_projects, {"Alpha", "Beta"}) + self.assertEqual(selection.selected_services, set()) + self.assertEqual(set(preservation.requirements_to_release), {"Alpha", "Beta"}) + + def test_explicit_service_restart_prestops_tracked_dependency_before_relaunch(self) -> None: + state = SimpleNamespace( + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + project="Main", + ), + "Main Voice Runtime": ServiceRecord( + name="Main Voice Runtime", + type="voice-runtime", + cwd="/repo/voice-runtime", + project="Main", + service_slug="voice-runtime", + ), + } + ) + runtime = SimpleNamespace( + config=SimpleNamespace( + additional_services=( + SimpleNamespace(name="voice-runtime", depends_on=("backend",)), + ) + ), + _project_name_from_service=lambda _name: "wrong-fallback", + ) + + selection = restart_prestop_selection( + state=state, + route=Route( + command="restart", + mode="main", + flags={"services": ["voice-runtime"]}, + ), + runtime=runtime, + ) + + self.assertEqual(selection.target_projects, {"Main"}) + self.assertEqual(selection.selected_services, {"Main Backend", "Main Voice Runtime"}) + + def test_explicit_service_restart_ignore_deps_does_not_prestop_dependency(self) -> None: + state = SimpleNamespace( + services={ + "Main Backend": ServiceRecord( + name="Main Backend", type="backend", cwd="/repo/backend", project="Main" + ), + "Main Voice Runtime": ServiceRecord( + name="Main Voice Runtime", + type="voice-runtime", + cwd="/repo/voice-runtime", + project="Main", + service_slug="voice-runtime", + ), + } + ) + runtime = SimpleNamespace( + config=SimpleNamespace( + additional_services=(SimpleNamespace(name="voice-runtime", depends_on=("backend",)),) + ), + _project_name_from_service=lambda _name: "Main", + ) + + selection = restart_prestop_selection( + state=state, + route=Route( + command="restart", + mode="main", + flags={"services": ["voice-runtime"], "ignore_service_deps": True}, + ), + runtime=runtime, + ) + + self.assertEqual(selection.selected_services, {"Main Voice Runtime"}) + def test_handle_restart_prestop_noops_for_non_restart_routes(self) -> None: route = Route(command="start", mode="main", raw_args=["start"], flags={}) session = SimpleNamespace(effective_route=route, runtime_mode="main") @@ -129,7 +518,7 @@ def test_handle_restart_prestop_stops_selected_services_and_preserves_remaining_ _try_load_existing_state=lambda *, mode: state, _project_name_from_service=lambda name: str(name).removesuffix(" Backend"), _terminate_services_from_state=lambda state, *, selected_services, aggressive, verify_ownership: ( - terminated.append((state, set(selected_services))) + terminated.append((state, set(selected_services))) or set() ), _release_requirement_ports=released.append, _emit=lambda event, **payload: events.append((event, payload)), @@ -139,7 +528,7 @@ def test_handle_restart_prestop_stops_selected_services_and_preserves_remaining_ runtime=runtime, session=session, suppress_progress_output=lambda route: False, - terminate_restart_orphan_listeners=lambda **kwargs: orphan_calls.append(dict(kwargs)), + terminate_restart_orphan_listeners=lambda **kwargs: orphan_calls.append(dict(kwargs)) or set(), spinner_factory=lambda *args, **kwargs: spinner, use_spinner_policy_fn=lambda policy: _NullContext(), resolve_spinner_policy_fn=lambda env: SimpleNamespace(enabled=True), @@ -211,13 +600,158 @@ def test_handle_restart_prestop_aborts_without_mutating_session_when_termination emit_spinner_policy_fn=lambda *_args, **_kwargs: None, ) - self.assertEqual(session.preserved_services, {}) - self.assertEqual(session.preserved_requirements, {}) - self.assertEqual(session.base_metadata, {}) + self.assertEqual(session.preserved_services, state.services) + self.assertEqual(session.preserved_requirements, state.requirements) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-old"]) self.assertTrue(session.preserve_existing_state_on_failure) self.assertIs(session.restart_state, state) self.assertIs(session.effective_route, route) + def test_handle_restart_prestop_treats_missing_termination_result_as_unconfirmed(self) -> None: + route = Route(command="restart", mode="main", flags={"services": ["Main Backend"]}) + session = SimpleNamespace( + effective_route=route, + runtime_mode="main", + restart_state=None, + preserved_requirements={}, + preserved_services={}, + base_metadata={}, + ) + state = SimpleNamespace( + run_id="run-old", + mode="main", + services={"Main Backend": object()}, + requirements={"Main": object()}, + metadata={}, + ) + runtime = SimpleNamespace( + env={}, + _effective_start_mode=lambda _route: "main", + _try_load_existing_state=lambda *, mode: state, + _project_name_from_service=lambda _name: "Main", + _terminate_services_from_state=lambda *_args, **_kwargs: None, + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(RuntimeError, "Main Backend"): + handle_restart_prestop( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: True, + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("orphans must not be reaped"), + spinner_factory=lambda *_args, **_kwargs: _RecordingSpinner(), + use_spinner_policy_fn=lambda _policy: _NullContext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, state.services) + self.assertEqual(session.preserved_requirements, state.requirements) + self.assertIs(session.effective_route, route) + + def test_handle_restart_prestop_preserves_source_authority_when_termination_raises(self) -> None: + route = Route(command="restart", mode="main", flags={"services": ["Main Backend"]}) + state = SimpleNamespace( + run_id="run-old", + mode="main", + services={"Main Backend": object()}, + requirements={"Main": object()}, + metadata={"authority": "old"}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="main", + restart_state=None, + preserved_requirements={}, + preserved_services={}, + base_metadata={}, + ) + + def fail_termination(*_args, **_kwargs): # noqa: ANN202 + raise OSError("service lock unlink failed") + + runtime = SimpleNamespace( + env={}, + _effective_start_mode=lambda _route: "main", + _try_load_existing_state=lambda *, mode: state, + _project_name_from_service=lambda _name: "Main", + _terminate_services_from_state=fail_termination, + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(OSError, "service lock unlink failed"): + handle_restart_prestop( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: True, + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("orphans must not be reaped"), + spinner_factory=lambda *_args, **_kwargs: _RecordingSpinner(), + use_spinner_policy_fn=lambda _policy: _NullContext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, state.services) + self.assertEqual(session.preserved_requirements, state.requirements) + self.assertEqual(session.base_metadata["authority"], "old") + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-old"]) + + def test_handle_restart_prestop_preserves_source_authority_when_requirement_release_raises(self) -> None: + route = Route( + command="restart", + mode="main", + flags={"services": ["Main Backend"], "restart_include_requirements": True}, + ) + state = SimpleNamespace( + run_id="run-old", + mode="main", + services={"Main Backend": object()}, + requirements={"Main": object()}, + metadata={"authority": "old"}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="main", + restart_state=None, + preserved_requirements={}, + preserved_services={}, + base_metadata={}, + ) + + def fail_release(_requirements): # noqa: ANN001, ANN202 + raise OSError("requirement lock unlink failed") + + runtime = SimpleNamespace( + env={}, + _effective_start_mode=lambda _route: "main", + _try_load_existing_state=lambda *, mode: state, + _project_name_from_service=lambda _name: "Main", + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + _release_requirement_ports=fail_release, + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(OSError, "requirement lock unlink failed"): + handle_restart_prestop( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: True, + terminate_restart_orphan_listeners=lambda **_kwargs: set(), + spinner_factory=lambda *_args, **_kwargs: _RecordingSpinner(), + use_spinner_policy_fn=lambda _policy: _NullContext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, state.services) + self.assertEqual(session.preserved_requirements, state.requirements) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-old"]) + def test_handle_restart_prestop_aborts_when_orphan_exit_is_unconfirmed(self) -> None: route = Route(command="restart", mode="main", flags={"services": ["Main Backend"]}) session = SimpleNamespace( @@ -258,7 +792,8 @@ def test_handle_restart_prestop_aborts_when_orphan_exit_is_unconfirmed(self) -> ) self.assertTrue(session.preserve_existing_state_on_failure) - self.assertEqual(session.preserved_services, {}) + self.assertEqual(session.preserved_services, state.services) + self.assertEqual(session.preserved_requirements, state.requirements) self.assertIs(session.effective_route, route) def test_restart_fallback_start_route_preserves_request_context_and_marks_restart(self) -> None: @@ -293,12 +828,120 @@ def test_restart_start_route_records_sorted_selection_policy(self) -> None: ) self.assertEqual(updated.command, "start") + self.assertEqual(updated.projects, ["Main"]) self.assertEqual(updated.flags["_restart_selected_services"], ["Main Backend", "Main Frontend"]) self.assertEqual(updated.flags["_restart_target_projects"], ["Main"]) self.assertFalse(updated.flags["_restart_include_requirements"]) self.assertTrue(updated.flags["_restart_request"]) self.assertTrue(updated.flags["force"]) + def test_restart_start_route_carries_resolved_tree_project_without_prompt(self) -> None: + updated = restart_start_route( + Route(command="restart", mode="trees", flags={"services": ["voice-runtime"]}), + restart_lookup_mode="trees", + selected_services=set(), + target_projects={"Customer Platform"}, + include_requirements=False, + ) + + self.assertEqual(updated.projects, ["Customer Platform"]) + self.assertFalse(trees_start_selection_required(route=updated, runtime_mode="trees")) + + def test_unmatched_service_selector_never_expands_to_all_projects(self) -> None: + state = SimpleNamespace( + services={ + "Alpha Backend": ServiceRecord( + name="Alpha Backend", type="backend", cwd=".", project="Alpha" + ), + "Alpha Voice Runtime": ServiceRecord( + name="Alpha Voice Runtime", + type="voice-runtime", + cwd=".", + project="Alpha", + service_slug="voice-runtime", + ), + "Beta Backend": ServiceRecord( + name="Beta Backend", type="backend", cwd=".", project="Beta" + ), + "Beta Voice Runtime": ServiceRecord( + name="Beta Voice Runtime", + type="voice-runtime", + cwd=".", + project="Beta", + service_slug="voice-runtime", + ), + }, + requirements={}, + metadata={"project_roots": {"Alpha": "/repo/alpha", "Beta": "/repo/beta"}}, + ) + route = Route(command="restart", mode="trees", flags={"services": ["typo-service"]}) + runtime = SimpleNamespace( + config=SimpleNamespace( + additional_services=(SimpleNamespace(name="voice-runtime", depends_on=("backend",)),) + ), + _project_name_from_service=lambda name: str(name).rsplit(" ", 1)[0], + ) + + selection = restart_prestop_selection(state=state, route=route, runtime=runtime) + + self.assertEqual(selection.selected_services, set()) + self.assertEqual(selection.target_projects, set()) + with self.assertRaisesRegex(RuntimeError, "No matching services found.*typo-service"): + _validate_restart_selection( + state=state, + route=route, + runtime=runtime, + target_projects=selection.target_projects, + ) + + project_route = Route( + command="restart", + mode="trees", + projects=["Alpha"], + flags={"services": ["typo-service"]}, + ) + project_selection = restart_prestop_selection( + state=state, + route=project_route, + runtime=runtime, + ) + self.assertEqual(project_selection.selected_services, set()) + with self.assertRaisesRegex(RuntimeError, "No matching services found.*typo-service"): + _validate_restart_selection( + state=state, + route=project_route, + runtime=runtime, + target_projects=project_selection.target_projects, + ) + + def test_unmatched_project_selector_never_selects_all_services(self) -> None: + state = SimpleNamespace( + services={ + "Alpha Backend": ServiceRecord( + name="Alpha Backend", type="backend", cwd=".", project="Alpha" + ), + "Beta Backend": ServiceRecord( + name="Beta Backend", type="backend", cwd=".", project="Beta" + ), + }, + requirements={}, + metadata={"project_roots": {"Alpha": "/repo/alpha", "Beta": "/repo/beta"}}, + ) + route = Route(command="restart", mode="trees", projects=["Typo"], flags={}) + runtime = SimpleNamespace(_project_name_from_service=lambda name: str(name).removesuffix(" Backend")) + + selection = restart_prestop_selection(state=state, route=route, runtime=runtime) + + self.assertEqual(selection.selected_services, set()) + self.assertEqual(selection.target_projects, {"Typo"}) + with self.assertRaisesRegex(RuntimeError, "No active restart target.*Typo"): + _validate_restart_selection( + state=state, + route=route, + runtime=runtime, + target_projects=selection.target_projects, + ) + def test_restart_prestop_state_rejects_mode_mismatch_and_builds_fallback_route(self) -> None: route = Route(command="restart", mode="trees", raw_args=["restart", "--trees"], flags={}) events: list[tuple[str, dict[str, object]]] = [] @@ -347,7 +990,7 @@ def test_restart_prestop_state_inherits_docker_runtime_from_saved_state(self) -> self.assertTrue(route.flags["docker"]) self.assertEqual(runtime.env["DOCKER_MODE"], "true") - def test_restart_prestop_preservation_splits_services_and_requirements(self) -> None: + def test_restart_prestop_preservation_matches_requirement_projects_case_insensitively(self) -> None: state = SimpleNamespace( services={ "Main Backend": object(), @@ -356,6 +999,10 @@ def test_restart_prestop_preservation_splits_services_and_requirements(self) -> }, requirements={ "Main": SimpleNamespace(name="main requirements"), + "Main Restart Collision": SimpleNamespace( + name="replacement requirements", + project="Main", + ), "Other": SimpleNamespace(name="other requirements"), }, ) @@ -364,11 +1011,14 @@ def test_restart_prestop_preservation_splits_services_and_requirements(self) -> state, selected_services={"Main Backend", "Main Frontend"}, include_requirements=True, - target_projects={"Main"}, + target_projects={"main"}, ) self.assertEqual(set(result.preserved_services), {"Other Backend"}) - self.assertEqual(set(result.requirements_to_release), {"Main"}) + self.assertEqual( + set(result.requirements_to_release), + {"Main", "Main Restart Collision"}, + ) self.assertEqual(set(result.preserved_requirements), {"Other"}) def test_restart_prestop_preservation_preserves_requirements_when_not_included(self) -> None: @@ -411,6 +1061,36 @@ def test_restart_prestop_selection_falls_back_to_selected_service_projects_for_r self.assertEqual(selection.target_projects, {"Main"}) self.assertTrue(selection.include_requirements) + def test_restart_prestop_selection_includes_project_additional_services(self) -> None: + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace(project="Main", type="backend"), + "Main Frontend": SimpleNamespace(project="Main", type="frontend"), + "Main Optional Bad": SimpleNamespace( + project="Main", + type="optional-bad", + service_slug="optional-bad", + ), + } + ) + route = Route( + command="restart", + mode="main", + raw_args=["restart", "--project", "Main", "--entire-system"], + projects=["Main"], + flags={"runtime_scope": "entire-system"}, + ) + runtime = SimpleNamespace(_project_name_from_service=lambda name: str(name).split(" ", 1)[0]) + + selection = restart_prestop_selection(state=state, route=route, runtime=runtime) + + self.assertEqual( + selection.selected_services, + {"Main Backend", "Main Frontend", "Main Optional Bad"}, + ) + self.assertEqual(selection.target_projects, {"Main"}) + self.assertTrue(selection.include_requirements) + def test_restart_port_assignments_use_actual_port_then_requested_port_for_selected_services(self) -> None: state = SimpleNamespace( services={ diff --git a/tests/python/startup/test_resume_restore_policy.py b/tests/python/startup/test_resume_restore_policy.py index 26c49ee52..848e8610c 100644 --- a/tests/python/startup/test_resume_restore_policy.py +++ b/tests/python/startup/test_resume_restore_policy.py @@ -8,6 +8,7 @@ from envctl_engine.runtime.command_router import Route, parse_route from envctl_engine.startup.resume_restore_execution import ( _apply_restore_runtime_mode, + _projects_to_restore, _route_for_resume_restore, ) from envctl_engine.startup.resume_restore_policy import ( @@ -20,6 +21,7 @@ apply_ports_to_context, project_root, ) +from envctl_engine.startup.resume_restore_results import apply_restore_result from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord @@ -184,6 +186,60 @@ def test_project_root_uses_metadata_then_service_cwd_fallback(self) -> None: self.assertEqual(project_root(SimpleNamespace(runtime=runtime), state, "Feature"), repo) + def test_resume_project_discovery_and_root_prefer_service_project_metadata(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + service_root = Path(tmpdir) / "customer-platform" / "voice" + service_root.mkdir(parents=True) + service = ServiceRecord( + name="Opaque Runtime Process", + type="voice-runtime", + cwd=str(service_root), + project="Customer Platform", + ) + state = RunState( + run_id="run-extra", + mode="trees", + services={service.name: service}, + ) + runtime = SimpleNamespace( + config=SimpleNamespace(base_dir=Path(tmpdir)), + _project_name_from_service=lambda _name: "Opaque Runtime", + _requirements_ready=lambda _requirements: True, + ) + + self.assertEqual( + _projects_to_restore(runtime, state=state, missing_services=[service.name]), + ["Customer Platform"], + ) + self.assertEqual( + project_root(SimpleNamespace(runtime=runtime), state, "Customer Platform"), + service_root, + ) + + def test_projects_to_restore_deduplicates_service_and_requirement_project_case(self) -> None: + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/repo/backend", + project="Main", + ) + requirements = RequirementsResult(project="main", health="degraded") + state = RunState( + run_id="run-casefold", + mode="main", + services={service.name: service}, + requirements={"main": requirements}, + ) + runtime = SimpleNamespace( + _project_name_from_service=lambda _name: "Main", + _requirements_ready=lambda value: value is not requirements, + ) + + self.assertEqual( + _projects_to_restore(runtime, state=state, missing_services=[service.name]), + ["Main"], + ) + def test_apply_ports_to_context_projects_service_ports(self) -> None: backend_plan = SimpleNamespace(port=None) frontend_plan = SimpleNamespace(port=None) @@ -206,6 +262,125 @@ def test_apply_ports_to_context_projects_service_ports(self) -> None: self.assertEqual(calls, [(backend_plan, 8000), (frontend_plan, 9000)]) + def test_apply_ports_to_context_uses_record_project_metadata(self) -> None: + backend_plan = SimpleNamespace(port=None) + calls: list[tuple[object, int]] = [] + runtime = SimpleNamespace( + _project_name_from_service=lambda _name: "Opaque Runtime", + _set_plan_port=lambda plan, port: calls.append((plan, port)), + _set_plan_port_from_component=lambda plan, component: None, + ) + context = SimpleNamespace( + name="Customer Platform", + ports={"backend": backend_plan, "frontend": SimpleNamespace(port=None)}, + ) + service = ServiceRecord( + name="Opaque API Process", + type="backend", + cwd="/tmp/customer-platform/api", + actual_port=8100, + project="Customer Platform", + ) + state = RunState(run_id="run-ports", mode="trees", services={service.name: service}) + + apply_ports_to_context(SimpleNamespace(runtime=runtime), context, state) + + self.assertEqual(calls, [(backend_plan, 8100)]) + + def test_apply_ports_to_context_finds_case_insensitive_requirements_key(self) -> None: + db_plan = SimpleNamespace(port=None) + projected: list[tuple[object, dict[str, object]]] = [] + runtime = SimpleNamespace( + _project_name_from_service=lambda _name: "Main", + _set_plan_port=lambda _plan, _port: None, + _set_plan_port_from_component=lambda plan, component: projected.append((plan, component)), + ) + context = SimpleNamespace(name="Main", ports={"db": db_plan}) + requirements = RequirementsResult( + project="main", + db={"enabled": True, "success": True, "final": 5544}, + ) + state = RunState( + run_id="run-requirement-case", + mode="main", + requirements={"main": requirements}, + ) + + apply_ports_to_context(SimpleNamespace(runtime=runtime), context, state) + + self.assertIn((db_plan, requirements.db), projected) + + def test_failed_restore_retains_unterminated_service_under_collision_safe_key(self) -> None: + old = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/repo/backend", + project="Main", + pid=1111, + status="running", + ) + replacement = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/repo/backend", + project="Main", + pid=2222, + status="starting", + ) + state = RunState(run_id="run-collision", mode="main", services={old.name: old}) + errors: list[str] = [] + + apply_restore_result( + state, + project="Main", + result={ + "error": "replacement cleanup was unconfirmed", + "unterminated_services": {replacement.name: replacement}, + }, + errors=errors, + timing_enabled=False, + timing_lines=[], + project_totals_ms={}, + ) + + self.assertEqual({service.pid for service in state.services.values()}, {1111, 2222}) + collision_name = next(name for name in state.services if "Resume Collision" in name) + self.assertEqual(state.services[collision_name].status, "termination_failed") + self.assertTrue(state.services[collision_name].degraded) + self.assertEqual(state.metadata["termination_failed_services"], [collision_name]) + self.assertEqual(errors, ["Main: replacement cleanup was unconfirmed"]) + + def test_successful_restore_retains_cross_project_service_name_collisions(self) -> None: + state = RunState(run_id="run-success-collision", mode="trees") + + for project, pid in (("Alpha", 101), ("Beta", 202)): + service = ServiceRecord( + name="Shared Worker", + type="worker", + cwd=f"/{project.casefold()}", + project=project, + pid=pid, + status="running", + ) + apply_restore_result( + state, + project=project, + result={ + "error": None, + "remove_service_names": ["Shared Worker"], + "project_services": {"Shared Worker": service}, + }, + errors=[], + timing_enabled=False, + timing_lines=[], + project_totals_ms={}, + ) + + self.assertEqual({service.pid for service in state.services.values()}, {101, 202}) + self.assertEqual({service.project for service in state.services.values()}, {"Alpha", "Beta"}) + collision_name = next(name for name in state.services if "Resume Collision" in name) + self.assertEqual(state.services[collision_name].name, collision_name) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/startup/test_run_reuse_identity.py b/tests/python/startup/test_run_reuse_identity.py index 607bd4fd2..5cb50cd30 100644 --- a/tests/python/startup/test_run_reuse_identity.py +++ b/tests/python/startup/test_run_reuse_identity.py @@ -18,6 +18,7 @@ project_identities_from_contexts, project_identities_from_state, ) +from envctl_engine.startup.run_reuse_decision import state_has_resumable_services from envctl_engine.state.models import RunState, ServiceRecord @@ -118,6 +119,32 @@ def test_project_identities_from_state_uses_metadata_requirements_and_services(s ], ) + def test_state_identity_and_resumability_prefer_multiword_record_project_metadata(self) -> None: + service = ServiceRecord( + name="Opaque API Runtime", + type="backend", + cwd="/tmp/customer-platform/runtime", + project="Customer Platform", + ) + state = RunState( + run_id="run-metadata", + mode="trees", + services={service.name: service}, + metadata={"project_roots": {"Customer Platform": "/tmp/customer-platform"}}, + ) + runtime = SimpleNamespace(_project_name_from_service=lambda _name: "Opaque API") + + self.assertEqual( + identities_to_payload(project_identities_from_state(runtime, state)), + [ + { + "name": "Customer Platform", + "root": normalize_project_root("/tmp/customer-platform"), + } + ], + ) + self.assertTrue(state_has_resumable_services(runtime, state)) + def test_context_identity_sorting_and_weak_keys_are_stable(self) -> None: identities = project_identities_from_contexts( [ diff --git a/tests/python/startup/test_run_reuse_support.py b/tests/python/startup/test_run_reuse_support.py index 687ff85bc..8aa9a1255 100644 --- a/tests/python/startup/test_run_reuse_support.py +++ b/tests/python/startup/test_run_reuse_support.py @@ -28,7 +28,7 @@ def test_run_reuse_debug_orch_groups_only_apply_to_plan_commands(self) -> None: self.assertEqual(run_reuse_debug_orch_groups(runtime, requested_command="plan"), {"alpha", "beta", "gamma"}) self.assertEqual(run_reuse_debug_orch_groups(runtime, requested_command="start"), set()) - def test_dashboard_stopped_service_entries_normalizes_valid_backend_frontend_entries(self) -> None: + def test_dashboard_stopped_service_entries_normalizes_all_valid_service_types(self) -> None: state = SimpleNamespace( metadata={ "dashboard_stopped_services": [ @@ -46,6 +46,7 @@ def test_dashboard_stopped_service_entries_normalizes_valid_backend_frontend_ent [ {"project": "Main", "type": "frontend", "name": "Main Frontend"}, {"project": "Main", "type": "backend", "name": "Main Backend"}, + {"project": "Main", "type": "worker", "name": "Main Worker"}, ], ) @@ -95,6 +96,24 @@ def test_metadata_without_dashboard_stopped_services_drops_key_when_all_entries_ {"keep": True}, ) + def test_metadata_without_dashboard_stopped_services_is_project_scoped(self) -> None: + entries = [ + { + "name": f"{project} Voice Runtime", + "project": project, + "type": "voice-runtime", + } + for project in ("Customer Platform", "Other") + ] + + updated = metadata_without_dashboard_stopped_services( + {"dashboard_stopped_services": entries, "keep": True}, + restored_service_types_by_project={"Customer Platform": {"voice-runtime"}}, + ) + + self.assertEqual(updated["dashboard_stopped_services"], [entries[1]]) + self.assertTrue(updated["keep"]) + def test_prepare_dashboard_stopped_service_restore_rewrites_route_and_preserves_active_state(self) -> None: route = Route(command="start", mode="trees", raw_args=["start"], flags={}) session = SimpleNamespace( @@ -113,6 +132,7 @@ def test_prepare_dashboard_stopped_service_restore_rewrites_route_and_preserves_ metadata={ "dashboard_stopped_services": [ {"project": "Main", "type": "frontend", "name": "Main Frontend"}, + {"project": "Main", "type": "voice-runtime", "name": "Main Voice Runtime"}, {"project": "Other", "type": "backend", "name": "Other Backend"}, ], "keep": True, @@ -136,12 +156,28 @@ def test_prepare_dashboard_stopped_service_restore_rewrites_route_and_preserves_ self.assertEqual(session.preserved_services, candidate_state.services) self.assertEqual(session.preserved_requirements, candidate_state.requirements) self.assertEqual(session.base_metadata["keep"], True) - self.assertNotIn("dashboard_stopped_services", session.base_metadata) + self.assertEqual( + session.base_metadata["dashboard_stopped_services"], + candidate_state.metadata["dashboard_stopped_services"], + ) self.assertEqual(session.effective_route.projects, ["Main", "Other"]) self.assertEqual(session.effective_route.flags["_restart_request"], True) self.assertEqual(session.effective_route.flags["_restore_dashboard_stopped_services"], True) - self.assertEqual(session.effective_route.flags["services"], ["Main Frontend", "Other Backend"]) - self.assertEqual(session.effective_route.flags["restart_service_types"], ["backend", "frontend"]) + self.assertEqual( + session.effective_route.flags["services"], + ["Main Frontend", "Main Voice Runtime", "Other Backend"], + ) + self.assertEqual( + session.effective_route.flags["restart_service_types"], + ["backend", "frontend", "voice-runtime"], + ) + self.assertEqual( + session.effective_route.flags["_restart_service_types_by_project"], + { + "Main": ["frontend", "voice-runtime"], + "Other": ["backend"], + }, + ) self.assertEqual(phases[0][0], "auto_resume_evaluate") self.assertEqual(phases[0][1]["status"], "restore_stopped_services") self.assertEqual(phases[0][1]["match_mode"], "subset") @@ -298,7 +334,7 @@ def test_replace_existing_project_services_for_fresh_start_terminates_selected_s runtime = SimpleNamespace( _emit=lambda event, **payload: events.append((event, payload)), _terminate_services_from_state=lambda state, *, selected_services, aggressive, verify_ownership: ( - terminated.append((state, set(selected_services), aggressive, verify_ownership)) + terminated.append((state, set(selected_services), aggressive, verify_ownership)) or set() ), _project_name_from_service=lambda name: str(name).removesuffix(" Backend").removesuffix(" Frontend"), _release_requirement_ports=lambda requirements: released.append(requirements), @@ -314,7 +350,7 @@ def test_replace_existing_project_services_for_fresh_start_terminates_selected_s report_progress=lambda route, message: progress.append((route, message)), terminate_restart_orphan_listeners=lambda *, state, selected_services, aggressive: orphaned.append( (state, set(selected_services), aggressive) - ), + ) or set(), ) self.assertEqual(announced, [session]) @@ -363,7 +399,7 @@ def test_explicit_no_resume_replaces_matching_services_instead_of_leaking_a_seco _emit=lambda *_args, **_kwargs: None, _terminate_services_from_state=lambda _state, *, selected_services, **_kwargs: terminated.append( set(selected_services) - ), + ) or set(), _project_name_from_service=lambda name: str(name).removesuffix(" Backend"), _release_requirement_ports=lambda _requirements: None, ) @@ -375,7 +411,7 @@ def test_explicit_no_resume_replaces_matching_services_instead_of_leaking_a_seco reason="explicit_no_resume", announce_session_identifiers=lambda _session: None, report_progress=lambda _route, message: progress.append(message), - terminate_restart_orphan_listeners=lambda **_kwargs: None, + terminate_restart_orphan_listeners=lambda **_kwargs: set(), ) self.assertEqual(terminated, [{"feature-a Backend"}]) @@ -415,7 +451,7 @@ def test_explicit_no_resume_rebinds_reserved_plan_to_released_service_port(self) ), _emit=lambda *_args, **_kwargs: None, _project_name_from_service=lambda _name: "Main", - _terminate_services_from_state=lambda *_args, **_kwargs: None, + _terminate_services_from_state=lambda *_args, **_kwargs: set(), _release_requirement_ports=lambda _requirements: None, ) @@ -426,7 +462,7 @@ def test_explicit_no_resume_rebinds_reserved_plan_to_released_service_port(self) reason="explicit_no_resume", announce_session_identifiers=lambda _session: None, report_progress=lambda _route, _message: None, - terminate_restart_orphan_listeners=lambda **_kwargs: None, + terminate_restart_orphan_listeners=lambda **_kwargs: set(), ) self.assertEqual(calls, [("release", 8001, "Main:backend")]) @@ -519,10 +555,138 @@ def test_explicit_no_resume_failure_keeps_previous_state_authoritative(self) -> terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("must not reap after failure"), ) - self.assertEqual(set(session.preserved_services), {"sentinel"}) - self.assertEqual(set(session.preserved_requirements), {"sentinel"}) - self.assertEqual(session.base_metadata, {"sentinel": True}) + self.assertEqual(set(session.preserved_services), {"sentinel", "feature-a Backend"}) + self.assertEqual(set(session.preserved_requirements), {"sentinel", "feature-a"}) + self.assertTrue(session.base_metadata["sentinel"]) + self.assertTrue(session.base_metadata["keep"]) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-existing"]) + self.assertTrue(session.preserve_existing_state_on_failure) + + def test_explicit_no_resume_missing_termination_result_keeps_previous_state_authoritative(self) -> None: + route = Route(command="start", mode="trees", flags={"no_resume": True}) + service = SimpleNamespace(project="feature-a") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": service}, + requirements={"feature-a": SimpleNamespace(project="feature-a")}, + metadata={}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "feature-a", + _terminate_services_from_state=lambda *_args, **_kwargs: None, + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + ) + + with self.assertRaisesRegex(RuntimeError, "feature-a Backend"): + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda *_args: None, + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("orphans must not be reaped"), + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, candidate_state.services) + self.assertEqual(session.preserved_requirements, candidate_state.requirements) + + def test_fresh_start_preserves_source_authority_when_termination_raises(self) -> None: + service = SimpleNamespace(project="feature-a") + requirements = SimpleNamespace(project="feature-a") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": service}, + requirements={"feature-a": requirements}, + metadata={"authority": "old"}, + ) + session = SimpleNamespace( + effective_route=Route(command="start", mode="trees", flags={"no_resume": True}), + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + + def fail_termination(*_args, **_kwargs): # noqa: ANN202 + raise OSError("service lock unlink failed") + + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "feature-a", + _terminate_services_from_state=fail_termination, + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + ) + + with self.assertRaisesRegex(OSError, "service lock unlink failed"): + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda *_args: None, + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("orphans must not be reaped"), + ) + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, candidate_state.services) + self.assertEqual(session.preserved_requirements, candidate_state.requirements) + self.assertEqual(session.base_metadata["authority"], "old") + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-existing"]) + + def test_fresh_start_preserves_source_authority_when_requirement_release_raises(self) -> None: + requirements = SimpleNamespace(project="feature-a") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={}, + requirements={"feature-a": requirements}, + metadata={"authority": "old"}, + ) + session = SimpleNamespace( + effective_route=Route(command="start", mode="trees", flags={"no_resume": True}), + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + + def fail_release(_requirements): # noqa: ANN001, ANN202 + raise OSError("requirement lock unlink failed") + + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "", + _terminate_services_from_state=lambda *_args, **_kwargs: self.fail("no services to terminate"), + _release_requirement_ports=fail_release, + ) + + with self.assertRaisesRegex(OSError, "requirement lock unlink failed"): + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: self.fail("no announcement expected"), + report_progress=lambda *_args: self.fail("no progress expected"), + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("no orphan scan expected"), + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_requirements, candidate_state.requirements) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-existing"]) def test_fresh_start_aborts_when_orphan_exit_is_unconfirmed(self) -> None: route = Route(command="start", mode="trees", flags={"no_resume": True}) @@ -559,8 +723,8 @@ def test_fresh_start_aborts_when_orphan_exit_is_unconfirmed(self) -> None: ) self.assertTrue(session.preserve_existing_state_on_failure) - self.assertEqual(session.preserved_services, {}) - self.assertEqual(session.preserved_requirements, {}) + self.assertEqual(session.preserved_services, candidate_state.services) + self.assertEqual(session.preserved_requirements, candidate_state.requirements) def test_explicit_no_resume_supersedes_requirement_only_state(self) -> None: route = Route( diff --git a/tests/python/startup/test_selected_context_startup.py b/tests/python/startup/test_selected_context_startup.py index a3619d347..e6bbc79bb 100644 --- a/tests/python/startup/test_selected_context_startup.py +++ b/tests/python/startup/test_selected_context_startup.py @@ -1,9 +1,11 @@ from __future__ import annotations from contextlib import nullcontext +import concurrent.futures from pathlib import Path from types import SimpleNamespace import unittest +from unittest.mock import patch from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.service_manager import ServiceCleanupError @@ -13,7 +15,12 @@ start_selected_contexts, start_selected_contexts_with_runtime, ) -from envctl_engine.startup.session import ProjectStartupResult, StartupSession, track_unterminated_services +from envctl_engine.startup.session import ( + ProjectStartupResult, + StartupSession, + track_unterminated_services, + unconfirmed_service_names, +) from envctl_engine.state.models import RequirementsResult, ServiceRecord @@ -79,6 +86,12 @@ def _start_project_context(self, *, context, mode: str, route: Route, run_id: st class SelectedContextStartupTests(unittest.TestCase): + def test_blank_termination_identity_fails_closed_for_all_services(self) -> None: + services = {"Main Backend": object(), "Main Frontend": object()} + + self.assertEqual(unconfirmed_service_names([" "], services), set(services)) + self.assertEqual(unconfirmed_service_names(["", "Main Backend"], services), set(services)) + def test_parallel_renderer_error_occurs_only_after_successful_resources_are_recorded(self) -> None: runtime = _RuntimeStub() runtime._tree_parallel_startup_config = ( # type: ignore[method-assign] @@ -109,6 +122,114 @@ def test_parallel_renderer_error_occurs_only_after_successful_resources_are_reco self.assertEqual(set(session.requirements_by_project), {"Alpha", "Beta"}) self.assertEqual(set(session.started_context_names), {"Alpha", "Beta"}) + def test_sequential_record_callback_interrupt_retains_returned_resources(self) -> None: + runtime = _RuntimeStub() + context = self._context("Alpha") + requirements = RequirementsResult(project=context.name) + service = ServiceRecord( + name="Alpha Backend", + type="backend", + cwd="/alpha", + pid=8101, + ) + runtime._start_project_context = lambda **_kwargs: ProjectStartupResult( # type: ignore[method-assign] + requirements=requirements, + services={service.name: service}, + ) + route = Route(command="start", mode="trees", flags={}) + session = self._session(route=route, contexts=[context]) + cancellation = KeyboardInterrupt("cancel during result handoff") + + with self.assertRaises(KeyboardInterrupt) as raised: + start_selected_contexts( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: False, + resolved_run_id=lambda _session: "run-1", + record_project_startup=lambda *_args: (_ for _ in ()).throw(cancellation), + render_project_startup_warnings=lambda **_kwargs: None, + should_degrade_to_plan_agent_handoff=lambda _session, _error: False, + record_plan_agent_handoff_local_startup_failure=lambda **_kwargs: None, + spinner_factory=_SpinnerFactory(), + use_spinner_policy_fn=lambda _policy: nullcontext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False, backend="rich"), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + project_spinner_group_factory=lambda **_kwargs: SimpleNamespace(), + ) + + self.assertIs(raised.exception, cancellation) + self.assertIs(session.requirements_by_project[context.name], requirements) + self.assertEqual( + {tracked.pid for tracked in session.unterminated_services.values()}, + {service.pid}, + ) + + def test_parallel_main_thread_interrupt_drains_every_successful_future(self) -> None: + runtime = _RuntimeStub() + runtime._tree_parallel_startup_config = lambda **_kwargs: (True, 3) # type: ignore[method-assign] + contexts = [self._context(name) for name in ("Alpha", "Beta", "Gamma")] + expected_pids = {context.name: 8200 + index for index, context in enumerate(contexts, start=1)} + + def start_context(*, context, **_kwargs): # noqa: ANN001, ANN202 + service = ServiceRecord( + name=f"{context.name} Backend", + type="backend", + cwd=f"/{context.name.lower()}", + pid=expected_pids[context.name], + ) + return ProjectStartupResult( + requirements=RequirementsResult(project=context.name), + services={service.name: service}, + ) + + runtime._start_project_context = start_context # type: ignore[method-assign] + route = Route(command="start", mode="trees", flags={}) + session = self._session(route=route, contexts=contexts) + cancellation = KeyboardInterrupt("cancel while collecting parallel results") + + def interrupting_as_completed(futures): # noqa: ANN001, ANN202 + done, _ = concurrent.futures.wait( + futures, + return_when=concurrent.futures.FIRST_COMPLETED, + ) + yield next(iter(done)) + raise cancellation + + with ( + patch( + "envctl_engine.startup.selected_context_startup.concurrent.futures.as_completed", + side_effect=interrupting_as_completed, + ), + self.assertRaises(KeyboardInterrupt) as raised, + ): + start_selected_contexts( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: False, + resolved_run_id=lambda _session: "run-1", + record_project_startup=record_project_startup, + render_project_startup_warnings=lambda **_kwargs: None, + should_degrade_to_plan_agent_handoff=lambda _session, _error: False, + record_plan_agent_handoff_local_startup_failure=lambda **_kwargs: None, + spinner_factory=_SpinnerFactory(), + use_spinner_policy_fn=lambda _policy: nullcontext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False, backend="rich"), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + project_spinner_group_factory=lambda **_kwargs: SimpleNamespace(), + ) + + self.assertIs(raised.exception, cancellation) + tracked_pids = { + service.pid + for project_services in session.services_by_project.values() + for service in project_services.values() + }.union(service.pid for service in session.unterminated_services.values()) + self.assertEqual(tracked_pids, set(expected_pids.values())) + self.assertEqual( + {requirements.project for requirements in session.requirements_by_project.values()}, + set(expected_pids), + ) + def test_parallel_failure_callback_error_does_not_hide_later_success(self) -> None: runtime = _RuntimeStub() runtime._tree_parallel_startup_config = lambda **_kwargs: (True, 2) # type: ignore[method-assign] @@ -153,6 +274,64 @@ def emit(event: str, **payload: object) -> None: self.assertIn("Beta", session.services_by_project) self.assertIn("Beta", session.requirements_by_project) + def test_parallel_name_collision_drains_and_records_every_completed_future(self) -> None: + runtime = _RuntimeStub() + runtime._tree_parallel_startup_config = lambda **_kwargs: (True, 3) # type: ignore[method-assign] + + def start_context(*, context, **_kwargs): # noqa: ANN001, ANN202 + if context.name == "Gamma": + service = ServiceRecord( + name="Gamma Worker", + type="worker", + cwd="/gamma", + pid=303, + project="Gamma", + service_slug="worker", + ) + else: + service = ServiceRecord( + name="Opaque Shared Runtime", + type="worker", + cwd=f"/{context.name.lower()}", + pid=101 if context.name == "Alpha" else 202, + project=context.name, + service_slug="worker", + ) + return ProjectStartupResult( + requirements=RequirementsResult(project=context.name), + services={service.name: service}, + ) + + runtime._start_project_context = start_context # type: ignore[method-assign] + route = Route(command="start", mode="trees", flags={}) + contexts = [self._context(name) for name in ("Alpha", "Beta", "Gamma")] + session = self._session(route=route, contexts=contexts) + + with self.assertRaisesRegex(RuntimeError, "tracked startup state"): + start_selected_contexts( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: False, + resolved_run_id=lambda _session: "run-1", + record_project_startup=record_project_startup, + render_project_startup_warnings=lambda **_kwargs: None, + should_degrade_to_plan_agent_handoff=lambda _session, _error: False, + record_plan_agent_handoff_local_startup_failure=lambda **_kwargs: None, + spinner_factory=_SpinnerFactory(), + use_spinner_policy_fn=lambda _policy: nullcontext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False, backend="rich"), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + project_spinner_group_factory=lambda **_kwargs: SimpleNamespace(), + ) + + tracked = { + service.pid + for services in session.services_by_project.values() + for service in services.values() + }.union(service.pid for service in session.unterminated_services.values()) + self.assertEqual(tracked, {101, 202, 303}) + self.assertIn("Gamma", session.requirements_by_project) + def test_unterminated_service_error_is_carried_into_session_state(self) -> None: route = Route(command="start", mode="main", flags={}) session = self._session(route=route, contexts=[]) @@ -164,6 +343,152 @@ def test_unterminated_service_error_is_carried_into_session_state(self) -> None: self.assertIs(session.unterminated_services[service.name], service) self.assertIs(session.merged_services[service.name], service) + def test_parallel_failure_records_with_duplicate_names_keep_both_processes(self) -> None: + route = Route(command="start", mode="trees", flags={}) + session = self._session(route=route, contexts=[]) + alpha = ServiceRecord( + name="Opaque Worker", + type="worker", + cwd="/alpha", + pid=111, + project="Alpha", + service_slug="worker", + ) + beta = ServiceRecord( + name="Opaque Worker", + type="worker", + cwd="/beta", + pid=222, + project="Beta", + service_slug="worker", + ) + + track_unterminated_services(session, ServiceCleanupError("alpha cleanup failed", {alpha.name: alpha})) + track_unterminated_services(session, ServiceCleanupError("beta cleanup failed", {beta.name: beta})) + + self.assertEqual({service.pid for service in session.unterminated_services.values()}, {111, 222}) + self.assertEqual(len(session.unterminated_services), 2) + self.assertTrue(any("Restart Collision" in name for name in session.unterminated_services)) + + def test_merged_services_rejects_preserved_new_name_collision(self) -> None: + route = Route(command="restart", mode="main", flags={}) + session = self._session(route=route, contexts=[]) + preserved = ServiceRecord(name="Main Backend", type="backend", cwd="/repo/old", pid=63001) + replacement = ServiceRecord(name="Main Backend", type="backend", cwd="/repo/new", pid=63002) + session.preserved_services[preserved.name] = preserved + session.services_by_project["Main"] = {replacement.name: replacement} + + with self.assertRaisesRegex( + RuntimeError, + "Refusing to overwrite preserved service state.*Main Backend", + ): + _ = session.merged_services + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.service_state_collisions, {"Main Backend"}) + + def test_merged_requirements_rejects_preserved_new_owner_collision(self) -> None: + route = Route(command="restart", mode="main", flags={}) + session = self._session(route=route, contexts=[]) + session.preserved_requirements["Main"] = RequirementsResult( + project="Main", + redis={"enabled": True, "final": 6380}, + ) + session.requirements_by_project["Main"] = RequirementsResult( + project="Main", + redis={"enabled": True, "final": 6381}, + ) + + with self.assertRaisesRegex(RuntimeError, "preserved requirement state.*Main"): + _ = session.merged_requirements + + self.assertTrue(session.preserve_existing_state_on_failure) + + def test_merged_requirements_deduplicates_reused_alias_authority_by_identity(self) -> None: + route = Route(command="restart", mode="main", flags={}) + session = self._session(route=route, contexts=[]) + requirements = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6381}, + ) + session.preserved_requirements["Main Restart Collision"] = requirements + session.requirements_by_project["Main"] = requirements + + merged = session.merged_requirements + + self.assertEqual(list(merged), ["Main Restart Collision"]) + self.assertIs(merged["Main Restart Collision"], requirements) + + def test_merged_requirements_ignores_disabled_projection_when_dependencies_are_skipped(self) -> None: + route = Route(command="start", mode="main", flags={"launch_dependencies": False}) + session = self._session(route=route, contexts=[]) + preserved = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6379}, + ) + session.preserved_requirements["Main Historical Authority"] = preserved + session.requirements_by_project["Main"] = RequirementsResult( + project="Main", + redis={"enabled": False, "success": True, "final": 6482}, + ) + + merged = session.merged_requirements + + self.assertEqual(list(merged), ["Main Historical Authority"]) + self.assertIs(merged["Main Historical Authority"], preserved) + + def test_record_project_startup_tracks_collision_for_failure_cleanup(self) -> None: + route = Route(command="restart", mode="main", flags={}) + context = self._context("Main") + session = self._session(route=route, contexts=[context]) + preserved = ServiceRecord(name="Main Backend", type="backend", cwd="/repo/old", pid=63001) + replacement = ServiceRecord(name="Main Backend", type="backend", cwd="/repo/new", pid=63002) + session.preserved_services[preserved.name] = preserved + requirements = RequirementsResult(project="Main") + + with self.assertRaisesRegex(RuntimeError, "Refusing to overwrite preserved service state"): + record_project_startup( + session, + context, + ProjectStartupResult(requirements=requirements, services={replacement.name: replacement}), + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + replacement_names = [ + name for name in session.unterminated_services if name.startswith("Main Backend Restart Collision") + ] + self.assertEqual(len(replacement_names), 1) + self.assertEqual(session.unterminated_services[replacement_names[0]].pid, replacement.pid) + self.assertEqual(session.unterminated_services[replacement_names[0]].name, replacement_names[0]) + self.assertIs(session.requirements_by_project["Main"], requirements) + self.assertEqual(session.services_by_project, {}) + + def test_duplicate_context_name_with_disjoint_services_retains_both_startups(self) -> None: + route = Route(command="start", mode="trees", flags={}) + context = self._context("Same") + session = self._session(route=route, contexts=[context, context]) + first = ServiceRecord(name="Opaque A", type="worker", cwd="/one", pid=1, project="Same") + second = ServiceRecord(name="Opaque B", type="worker", cwd="/two", pid=2, project="Same") + first_requirements = RequirementsResult(project="Same", redis={"enabled": True, "final": 6379}) + second_requirements = RequirementsResult(project="Same", redis={"enabled": True, "final": 6381}) + + record_project_startup( + session, + context, + ProjectStartupResult(requirements=first_requirements, services={first.name: first}), + ) + with self.assertRaisesRegex(RuntimeError, "project:Same"): + record_project_startup( + session, + context, + ProjectStartupResult(requirements=second_requirements, services={second.name: second}), + ) + + self.assertEqual({service.pid for service in session.services_by_project["Same"].values()}, {1}) + self.assertEqual({service.pid for service in session.unterminated_services.values()}, {2}) + self.assertEqual(set(session.requirements_by_project), {"Same", "Same Restart Collision"}) + self.assertIs(session.requirements_by_project["Same"], first_requirements) + self.assertIs(session.requirements_by_project["Same Restart Collision"], second_requirements) + def test_project_startup_failure_carries_requirements_into_failure_state(self) -> None: route = Route(command="start", mode="main", flags={}) session = self._session(route=route, contexts=[]) diff --git a/tests/python/startup/test_service_attach_execution.py b/tests/python/startup/test_service_attach_execution.py index 78eeafc16..dcbdf37b0 100644 --- a/tests/python/startup/test_service_attach_execution.py +++ b/tests/python/startup/test_service_attach_execution.py @@ -12,6 +12,7 @@ from envctl_engine.runtime.docker_service_runtime import DockerServiceLaunch from envctl_engine.runtime.service_manager import ServiceCleanupError, ServiceManager, ServiceStartDescriptor from envctl_engine.shared.ports import PortPlanner +from envctl_engine.shared.process_runner import ProcessHandoffCleanupError from envctl_engine.startup.service_attach_execution import ServiceAttachRunner from envctl_engine.startup.service_execution_records import PreparedServiceLaunch from envctl_engine.state.models import PortPlan, ServiceRecord @@ -270,6 +271,60 @@ def test_process_spawn_error_is_structured_without_listener_wait(self) -> None: failure_events = [payload for event, payload in events if event == "service.failure"] self.assertEqual(failure_events[0]["failure_class"], "process_spawn_failed") + def test_unconfirmed_process_handoff_returns_pid_authority_to_service_manager(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + process = SimpleNamespace(pid=62002) + runtime = SimpleNamespace( + _conflict_remaining={}, + _emit=lambda event, **payload: events.append((event, payload)), + _service_start_command_resolved=lambda **_kwargs: (["backend"], "configured"), + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace( + start_background=lambda *_args, **_kwargs: (_ for _ in ()).throw( + ProcessHandoffCleanupError( + process=process, + launch_intent="background_service", + ) + ) + ), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="Main", + project_root=Path("/repo"), + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=Path("/repo/backend"), + frontend_cwd=Path("/repo/frontend"), + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=Path("/repo/backend"), + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + success, error, pid = runner.start_backend(8000) + + self.assertFalse(success) + self.assertIn("cleanup could not confirm exit", error) + self.assertEqual(pid, 62002) + failure_events = [payload for event, payload in events if event == "service.failure"] + self.assertEqual(failure_events[0]["failure_class"], "process_cleanup_unconfirmed") + def test_explicit_docker_mode_launches_and_annotates_container_service(self) -> None: project_root = Path("/tmp/envctl-project") runtime = SimpleNamespace( @@ -450,6 +505,97 @@ def test_failed_optional_docker_readiness_removes_container_before_degrading(sel ) self.assertEqual(runner._container_launches, {"voice-runtime": launch}) + def test_failed_critical_docker_cleanup_retains_container_identity_and_port_lock(self) -> None: + project_root = Path("/tmp/envctl-project") + releases: list[tuple[int, str]] = [] + runtime = SimpleNamespace( + env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}, + config=SimpleNamespace(raw={}), + services=ServiceManager(), + _conflict_remaining={}, + _emit=lambda *_args, **_kwargs: None, + _service_start_command_resolved=lambda **_kwargs: [], + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(), + port_allocator=SimpleNamespace( + session_id="planner-session-docker", + reserve_next=lambda port, owner: port, + release=lambda port, *, owner: releases.append((port, owner)), + ), + project_name="Main", + project_root=project_root, + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=project_root / "backend", + frontend_cwd=project_root / "frontend", + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: {**extra, "PORT": str(port)}, + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=project_root / "backend", + log_path="/logs/backend.txt", + requested_port=8000, + env={"PORT": "8000"}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + docker_mode=True, + ) + launch = DockerServiceLaunch( + container_id="live-container-id", + container_name="envctl-app-main-backend", + image="example/backend:dev", + log_path="/logs/backend.txt", + launch_token="launch-token-live", + cleanup_pending_since=1_700_000_000.25, + ) + + with ( + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.launch", + return_value=launch, + ), + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.wait_until_ready", + return_value=False, + ), + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.stop", + return_value=False, + ) as stop_container, + self.assertRaises(ServiceCleanupError) as raised, + ): + runner.start(attach_parallel=False, on_service_retry=lambda *_args: None) + + self.assertEqual(stop_container.call_count, 2) + self.assertTrue( + all( + call.kwargs["pending_cleanup_since"] == 1_700_000_000.25 + for call in stop_container.call_args_list + ) + ) + self.assertEqual(runner._container_launches, {"backend": launch}) + self.assertEqual(releases, []) + record = raised.exception.unterminated_services["Main Backend"] + self.assertEqual(record.runtime_kind, "docker") + self.assertEqual(record.container_id, "live-container-id") + self.assertEqual(record.container_name, "envctl-app-main-backend") + self.assertEqual(record.container_image, "example/backend:dev") + self.assertEqual(record.container_launch_token, "launch-token-live") + self.assertEqual(record.container_cleanup_pending_since, 1_700_000_000.25) + self.assertEqual(record.actual_port, 8000) + self.assertEqual(record.port_lock_session, "planner-session-docker") + def test_runner_builds_layered_descriptors_and_preserves_additional_urls(self) -> None: project_root = Path("/tmp/envctl-project") events: list[tuple[str, dict[str, object]]] = [] diff --git a/tests/python/startup/test_service_manager.py b/tests/python/startup/test_service_manager.py index 1c126d3c8..c21eb108d 100644 --- a/tests/python/startup/test_service_manager.py +++ b/tests/python/startup/test_service_manager.py @@ -1,5 +1,7 @@ from __future__ import annotations +import concurrent.futures +import threading import time import unittest from pathlib import Path @@ -17,6 +19,19 @@ class ServiceManagerTests(unittest.TestCase): + @staticmethod + def _tracking_manager(live_pids: set[int], terminated: list[int]) -> ServiceManager: + return ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + terminate_process_group=lambda pid, **_kwargs: ( + terminated.append(pid), + live_pids.discard(pid), + True, + )[-1], + ) + ) + def test_pidless_runtime_descriptor_records_success_without_inventing_a_process(self) -> None: records = ServiceManager().start_services_with_attach( project="Main", @@ -393,6 +408,178 @@ def test_start_services_with_attach_cleans_partial_records_on_failure(self) -> N self.assertEqual(terminated, [41001]) + def test_detection_interrupt_terminates_launched_pid_and_preserves_cancellation(self) -> None: + live_pids: set[int] = set() + terminated: list[int] = [] + manager = self._tracking_manager(live_pids, terminated) + cancellation = KeyboardInterrupt("listener probe interrupted") + + def start(_port: int) -> tuple[bool, str | None, int | None]: + live_pids.add(61001) + return True, None, 61001 + + with self.assertRaises(KeyboardInterrupt) as raised: + manager.start_service_with_retry( + project="Main", + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=start, + reserve_next=lambda port: port, + detect_actual=lambda *_args: (_ for _ in ()).throw(cancellation), + ) + + self.assertIs(raised.exception, cancellation) + self.assertEqual(terminated, [61001]) + self.assertEqual(live_pids, set()) + + def test_sequential_interrupt_cleans_prior_service_and_preserves_cancellation(self) -> None: + live_pids: set[int] = set() + terminated: list[int] = [] + manager = self._tracking_manager(live_pids, terminated) + cancellation = KeyboardInterrupt("second descriptor interrupted") + + def start_backend(_port: int) -> tuple[bool, str | None, int | None]: + live_pids.add(62001) + return True, None, 62001 + + descriptors = ( + ServiceStartDescriptor( + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=start_backend, + detect_actual=lambda _pid, requested: requested, + ), + ServiceStartDescriptor( + service_type="frontend", + cwd="/repo/frontend", + requested_port=3000, + start=lambda _port: (_ for _ in ()).throw(cancellation), + ), + ) + + with self.assertRaises(KeyboardInterrupt) as raised: + manager.start_services_with_attach( + project="Main", + descriptors=descriptors, + reserve_next=lambda port: port, + ) + + self.assertIs(raised.exception, cancellation) + self.assertEqual(terminated, [62001]) + self.assertEqual(live_pids, set()) + + def test_parallel_interrupt_drains_futures_and_cleans_every_launched_service(self) -> None: + live_pids: set[int] = set() + terminated: list[int] = [] + manager = self._tracking_manager(live_pids, terminated) + cancellation = KeyboardInterrupt("parallel descriptor interrupted") + interrupt_started = threading.Event() + + def start_service(pid: int) -> tuple[bool, str | None, int | None]: + live_pids.add(pid) + return True, None, pid + + def interrupt_detection(_pid: int | None, _requested: int) -> int: + interrupt_started.set() + raise cancellation + + def start_slow_sibling(_port: int) -> tuple[bool, str | None, int | None]: + self.assertTrue(interrupt_started.wait(timeout=1.0)) + time.sleep(0.02) + return start_service(63003) + + descriptors = ( + ServiceStartDescriptor( + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=lambda _port: start_service(63001), + detect_actual=lambda _pid, requested: requested, + ), + ServiceStartDescriptor( + service_type="voice-runtime", + cwd="/repo/voice-runtime", + requested_port=8010, + start=lambda _port: start_service(63002), + detect_actual=interrupt_detection, + ), + ServiceStartDescriptor( + service_type="frontend", + cwd="/repo/frontend", + requested_port=3000, + start=start_slow_sibling, + detect_actual=lambda _pid, requested: requested, + ), + ) + + with self.assertRaises(KeyboardInterrupt) as raised: + manager.start_services_with_attach( + project="Main", + descriptors=descriptors, + reserve_next=lambda port: port, + parallel_start=True, + max_workers=3, + ) + + self.assertIs(raised.exception, cancellation) + self.assertEqual(set(terminated), {63001, 63002, 63003}) + self.assertEqual(live_pids, set()) + + def test_main_thread_parallel_interrupt_drains_late_success_before_cleanup(self) -> None: + live_pids: set[int] = set() + terminated: list[int] = [] + manager = self._tracking_manager(live_pids, terminated) + cancellation = KeyboardInterrupt("main thread interrupted") + launch_barrier = threading.Barrier(2) + + def start_service(pid: int, *, delay: float = 0.0) -> tuple[bool, str | None, int | None]: + live_pids.add(pid) + launch_barrier.wait(timeout=1.0) + if delay: + time.sleep(delay) + return True, None, pid + + descriptors = ( + ServiceStartDescriptor( + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=lambda _port: start_service(64001), + detect_actual=lambda _pid, requested: requested, + ), + ServiceStartDescriptor( + service_type="frontend", + cwd="/repo/frontend", + requested_port=3000, + start=lambda _port: start_service(64002, delay=0.03), + detect_actual=lambda _pid, requested: requested, + ), + ) + real_as_completed = concurrent.futures.as_completed + + def interrupted_as_completed(futures): # noqa: ANN001, ANN202 + completed = iter(real_as_completed(futures)) + yield next(completed) + raise cancellation + + with ( + patch("envctl_engine.runtime.service_manager.concurrent.futures.as_completed", interrupted_as_completed), + self.assertRaises(KeyboardInterrupt) as raised, + ): + manager.start_services_with_attach( + project="Main", + descriptors=descriptors, + reserve_next=lambda port: port, + parallel_start=True, + max_workers=2, + ) + + self.assertIs(raised.exception, cancellation) + self.assertEqual(set(terminated), {64001, 64002}) + self.assertEqual(live_pids, set()) + def test_listener_failure_fails_closed_when_group_and_raw_cleanup_cannot_prove_exit(self) -> None: runner = SimpleNamespace( is_pid_running=lambda _pid: True, diff --git a/tests/python/startup/test_startup_finalization.py b/tests/python/startup/test_startup_finalization.py index db90245f4..3b2b34f0f 100644 --- a/tests/python/startup/test_startup_finalization.py +++ b/tests/python/startup/test_startup_finalization.py @@ -4,10 +4,12 @@ from types import SimpleNamespace import tempfile import unittest +from unittest.mock import patch from envctl_engine.config import load_config from envctl_engine.runtime.command_router import parse_route from envctl_engine.startup.finalization import ( + _write_artifacts_at_commit_boundary, emit_preserved_service_merge, failure_context_label, finalize_failed_startup, @@ -33,7 +35,9 @@ ) from envctl_engine.planning.plan_agent.models import CreatedPlanWorktree, PlanAgentLaunchOutcome, PlanAgentLaunchResult from envctl_engine.startup.finalization_failure import StartupFailureFinalizer -from envctl_engine.startup.session import LocalStartupFailure, StartupSession +from envctl_engine.startup.selected_context_startup import record_project_startup +from envctl_engine.startup.session import LocalStartupFailure, ProjectStartupResult, StartupSession +from envctl_engine.startup.lifecycle import _finalize_startup_exception, execute_startup_lifecycle from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord @@ -72,6 +76,160 @@ def _runtime_stub(**overrides: object) -> SimpleNamespace: class StartupFinalizationTests(unittest.TestCase): + def test_direct_collision_is_disambiguated_before_cleanup_without_prior_merge_access(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + preserved = ServiceRecord(name="Main Backend", type="backend", cwd="/old", pid=41, project="Main") + replacement = ServiceRecord(name="Main Backend", type="backend", cwd="/new", pid=42, project="Main") + occupied = ServiceRecord( + name="Main Backend Restart Collision 42", + type="worker", + cwd="/other", + pid=43, + project="Aux", + service_slug="worker", + ) + session.preserved_services[preserved.name] = preserved + session.services_by_project["Main"] = {replacement.name: replacement} + session.unterminated_services[occupied.name] = occupied + writes: list[RunState] = [] + cleanup_requests: list[dict[str, ServiceRecord]] = [] + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda services: ( + cleanup_requests.append(dict(services)), + set(services), + )[1], + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="collision", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual(len(cleanup_requests), 1) + self.assertEqual({service.pid for service in cleanup_requests[0].values()}, {42, 43}) + self.assertEqual({service.pid for service in writes[0].services.values()}, {41, 42, 43}) + replacement_names = [name for name, service in writes[0].services.items() if service.pid == 42] + self.assertEqual(replacement_names, ["Main Backend Restart Collision 42 2"]) + self.assertTrue( + any( + row.get("replacement_name") == replacement_names[0] + and row.get("replacement_exit_unconfirmed") is True + for row in writes[0].metadata["startup_state_collisions"] + ) + ) + + def test_duplicate_new_service_names_both_survive_unconfirmed_cleanup(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Alpha"), SimpleNamespace(name="Beta")]) + alpha = ServiceRecord(name="Shared Runtime", type="worker", cwd="/alpha", pid=101, project="Alpha") + beta = ServiceRecord(name="Shared Runtime", type="worker", cwd="/beta", pid=202, project="Beta") + record_project_startup( + session, + SimpleNamespace(name="Alpha"), + ProjectStartupResult(requirements=RequirementsResult(project="Alpha"), services={alpha.name: alpha}), + ) + with self.assertRaisesRegex(RuntimeError, "tracked startup state"): + record_project_startup( + session, + SimpleNamespace(name="Beta"), + ProjectStartupResult(requirements=RequirementsResult(project="Beta"), services={beta.name: beta}), + ) + writes: list[RunState] = [] + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda services: set(services), + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="duplicate hook identity", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual({service.pid for service in writes[0].services.values()}, {101, 202}) + self.assertEqual(len(writes[0].services), 2) + self.assertTrue(any("Restart Collision" in name for name in writes[0].services)) + + def test_collision_failure_persists_unconfirmed_replacement_without_overwriting_authority(self) -> None: + context = SimpleNamespace(name="Main") + session = _session(contexts=[context]) + preserved = ServiceRecord(name="Main Backend", type="backend", cwd="/repo/old", pid=64000) + replacement = ServiceRecord(name="Main Backend", type="backend", cwd="/repo/new", pid=64001) + session.preserved_services[preserved.name] = preserved + preserved_requirements = RequirementsResult( + project="Main", + db={"enabled": True, "success": True, "final": 5432}, + ) + replacement_requirements = RequirementsResult( + project="Main", + db={"enabled": True, "success": True, "final": 5544}, + ) + session.preserved_requirements["Main"] = preserved_requirements + with self.assertRaisesRegex(RuntimeError, "Refusing to overwrite preserved service state") as raised: + record_project_startup( + session, + context, + ProjectStartupResult( + requirements=replacement_requirements, + services={replacement.name: replacement}, + ), + ) + + terminated: list[dict[str, ServiceRecord]] = [] + writes: list[RunState] = [] + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda services: ( + terminated.append(dict(services)), + {replacement.name}, + )[1], + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + result = finalize_failed_startup( + runtime=runtime, + session=session, + error=str(raised.exception), + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual(result, 1) + self.assertEqual(len(terminated), 1) + terminated_names = list(terminated[0]) + self.assertEqual(len(terminated_names), 1) + self.assertTrue(terminated_names[0].startswith("Main Backend Restart Collision")) + self.assertEqual(terminated[0][terminated_names[0]].pid, replacement.pid) + self.assertEqual(len(writes), 1) + collision_names = [name for name in writes[0].services if "Restart Collision" in name] + self.assertEqual(len(collision_names), 1) + self.assertEqual(writes[0].services[preserved.name].pid, preserved.pid) + self.assertEqual(writes[0].services[collision_names[0]].pid, replacement.pid) + collision_projects = [project for project in writes[0].requirements if "Restart Collision" in project] + self.assertEqual(len(collision_projects), 1) + self.assertEqual(writes[0].requirements["Main"].db["final"], 5432) + self.assertEqual(writes[0].requirements[collision_projects[0]].db["final"], 5544) + self.assertEqual(writes[0].requirements[collision_projects[0]].project, "Main") + self.assertEqual( + writes[0].metadata["startup_state_collisions"][0]["replacement_name"], + collision_names[0], + ) + self.assertIs(session.preserved_services[preserved.name], preserved) + self.assertEqual(set(session.unterminated_services), set(collision_names)) + def test_failure_finalization_persists_service_when_exit_remains_unconfirmed(self) -> None: session = _session(contexts=[SimpleNamespace(name="Main")]) service = ServiceRecord( @@ -108,6 +266,31 @@ def test_failure_finalization_persists_service_when_exit_remains_unconfirmed(sel self.assertEqual(service.status, "termination_failed") self.assertFalse(released["value"]) + def test_unknown_termination_identity_fails_closed_for_every_started_service(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + backend = ServiceRecord(name="Main Backend", type="backend", cwd="/repo", pid=1) + frontend = ServiceRecord(name="Main Frontend", type="frontend", cwd="/repo", pid=2) + session.services_by_project["Main"] = {backend.name: backend, frontend.name: frontend} + writes: list[RunState] = [] + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: {"main backend"}, + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="cleanup adapter identity mismatch", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual(set(writes[0].services), {backend.name, frontend.name}) + self.assertEqual({service.status for service in writes[0].services.values()}, {"termination_failed"}) + def test_failure_finalization_preserves_port_session_for_started_requirements(self) -> None: session = _session(contexts=[SimpleNamespace(name="Main")]) requirements = RequirementsResult( @@ -477,6 +660,158 @@ def test_finalize_successful_startup_writes_artifacts_summary_snapshot_and_retur self.assertEqual(snapshots[0][0], "before_dashboard_entry") self.assertEqual(snapshots[0][1]["service_count"], 0) + def test_post_commit_finalization_error_marks_authority_committed_before_propagating(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + run_state = RunState(run_id="run-finalization", mode="trees") + writes: list[RunState] = [] + runtime = SimpleNamespace( + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(OSError, "phase sink unavailable"): + finalize_successful_startup( + runtime=runtime, + session=session, + ensure_run_id=lambda _session: None, + validate_plan_agent_handoff=lambda _session, *, phase: None, + build_success_run_state=lambda _runtime, _session: run_state, + emit_preserved_service_merge=lambda _session: None, + emit_phase=lambda *_args, **_kwargs: (_ for _ in ()).throw( + OSError("phase sink unavailable") + ), + requirements_timing_enabled=lambda _route: False, + suppress_timing_output=lambda _route: False, + print_startup_summary=lambda **_kwargs: None, + startup_breakdown_enabled=lambda _route: False, + suppress_progress_output=lambda _route: False, + print_restart_port_rebound_summary=lambda _session: None, + emit_snapshot=lambda *_args, **_kwargs: None, + headless_plan_output_only=lambda _session: False, + print_headless_plan_session_summary=lambda _session: None, + maybe_attach_plan_agent_terminal=lambda _session: None, + finalize_plan_agent_degraded_handoff=lambda _session: 1, + ) + + self.assertEqual(writes, [run_state]) + self.assertTrue(session.authority_committed) + + def test_artifact_writer_post_index_error_marks_authority_before_propagating(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + run_state = RunState(run_id="run-post-index", mode="main") + + def write_artifacts(_state, _contexts, *, errors, on_commit): # noqa: ANN001 + self.assertEqual(errors, []) + on_commit() + raise RuntimeError("alias promotion failed after index commit") + + runtime = SimpleNamespace( + _write_artifacts=write_artifacts, + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(RuntimeError, "alias promotion failed after index commit"): + finalize_successful_startup( + runtime=runtime, + session=session, + ensure_run_id=lambda _session: None, + validate_plan_agent_handoff=lambda _session, *, phase: None, + build_success_run_state=lambda _runtime, _session: run_state, + emit_preserved_service_merge=lambda _session: None, + emit_phase=lambda *_args, **_kwargs: None, + requirements_timing_enabled=lambda _route: False, + suppress_timing_output=lambda _route: False, + print_startup_summary=lambda **_kwargs: None, + startup_breakdown_enabled=lambda _route: False, + suppress_progress_output=lambda _route: False, + print_restart_port_rebound_summary=lambda _session: None, + emit_snapshot=lambda *_args, **_kwargs: None, + headless_plan_output_only=lambda _session: False, + print_headless_plan_session_summary=lambda _session: None, + maybe_attach_plan_agent_terminal=lambda _session: None, + finalize_plan_agent_degraded_handoff=lambda _session: 1, + ) + + self.assertTrue(session.authority_committed) + + def test_generic_kwargs_writer_is_never_probed_or_invoked_twice(self) -> None: + session = _session(contexts=[]) + run_state = RunState(run_id="run-proxy", mode="main") + calls: list[dict[str, object]] = [] + + def proxy_writer(*_args, **kwargs): # noqa: ANN003 + calls.append(dict(kwargs)) + + runtime = SimpleNamespace(_write_artifacts=proxy_writer) + + _write_artifacts_at_commit_boundary(runtime, session, run_state) + + self.assertEqual(calls, [{"errors": []}]) + self.assertTrue(session.authority_committed) + + def test_generic_kwargs_writer_type_error_is_not_retried(self) -> None: + session = _session(contexts=[]) + run_state = RunState(run_id="run-proxy-error", mode="main") + calls: list[dict[str, object]] = [] + + def proxy_writer(*_args, **kwargs): # noqa: ANN003 + calls.append(dict(kwargs)) + raise TypeError("unexpected keyword argument 'on_commit'") + + runtime = SimpleNamespace(_write_artifacts=proxy_writer) + + with self.assertRaisesRegex(TypeError, "unexpected keyword argument"): + _write_artifacts_at_commit_boundary(runtime, session, run_state) + + self.assertEqual(calls, [{"errors": []}]) + self.assertFalse(session.authority_committed) + + def test_committed_authority_exception_never_invokes_failure_finalizer(self) -> None: + session = _session(contexts=[]) + session.authority_committed = True + events: list[tuple[str, dict[str, object]]] = [] + runtime = SimpleNamespace(_emit=lambda event, **payload: events.append((event, payload))) + + result = _finalize_startup_exception( + runtime=runtime, + session=session, + error=OSError("terminal attach failed"), + finalize_failure=lambda **_kwargs: self.fail("committed authority must not be rewritten"), + ) + + self.assertEqual(result, 1) + self.assertEqual(events[-1][0], "startup.post_commit_error") + + def test_execute_startup_lifecycle_finalizes_then_reraises_keyboard_interrupt(self) -> None: + session = _session(contexts=[]) + runtime = SimpleNamespace() + orchestrator = SimpleNamespace(runtime=runtime) + cancellation = KeyboardInterrupt("cancel startup") + + def interrupt_phase(_session: StartupSession) -> None: + raise cancellation + + with ( + patch( + "envctl_engine.startup.lifecycle.create_startup_session", + return_value=session, + ), + patch( + "envctl_engine.startup.lifecycle._pre_start_phases", + return_value=(interrupt_phase,), + ), + patch( + "envctl_engine.startup.lifecycle.finalize_failed_startup", + return_value=1, + ) as finalize_failure, + self.assertRaises(KeyboardInterrupt) as raised, + ): + execute_startup_lifecycle(orchestrator, session.requested_route) + + self.assertIs(raised.exception, cancellation) + finalize_failure.assert_called_once() + self.assertEqual(finalize_failure.call_args.kwargs["error"], str(cancellation)) + def test_finalize_successful_startup_delegates_degraded_plan_agent_handoff(self) -> None: session = _session(contexts=[]) session.plan_agent_handoff_degraded = True diff --git a/tests/python/startup/test_startup_fingerprint_replacement.py b/tests/python/startup/test_startup_fingerprint_replacement.py index 86ab97806..ce8967986 100644 --- a/tests/python/startup/test_startup_fingerprint_replacement.py +++ b/tests/python/startup/test_startup_fingerprint_replacement.py @@ -81,7 +81,8 @@ def test_start_fingerprint_mismatch_replaces_existing_project_services_before_ne lambda state, selected_services, aggressive, verify_ownership: ( call_order.append("terminate"), terminate_calls.append((selected_services, verify_ownership)), - ) + set(), + )[-1] ) engine._start_project_context = ( # type: ignore[method-assign] lambda context, mode, route, run_id: ( diff --git a/tests/python/startup/test_startup_orchestrator_flow_reuse.py b/tests/python/startup/test_startup_orchestrator_flow_reuse.py index 63d80557e..5d3d8f8f4 100644 --- a/tests/python/startup/test_startup_orchestrator_flow_reuse.py +++ b/tests/python/startup/test_startup_orchestrator_flow_reuse.py @@ -36,6 +36,7 @@ def test_start_heals_multiple_historical_active_owners_for_main(self) -> None: ) engine.state_repository.run_index.replace_all(historical_candidates) + preserved_stale_locks: dict[str, str] = {} for offset, candidate in enumerate(historical_candidates): stale_lock = engine.port_planner.lock_dir / f"{44000 + offset}.lock" stale_lock.write_text( @@ -49,6 +50,7 @@ def test_start_heals_multiple_historical_active_owners_for_main(self) -> None: ), encoding="utf-8", ) + preserved_stale_locks[stale_lock.name] = stale_lock.read_text(encoding="utf-8") third_engine = self._engine(repo, runtime) with ( patch.object(third_engine, "_should_enter_post_start_interactive", return_value=False), @@ -65,7 +67,13 @@ def test_start_heals_multiple_historical_active_owners_for_main(self) -> None: set(index_payload["retired_run_ids"]), {candidate.run_id for candidate in historical_candidates}, ) - self.assertEqual(list(third_engine.port_planner.lock_dir.glob("*.lock")), []) + self.assertEqual( + { + path.name: path.read_text(encoding="utf-8") + for path in third_engine.port_planner.lock_dir.glob("*.lock") + }, + preserved_stale_locks, + ) def test_repeated_no_infra_start_supersedes_old_authority_without_port_locks(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: diff --git a/tests/python/startup/test_startup_orchestrator_profiles.py b/tests/python/startup/test_startup_orchestrator_profiles.py index 2fae84505..e3f3ff394 100644 --- a/tests/python/startup/test_startup_orchestrator_profiles.py +++ b/tests/python/startup/test_startup_orchestrator_profiles.py @@ -10,6 +10,7 @@ from envctl_engine.state.models import RunState, ServiceRecord from envctl_engine.runtime.engine_runtime_env import requirement_enabled_for_mode from envctl_engine.startup.finalization import build_success_run_state +from envctl_engine.startup.finalization_run_state import _build_run_state from envctl_engine.startup.session import StartupSession from envctl_engine.state.models import RequirementsResult from envctl_engine.startup.startup_selection_support import ( @@ -103,6 +104,84 @@ def test_build_run_state_writes_project_scoped_configured_services_for_multiple_ {"Alpha": ["frontend"], "Zeta": ["frontend"]}, ) + def test_build_run_state_includes_enabled_additional_services_per_project(self) -> None: + additional_services = ( + SimpleNamespace( + name="voice-runtime", + enabled_for_project_root=lambda _mode, _root: True, + ), + SimpleNamespace( + name="worker", + enabled_for_project_root=lambda _mode, root: Path(root).name == "Alpha", + ), + ) + runtime = SimpleNamespace( + config=SimpleNamespace(runtime_scope_id="scope-1", additional_services=additional_services), + _run_dir_path=lambda run_id: Path("/tmp/runtime") / run_id, + _service_enabled_for_mode=lambda _mode, service_name: service_name == "backend", + ) + contexts = [ + SimpleNamespace(name=name, root=Path(f"/tmp/repo/{name}"), ports={}) + for name in ("Alpha", "Beta") + ] + route = Route(command="up", mode="trees", flags={}) + session = StartupSession( + requested_route=route, + effective_route=route, + requested_command="up", + runtime_mode="trees", + run_id="run-1", + selected_contexts=contexts, + ) + + state = build_success_run_state(runtime, session) # type: ignore[arg-type] + + self.assertEqual( + state.metadata["dashboard_project_configured_services"], + { + "Alpha": ["backend", "voice-runtime", "worker"], + "Beta": ["backend", "voice-runtime"], + }, + ) + + def test_stopped_service_metadata_is_removed_only_after_project_success(self) -> None: + entries = [ + {"project": project, "type": "voice-runtime", "name": f"{project} Agent"} + for project in ("Main", "Other") + ] + runtime = SimpleNamespace( + config=SimpleNamespace(runtime_scope_id="scope-1", additional_services=()), + _run_dir_path=lambda run_id: Path("/tmp/runtime") / run_id, + _service_enabled_for_mode=lambda _mode, _service_name: True, + ) + + def session() -> StartupSession: + route = Route(command="start", mode="main", flags={"_restart_request": True}) + value = StartupSession( + requested_route=route, + effective_route=route, + requested_command="restart", + runtime_mode="main", + run_id="run-1", + selected_contexts=[SimpleNamespace(name="Main", root=Path("/repo"), ports={})], + base_metadata={"dashboard_stopped_services": entries}, + ) + replacement = ServiceRecord( + name="opaque-main-agent", + type="voice-runtime", + service_slug="voice-runtime", + project="Main", + cwd="/repo", + ) + value.services_by_project["Main"] = {replacement.name: replacement} + return value + + successful = build_success_run_state(runtime, session()) # type: ignore[arg-type] + failed = _build_run_state(runtime, session(), failed=True) # type: ignore[arg-type] + + self.assertEqual(successful.metadata["dashboard_stopped_services"], [entries[1]]) + self.assertEqual(failed.metadata["dashboard_stopped_services"], entries) + def test_build_run_state_marks_tree_shared_dependency_dashboard_scope(self) -> None: runtime = SimpleNamespace( config=SimpleNamespace(runtime_scope_id="scope-1"), @@ -318,6 +397,34 @@ def test_restart_service_types_accept_additional_service_slugs(self) -> None: self.assertEqual(selected, {"voice-runtime"}) + def test_restart_start_route_honors_normalized_prestop_selection(self) -> None: + route = Route( + command="start", + mode="main", + flags={ + "_restart_request": True, + "_restart_selected_services": ["Main Backend", "Main Frontend"], + }, + ) + + selected = _restart_service_types_for_project( + route=route, + project_name="Main", + default_service_types={"backend", "frontend", "optional-bad"}, + ) + + self.assertEqual(selected, {"backend", "frontend"}) + + route.flags["_restart_selected_services"] = [] + self.assertEqual( + _restart_service_types_for_project( + route=route, + project_name="Main", + default_service_types={"backend", "frontend", "optional-bad"}, + ), + set(), + ) + def test_runtime_scope_flags_select_startup_service_types(self) -> None: backend_route = parse_route(["--backend"], env={}) self.assertEqual( diff --git a/tests/python/startup/test_startup_restart_port_preservation.py b/tests/python/startup/test_startup_restart_port_preservation.py index 889ae3b72..fc4e5d972 100644 --- a/tests/python/startup/test_startup_restart_port_preservation.py +++ b/tests/python/startup/test_startup_restart_port_preservation.py @@ -98,7 +98,9 @@ def is_pid_running(*_args, **_kwargs): # noqa: ANN001 }, ) engine._terminate_services_from_state = ( # type: ignore[method-assign] - lambda state, selected_services, aggressive, verify_ownership: terminated.append(state.run_id) + lambda state, selected_services, aggressive, verify_ownership: ( + terminated.append(state.run_id) or set() + ) ) spinner_calls: list[tuple[str, bool]] = [] @@ -217,7 +219,7 @@ def test_interactive_main_restart_reuses_previous_backend_and_frontend_ports_aft captured_env: dict[str, str] = {} engine._try_load_existing_state = lambda mode=None, strict_mode_match=False: previous_state # type: ignore[method-assign] - engine._terminate_services_from_state = lambda *_args, **_kwargs: None # type: ignore[method-assign] + engine._terminate_services_from_state = lambda *_args, **_kwargs: set() # type: ignore[method-assign] engine._listener_pids_for_port = lambda _port: [] # type: ignore[method-assign] engine._start_project_context = ( # type: ignore[method-assign] lambda context, mode, route, run_id: ( @@ -358,7 +360,7 @@ def test_interactive_main_restart_rebounds_only_when_previous_port_still_unavail captured_env: dict[str, str] = {} engine._try_load_existing_state = lambda mode=None, strict_mode_match=False: previous_state # type: ignore[method-assign] - engine._terminate_services_from_state = lambda *_args, **_kwargs: None # type: ignore[method-assign] + engine._terminate_services_from_state = lambda *_args, **_kwargs: set() # type: ignore[method-assign] engine._listener_pids_for_port = lambda _port: [] # type: ignore[method-assign] engine._start_project_context = ( # type: ignore[method-assign] lambda context, mode, route, run_id: ( @@ -493,7 +495,7 @@ def test_interactive_main_restart_reuses_selected_additional_service_port_after_ captured_state: dict[str, RunState] = {} engine._try_load_existing_state = lambda mode=None, strict_mode_match=False: previous_state # type: ignore[method-assign] - engine._terminate_services_from_state = lambda *_args, **_kwargs: None # type: ignore[method-assign] + engine._terminate_services_from_state = lambda *_args, **_kwargs: set() # type: ignore[method-assign] engine._listener_pids_for_port = lambda _port: [] # type: ignore[method-assign] engine._start_project_context = ( # type: ignore[method-assign] lambda context, mode, route, run_id: ( @@ -592,7 +594,7 @@ def test_interactive_main_restart_rebounds_selected_additional_service_when_prio captured_state: dict[str, RunState] = {} engine._try_load_existing_state = lambda mode=None, strict_mode_match=False: previous_state # type: ignore[method-assign] - engine._terminate_services_from_state = lambda *_args, **_kwargs: None # type: ignore[method-assign] + engine._terminate_services_from_state = lambda *_args, **_kwargs: set() # type: ignore[method-assign] engine._listener_pids_for_port = lambda _port: [] # type: ignore[method-assign] engine._start_project_context = ( # type: ignore[method-assign] lambda context, mode, route, run_id: ( diff --git a/tests/python/startup/test_startup_restart_service_scope.py b/tests/python/startup/test_startup_restart_service_scope.py index c221556bf..c30e65b88 100644 --- a/tests/python/startup/test_startup_restart_service_scope.py +++ b/tests/python/startup/test_startup_restart_service_scope.py @@ -87,7 +87,9 @@ def test_restart_service_only_preserves_other_services_and_requirements(self) -> engine._discover_projects = lambda mode: [context] # type: ignore[method-assign] engine._reserve_project_ports = lambda _context, route=None: None # type: ignore[method-assign] engine._terminate_services_from_state = ( # type: ignore[method-assign] - lambda state, selected_services, aggressive, verify_ownership: terminate_calls.append(selected_services) + lambda state, selected_services, aggressive, verify_ownership: ( + terminate_calls.append(selected_services) or set() + ) ) engine._release_requirement_ports = ( # type: ignore[method-assign] lambda _requirements: release_calls.append("released") @@ -217,7 +219,9 @@ def test_restart_stopped_service_starts_it_without_terminating_running_sibling(s engine._discover_projects = lambda mode: [context] # type: ignore[method-assign] engine._reserve_project_ports = lambda _context, route=None: None # type: ignore[method-assign] engine._terminate_services_from_state = ( # type: ignore[method-assign] - lambda state, selected_services, aggressive, verify_ownership: terminate_calls.append(selected_services) + lambda state, selected_services, aggressive, verify_ownership: ( + terminate_calls.append(selected_services) or set() + ) ) engine._release_requirement_ports = ( # type: ignore[method-assign] lambda _requirements: self.fail("requirements should be preserved for stopped service restart") @@ -354,7 +358,9 @@ def test_restart_service_only_preserves_shared_tree_dependency_state(self) -> No engine._discover_projects = lambda mode: [context] # type: ignore[method-assign] engine._reserve_project_ports = lambda _context, route=None: None # type: ignore[method-assign] engine._terminate_services_from_state = ( # type: ignore[method-assign] - lambda state, selected_services, aggressive, verify_ownership: terminate_calls.append(selected_services) + lambda state, selected_services, aggressive, verify_ownership: ( + terminate_calls.append(selected_services) or set() + ) ) engine._release_requirement_ports = ( # type: ignore[method-assign] lambda _requirements: self.fail("shared requirements should be preserved for app-only restart") diff --git a/tests/python/startup/test_startup_run_reuse_resolution.py b/tests/python/startup/test_startup_run_reuse_resolution.py index 8bdfee133..68d2e8e76 100644 --- a/tests/python/startup/test_startup_run_reuse_resolution.py +++ b/tests/python/startup/test_startup_run_reuse_resolution.py @@ -12,6 +12,7 @@ ) from envctl_engine.startup.run_reuse_support import RunReuseDecision from envctl_engine.startup.session import StartupSession +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord def _session(route: Route) -> StartupSession: @@ -58,6 +59,67 @@ def test_run_reuse_resume_uses_named_resume_handler_owner(self) -> None: self.assertTrue(callable(StartupRunReuseResumeHandler.resume_matching_run)) self.assertTrue(callable(StartupRunReuseResumeHandler.resume_dashboard_run)) + def test_auto_resume_exception_preserves_existing_run_authority(self) -> None: + route = Route(command="start", mode="main") + session = _session(route) + candidate = RunState( + run_id="old-run", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main", + project="Main", + pid=101, + ) + }, + requirements={"Main": RequirementsResult(project="Main")}, + metadata={"authority": "old"}, + ) + runtime = _RuntimeStub() + + def fail_resume(_route: Route) -> int: + raise OSError("resume lock unlink failed") + + runtime._resume = fail_resume # type: ignore[attr-defined] + handler = StartupRunReuseResumeHandler( + runtime=runtime, + session=session, + route=route, + runtime_mode="main", + decision=RunReuseDecision( + candidate_state=candidate, + decision_kind="resume_exact", + reason="exact_match", + selected_projects=[{"name": "Main", "root": None}], + state_projects=[{"name": "Main", "root": None}], + ), + candidate_state=candidate, + reuse_started=0.0, + announce_session_identifiers=lambda current: setattr(current, "identifiers_announced", True), + emit_phase=lambda *_args, **_kwargs: None, + headless_plan_output_only=lambda _session: False, + maybe_attach_plan_agent_terminal=lambda _session: None, + print_headless_plan_session_summary=lambda _session: None, + print_plan_dry_run_preview=lambda _session: None, + configured_service_types_for_mode=lambda _mode: [], + replace_existing_project_services_for_fresh_start=lambda *_args, **_kwargs: self.fail( + "exception must not enter fresh fallback" + ), + ) + + with self.assertRaisesRegex(OSError, "resume lock unlink failed"): + handler.resume_matching_run() + + self.assertIsNone(session.run_id) + self.assertFalse(session.identifiers_announced) + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, candidate.services) + self.assertEqual(session.preserved_requirements, candidate.requirements) + self.assertEqual(session.base_metadata["authority"], "old") + self.assertEqual(session.base_metadata["state_source_run_ids"], ["old-run"]) + def test_planning_prs_branch_runs_pr_action_after_reuse_evaluation_and_skips_startup(self) -> None: route = Route(command="plan", mode="trees", raw_args=[], passthrough_args=[], projects=[], flags={"planning_prs": True}) runtime = _RuntimeStub() @@ -142,7 +204,7 @@ def test_runtime_bound_run_reuse_helper_handles_planning_prs(self) -> None: result = resolve_startup_run_reuse_with_runtime( runtime, session, - terminate_restart_orphan_listeners=lambda **kwargs: None, + terminate_restart_orphan_listeners=lambda **kwargs: set(), validate_attach_target_fn=lambda *args, **kwargs: None, attach_plan_agent_terminal=lambda *args, **kwargs: None, progress_lock=None, diff --git a/tests/python/startup/test_startup_selection_support.py b/tests/python/startup/test_startup_selection_support.py index d99dcfac1..4b675c867 100644 --- a/tests/python/startup/test_startup_selection_support.py +++ b/tests/python/startup/test_startup_selection_support.py @@ -9,7 +9,10 @@ from envctl_engine.startup.startup_selection_support import ( _restart_service_types_for_project, restart_include_requirements, + restart_target_projects, + restart_target_projects_for_selected_services, select_start_tree_projects, + state_project_names, trees_start_selection_required, ) from envctl_engine.state.models import RunState, ServiceRecord @@ -89,6 +92,103 @@ def test_trees_start_selection_required_for_default_or_explicit_trees_start(self ) ) + def test_state_project_discovery_prefers_explicit_multiword_service_metadata(self) -> None: + runtime = _RuntimeStub() + service_name = "My Project Optional Bad" + state = RunState( + run_id="run-1", + mode="main", + services={ + service_name: ServiceRecord( + name=service_name, + type="optional-bad", + cwd="/repo/optional-bad", + project="My Project", + service_slug="optional-bad", + ) + }, + ) + + self.assertEqual(state_project_names(runtime=runtime, state=state), {"My Project"}) + self.assertEqual( + restart_target_projects( + state=state, + route=Route(command="restart", mode="main", flags={}), + runtime=runtime, + ), + {"My Project"}, + ) + self.assertEqual( + restart_target_projects_for_selected_services( + selected_services={service_name}, + state=state, + runtime=runtime, + ), + {"My Project"}, + ) + + def test_state_project_discovery_uses_requirement_project_not_collision_storage_key(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-1", + mode="trees", + requirements={ + "Main Restart Collision": SimpleNamespace(project="Main") # type: ignore[dict-item] + }, + ) + + self.assertEqual(state_project_names(runtime=runtime, state=state), {"Main"}) + self.assertEqual( + restart_target_projects( + state=state, + route=Route(command="restart", mode="trees", flags={"runtime_scope": "dependencies"}), + runtime=runtime, + ), + {"Main"}, + ) + def test_restart_target_projects_resolves_stopped_additional_service_metadata(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-1", + mode="trees", + services={ + "Other Backend": ServiceRecord( + name="Other Backend", + type="backend", + cwd="/repo/other/backend", + project="Other", + ) + }, + metadata={ + "dashboard_stopped_services": [ + { + "name": "Customer Platform Voice Runtime", + "project": "Customer Platform", + "type": "voice-runtime", + } + ] + }, + ) + + for selector in ( + "voice-runtime", + "Voice Runtime", + "Customer Platform Voice Runtime", + "service:voice-runtime", + ): + with self.subTest(selector=selector): + targets = restart_target_projects( + state=state, + route=Route( + command="restart", + mode="trees", + flags={"services": [selector]}, + ), + runtime=runtime, + ) + + self.assertEqual(targets, {"Customer Platform"}) + def test_select_start_tree_projects_preselects_from_previous_trees_state(self) -> None: runtime = _RuntimeStub(can_tty=True) runtime.state = RunState( diff --git a/tests/python/startup/test_support_module_decoupling.py b/tests/python/startup/test_support_module_decoupling.py index 6dc7215f4..989ff2f30 100644 --- a/tests/python/startup/test_support_module_decoupling.py +++ b/tests/python/startup/test_support_module_decoupling.py @@ -18,6 +18,7 @@ import envctl_engine.startup.requirements_project_startup as requirements_project_startup_module # noqa: E402 from envctl_engine.startup.protocols import StartupOrchestratorLike # noqa: E402 from envctl_engine.startup.requirements_execution import ( # noqa: E402 + requirements_for_restart_context, requirements_parallel_enabled, start_requirements_for_project as requirements_start_impl, ) @@ -36,6 +37,7 @@ from envctl_engine.startup.startup_selection_support import _tree_preselected_projects_from_state # noqa: E402 from envctl_engine.config import AppServiceConfig # noqa: E402 from envctl_engine.runtime.command_router import Route, parse_route # noqa: E402 +from envctl_engine.runtime.service_manager import ServiceCleanupError # noqa: E402 from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord # noqa: E402 @@ -483,6 +485,104 @@ def reconcile(project, req, *, project_root=None): # noqa: ANN001, ANN202 self.assertEqual(reused.component("supabase")["resources"]["db"], 5574) self.assertEqual(reused.component("supabase")["resources"]["api"], 54463) + def test_restart_reuses_authoritative_requirements_under_collision_storage_key(self) -> None: + requirements = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6381, "runtime_status": "healthy"}, + ) + events: list[str] = [] + runtime = SimpleNamespace( + _try_load_existing_state=lambda **_kwargs: RunState( + run_id="run-alias", + mode="main", + requirements={"Main Restart Collision": requirements}, + ), + _emit=lambda event, **_payload: events.append(event), + ) + orchestrator = SimpleNamespace( + runtime=runtime, + start_requirements_for_project=lambda *_args, **_kwargs: self.fail( + "authoritative requirements must be reused" + ), + ) + context = SimpleNamespace(name="Main") + route = Route( + command="restart", + mode="main", + projects=["Main"], + flags={"_restart_request": True, "restart_include_requirements": False}, + ) + + reused = requirements_for_restart_context( + cast(StartupOrchestratorLike, orchestrator), + context=context, + mode="main", + route=route, + ) + + self.assertIs(reused, requirements) + self.assertEqual(events, ["requirements.restart.reuse"]) + + def test_shared_main_reuses_authoritative_requirements_under_collision_storage_key(self) -> None: + requirements = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6381, "runtime_status": "healthy"}, + ) + events: list[str] = [] + runtime = SimpleNamespace( + config=SimpleNamespace(execution_root=Path("/tmp/repo")), + _try_load_existing_state=lambda **_kwargs: RunState( + run_id="run-alias", + mode="main", + requirements={"Main Restart Collision": requirements}, + ), + _requirements_ready=lambda candidate: candidate is requirements, + _emit=lambda event, **_payload: events.append(event), + ) + orchestrator = cast(StartupOrchestratorLike, SimpleNamespace(runtime=runtime)) + + with mock.patch( + "envctl_engine.startup.startup_execution_support.requirements_for_restart_context", + side_effect=AssertionError("authoritative shared requirements must be reused"), + ): + reused = _shared_main_requirements(orchestrator, route=parse_route(["start"], env={})) + + self.assertEqual(reused.component("redis")["final"], 6381) + self.assertIn("requirements.shared.reuse", events) + + def test_ambiguous_requirement_aliases_fail_closed_instead_of_starting_duplicate_stack(self) -> None: + state = RunState( + run_id="run-ambiguous", + mode="main", + requirements={ + "Main Restart Collision": RequirementsResult(project="Main"), + "Main Restart Collision 2": RequirementsResult(project="Main"), + }, + ) + runtime = SimpleNamespace( + _try_load_existing_state=lambda **_kwargs: state, + _emit=lambda *_args, **_kwargs: None, + ) + orchestrator = SimpleNamespace( + runtime=runtime, + start_requirements_for_project=lambda *_args, **_kwargs: self.fail( + "ambiguous requirements must not start a duplicate stack" + ), + ) + + with self.assertRaisesRegex(RuntimeError, "Ambiguous requirements authority for Main"): + requirements_for_restart_context( + cast(StartupOrchestratorLike, orchestrator), + context=SimpleNamespace(name="Main"), + mode="main", + route=Route( + command="restart", + mode="main", + projects=["Main"], + flags={"_restart_request": True, "restart_include_requirements": False}, + ), + ) + def test_shared_main_requirements_starts_fresh_when_existing_state_reconciles_unreachable(self) -> None: stale = RequirementsResult( project="Main", @@ -798,6 +898,50 @@ def test_restore_missing_uses_runtime_port_allocator_without_resume_wrappers(sel self.assertEqual(events[0][0], "resume.restore.execution") self.assertEqual(events[-1][0], "resume.restore.timing") + def test_resume_project_restore_converts_requirement_release_oserror_to_retryable_result(self) -> None: + requirements = RequirementsResult(project="Main", health="degraded") + state = RunState( + run_id="run-release-error", + mode="main", + requirements={"Main Restart Collision": requirements}, + ) + context = SimpleNamespace(name="Main", root=Path("/tmp/repo"), ports={}) + events: list[str] = [] + + def fail_release(_requirements): # noqa: ANN001, ANN202 + raise OSError("requirement lock unlink failed") + + runtime = SimpleNamespace( + env={}, + config=SimpleNamespace(raw={}, additional_services=()), + _emit=lambda event, **_payload: events.append(event), + _resume_context_for_project=lambda _state, _project: context, + _project_name_from_service=lambda _name: "Main", + _requirements_ready=lambda _requirements: False, + _release_requirement_ports=fail_release, + ) + runner = ResumeProjectRestoreRunner( + orchestrator=SimpleNamespace(runtime=runtime), + rt=runtime, + state=state, + missing_services=[], + route_for_startup=Route(command="resume", mode="main"), + total_projects=1, + use_project_spinner_group=False, + project_spinner_group=SimpleNamespace(), + use_single_spinner=False, + active_spinner=SimpleNamespace(), + port_allocator=_PortAllocatorStub(), + ) + + result = runner.restore_project(1, "Main") + + self.assertEqual(result["error"], "requirement lock unlink failed") + self.assertEqual(result["remove_service_names"], []) + self.assertIsNone(result["requirements"]) + self.assertIn("resume.restore.project_timing", events) + self.assertIs(state.requirements["Main Restart Collision"], requirements) + def test_restore_missing_restarts_only_stale_services_when_requirements_are_healthy(self) -> None: released_ports: list[int] = [] terminated_services: list[str] = [] @@ -828,6 +972,7 @@ def release(self, port: int) -> None: name="Main Backend", type="backend", cwd="/tmp/repo/backend", + project="Main", pid=1111, requested_port=8000, actual_port=8000, @@ -837,6 +982,7 @@ def release(self, port: int) -> None: name="Main Frontend", type="frontend", cwd="/tmp/repo/frontend", + project="Main", pid=2222, requested_port=9000, actual_port=9000, @@ -851,6 +997,7 @@ def start_project_services(_context, *, requirements, run_id, route): # noqa: A name="Main Frontend", type="frontend", cwd="/tmp/repo/frontend", + project="Main", pid=3333, requested_port=9000, actual_port=9000, @@ -862,7 +1009,7 @@ def start_project_services(_context, *, requirements, run_id, route): # noqa: A port_planner=RecordingPortAllocator(), env={}, config=SimpleNamespace(raw={}, base_dir=Path("/tmp/repo")), - _project_name_from_service=lambda name: "Main", + _project_name_from_service=lambda _name: "Incorrect Legacy Parse", _requirements_ready=lambda value: value is requirements and value.health == "healthy", _reconcile_project_requirement_truth=lambda project, req, project_root=None: [], _emit=lambda event, **payload: None, @@ -881,7 +1028,7 @@ def start_project_services(_context, *, requirements, run_id, route): # noqa: A run_id="run-1", mode="main", services={"Main Backend": backend, "Main Frontend": frontend}, - requirements={"Main": requirements}, + requirements={"main": requirements}, ) errors = restore_missing( @@ -901,10 +1048,178 @@ def start_project_services(_context, *, requirements, run_id, route): # noqa: A self.assertEqual(released_ports, [9000]) self.assertIs(state.services["Main Backend"], backend) self.assertEqual(state.services["Main Frontend"].pid, 3333) + self.assertEqual(state.requirements, {"main": requirements}) self.assertEqual(len(startup_routes), 1) self.assertTrue(startup_routes[0].flags.get("_restart_request")) self.assertEqual(startup_routes[0].flags.get("restart_service_types"), ["frontend"]) + def test_restore_missing_fails_closed_when_stale_service_exit_is_unconfirmed(self) -> None: + release_calls: list[str] = [] + requirements_start_calls: list[str] = [] + service_start_calls: list[str] = [] + context = SimpleNamespace( + name="Main", + root=Path("/tmp/repo"), + ports={ + "backend": SimpleNamespace(assigned=8000, final=8000), + "frontend": SimpleNamespace(assigned=9000, final=9000), + }, + ) + requirements = RequirementsResult( + project="main", + health="degraded", + failures=["redis unavailable"], + ) + stale = ServiceRecord( + name="Main Frontend", + type="frontend", + cwd="/tmp/repo/frontend", + project="Main", + pid=2222, + requested_port=9000, + actual_port=9000, + status="stale", + ) + runtime = SimpleNamespace( + port_planner=_PortAllocatorStub(), + env={}, + config=SimpleNamespace(raw={}, base_dir=Path("/tmp/repo")), + _project_name_from_service=lambda _name: "Main", + _requirements_ready=lambda _value: False, + _emit=lambda _event, **_payload: None, + _tree_parallel_startup_config=lambda **_kwargs: (False, 1), + _resume_context_for_project=lambda _state, _project: context, + _terminate_service_record=lambda _service, **_kwargs: False, + _release_requirement_ports=lambda _requirements: release_calls.append("released"), + _reserve_project_ports=lambda _context: self.fail("ports must not be reserved after failed cleanup"), + _start_requirements_for_project=lambda *_args, **_kwargs: ( + requirements_start_calls.append("started") or RequirementsResult(project="Main") + ), + _start_project_services=lambda *_args, **_kwargs: service_start_calls.append("started") or {}, + ) + state = RunState( + run_id="run-unconfirmed", + mode="main", + services={stale.name: stale}, + requirements={"main": requirements}, + ) + + errors = restore_missing( + SimpleNamespace(runtime=runtime), + state, + [stale.name], + route=parse_route(["resume", "--main"], env={}), + spinner_factory=lambda *args, **kwargs: _SpinnerStub(), + spinner_enabled_fn=lambda env: False, + use_spinner_policy_fn=lambda policy: nullcontext(), + emit_spinner_policy_fn=lambda emit, policy, context: None, + resolve_spinner_policy_fn=lambda env: SimpleNamespace(enabled=False, backend="rich", style="dots"), + ) + + self.assertEqual( + errors, + ["Main: stale service termination could not be confirmed: Main Frontend"], + ) + self.assertEqual(release_calls, []) + self.assertEqual(requirements_start_calls, []) + self.assertEqual(service_start_calls, []) + self.assertIs(state.services[stale.name], stale) + self.assertEqual(state.requirements, {"main": requirements}) + self.assertEqual(requirements.health, "degraded") + + def test_restore_missing_retains_unterminated_replacement_after_confirmed_stale_stop(self) -> None: + context = SimpleNamespace( + name="Main", + root=Path("/tmp/repo"), + ports={ + "backend": SimpleNamespace(assigned=8000, final=8000), + "frontend": SimpleNamespace(assigned=9000, final=9000), + }, + ) + original_requirements = RequirementsResult( + project="main", + health="degraded", + failures=["redis stale"], + ) + replacement_requirements = RequirementsResult(project="Main", health="healthy") + released_requirements: list[RequirementsResult] = [] + stale = ServiceRecord( + name="Main Frontend", + type="frontend", + cwd="/tmp/repo/frontend", + project="Main", + pid=2222, + requested_port=9000, + actual_port=9000, + status="stale", + ) + replacement = ServiceRecord( + name="Main Frontend", + type="frontend", + cwd="/tmp/repo/frontend", + project="Main", + pid=3333, + requested_port=9000, + actual_port=9000, + status="starting", + ) + + def fail_service_start(*_args, **_kwargs): # noqa: ANN002, ANN003 + raise ServiceCleanupError( + "frontend listener startup failed; cleanup could not confirm exit", + {replacement.name: replacement}, + ) + + runtime = SimpleNamespace( + port_planner=_PortAllocatorStub(), + env={}, + config=SimpleNamespace(raw={}, base_dir=Path("/tmp/repo")), + _project_name_from_service=lambda _name: "Main", + _requirements_ready=lambda value: value is replacement_requirements, + _emit=lambda _event, **_payload: None, + _tree_parallel_startup_config=lambda **_kwargs: (False, 1), + _resume_context_for_project=lambda _state, _project: context, + _terminate_service_record=lambda _service, **_kwargs: True, + _release_requirement_ports=lambda value: released_requirements.append(value), + _reserve_project_ports=lambda _context: self.fail( + "partial restore should reserve only selected service ports" + ), + _start_requirements_for_project=lambda *_args, **_kwargs: replacement_requirements, + _start_project_services=fail_service_start, + ) + state = RunState( + run_id="run-service-cleanup", + mode="main", + services={stale.name: stale}, + requirements={"main": original_requirements}, + ) + + errors = restore_missing( + SimpleNamespace(runtime=runtime), + state, + [stale.name], + route=parse_route(["resume", "--main"], env={}), + spinner_factory=lambda *args, **kwargs: _SpinnerStub(), + spinner_enabled_fn=lambda env: False, + use_spinner_policy_fn=lambda policy: nullcontext(), + emit_spinner_policy_fn=lambda emit, policy, context: None, + resolve_spinner_policy_fn=lambda env: SimpleNamespace(enabled=False, backend="rich", style="dots"), + ) + + self.assertEqual( + errors, + ["Main: frontend listener startup failed; cleanup could not confirm exit"], + ) + self.assertEqual(set(state.services), {"Main Frontend"}) + retained = state.services["Main Frontend"] + self.assertEqual(retained.pid, 3333) + self.assertEqual(retained.status, "termination_failed") + self.assertTrue(retained.degraded) + self.assertEqual(state.metadata["termination_failed_services"], ["Main Frontend"]) + self.assertEqual(released_requirements, [original_requirements]) + self.assertEqual(state.requirements, {"main": replacement_requirements}) + self.assertEqual(replacement_requirements.health, "degraded") + def test_requirements_failure_message_summarizes_docker_daemon_outage(self) -> None: requirements = RequirementsResult( project="Main", diff --git a/tests/python/state/test_project_runtime_resolution.py b/tests/python/state/test_project_runtime_resolution.py index 1502a9e09..7dc7298c5 100644 --- a/tests/python/state/test_project_runtime_resolution.py +++ b/tests/python/state/test_project_runtime_resolution.py @@ -30,6 +30,21 @@ def test_active_project_names_include_requirements_services_and_metadata_roots(s self.assertEqual(active_project_names(state), ["alpha", "beta", "delta", "gamma"]) + def test_requirement_collision_storage_key_is_not_exposed_as_an_active_project(self) -> None: + state = RunState( + run_id="run-collision", + mode="main", + requirements={ + "Main": RequirementsResult(project="Main"), + "Main Restart Collision": RequirementsResult(project="Main"), + }, + ) + + self.assertEqual(active_project_names(state), ["Main"]) + resolution = resolve_requested_project_state(state, [], command="endpoints") + self.assertTrue(resolution.ok) + self.assertEqual(resolution.selected_projects, ["Main"]) + def test_exact_requested_project_match_filters_state_to_canonical_project(self) -> None: state = RunState( run_id="run-active", diff --git a/tests/python/state/test_state_action_orchestrator_logs.py b/tests/python/state/test_state_action_orchestrator_logs.py index 1da01021b..b773917a5 100644 --- a/tests/python/state/test_state_action_orchestrator_logs.py +++ b/tests/python/state/test_state_action_orchestrator_logs.py @@ -192,6 +192,35 @@ def test_logs_combined_project_and_service_filters_intersect(self) -> None: self.assertEqual(code, 0) self.assertEqual([service["name"] for service in payload["services"]], ["Alpha Backend"]) + def test_logs_project_filter_uses_additional_service_project_metadata(self) -> None: + service = ServiceRecord( + name="Opaque Runtime Process", + type="voice-runtime", + cwd="/tmp/customer-platform/voice", + project="Customer Platform", + service_slug="voice-runtime", + ) + requirements = RequirementsResult(project="Customer Platform") + state = RunState( + run_id="run-metadata-project", + mode="trees", + services={service.name: service}, + requirements={"Customer Platform": requirements}, + ) + runtime = _RuntimeStub(state) + runtime._project_name_from_service = lambda _name: "Opaque Runtime" # type: ignore[method-assign] + orchestrator = StateActionOrchestrator(runtime) + + code = orchestrator.execute( + Route(command="logs", mode="trees", projects=["Customer Platform"]) + ) + + self.assertEqual(code, 0) + self.assertIsNotNone(runtime.seen_logs_state) + assert runtime.seen_logs_state is not None + self.assertEqual(set(runtime.seen_logs_state.services), {service.name}) + self.assertEqual(runtime.seen_logs_state.requirements, {"Customer Platform": requirements}) + def test_health_prints_enabled_dependency_statuses(self) -> None: state = RunState( run_id="run-1", diff --git a/tests/python/state/test_state_aggregation.py b/tests/python/state/test_state_aggregation.py index 2274bf75e..d4d5993e2 100644 --- a/tests/python/state/test_state_aggregation.py +++ b/tests/python/state/test_state_aggregation.py @@ -2,10 +2,13 @@ from pathlib import Path -from envctl_engine.state.models import RunState, ServiceRecord +import pytest + +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord from envctl_engine.state.run_index import RunIndexCandidate from envctl_engine.state.state_aggregation import ( IndexedState, + filter_state_to_owned_projects, select_indexed_owners, state_from_indexed_owners, ) @@ -99,6 +102,87 @@ def test_owner_selection_filters_shadowed_run_to_projects_it_still_owns() -> Non assert selected[1].state.metadata["project_names"] == ["beta"] +def test_shadowed_multi_project_owner_does_not_relabel_opaque_services_as_remaining_project() -> None: + newest = IndexedState( + candidate=_candidate("new", "alpha", sequence=2, activation_sequence=3), + state=RunState( + run_id="new", + mode="trees", + services={ + "New Alpha": ServiceRecord( + name="New Alpha", + type="worker", + cwd="/new-alpha", + pid=3, + project="Alpha", + ) + }, + metadata={"project_names": ["Alpha"]}, + ), + ) + previous = IndexedState( + candidate=_candidate("old", "alpha", "beta", sequence=1, activation_sequence=1), + state=RunState( + run_id="old", + mode="trees", + services={ + "Opaque Old Alpha": ServiceRecord( + name="Opaque Old Alpha", + type="worker", + cwd="/old-alpha", + pid=1, + ), + "Opaque Beta": ServiceRecord( + name="Opaque Beta", + type="worker", + cwd="/beta", + pid=2, + ), + }, + metadata={ + "project_names": ["Alpha", "Beta"], + "project_roots": {"Alpha": "/old-alpha", "Beta": "/beta"}, + }, + ), + ) + + selected = select_indexed_owners( + [newest, previous], + selected_projects=(), + service_project_name=_service_project_name, + ) + + old = next(item.state for item in selected if item.candidate.run_id == "old") + assert {service.pid for service in old.services.values()} == {2} + assert {service.project for service in old.services.values()} == {"Beta"} + + +def test_multi_project_opaque_service_without_root_evidence_fails_closed() -> None: + ambiguous = IndexedState( + candidate=_candidate("ambiguous", "alpha", "beta", sequence=1, activation_sequence=1), + state=RunState( + run_id="ambiguous", + mode="trees", + services={ + "Opaque Shared Runtime": ServiceRecord( + name="Opaque Shared Runtime", + type="worker", + cwd="/unknown", + pid=101, + ) + }, + metadata={"project_names": ["Alpha", "Beta"]}, + ), + ) + + with pytest.raises(RuntimeError, match="Cannot safely determine project ownership"): + select_indexed_owners( + [ambiguous], + selected_projects=(), + service_project_name=_service_project_name, + ) + + def test_merged_state_uses_activation_owner_for_global_identity() -> None: activated = IndexedState( candidate=_candidate("activated", "alpha", sequence=1, activation_sequence=5), @@ -137,3 +221,87 @@ def test_merged_state_uses_activation_owner_for_global_identity() -> None: assert merged.metadata["global_marker"] == "activated" assert merged.metadata["project_names"] == ["alpha", "beta"] assert merged.metadata["state_source_run_ids"] == ["activated", "ancestor", "stable"] + + +def test_owned_project_filter_keeps_every_authoritative_requirement_storage_row() -> None: + state = RunState( + run_id="collision", + mode="trees", + requirements={ + "Main": RequirementsResult(project="Main", redis={"enabled": True, "final": 6380}), + "Main Restart Collision": RequirementsResult( + project="Main", + redis={"enabled": True, "final": 6381}, + ), + "Other": RequirementsResult(project="Other", redis={"enabled": True, "final": 6390}), + }, + ) + + filtered = filter_state_to_owned_projects( + state, + frozenset({"main"}), + service_project_name=_service_project_name, + ) + + assert set(filtered.requirements) == {"Main", "Main Restart Collision"} + + +def test_indexed_merge_disambiguates_opaque_legacy_names_using_single_owner_context() -> None: + alpha = IndexedState( + candidate=_candidate("alpha-run", "Alpha", sequence=1, activation_sequence=1), + state=RunState( + run_id="alpha-run", + mode="trees", + services={ + "Opaque Shared Runtime": ServiceRecord( + name="Opaque Shared Runtime", + type="worker", + cwd="/alpha", + pid=101, + ) + }, + metadata={"project_names": ["Alpha"]}, + ), + ) + beta = IndexedState( + candidate=_candidate("beta-run", "Beta", sequence=2, activation_sequence=2), + state=RunState( + run_id="beta-run", + mode="trees", + services={ + "Opaque Shared Runtime": ServiceRecord( + name="Opaque Shared Runtime", + type="worker", + cwd="/beta", + pid=202, + ) + }, + metadata={"project_names": ["Beta"]}, + ), + ) + + merged = state_from_indexed_owners( + [alpha, beta], + project_names_from_state=_project_names, + source_run_ids=_source_run_ids, + ) + + assert merged is not None + assert {service.pid for service in merged.services.values()} == {101, 202} + assert {service.project for service in merged.services.values()} == {"Alpha", "Beta"} + assert len(merged.services) == 2 + + selected = select_indexed_owners( + [alpha, beta], + selected_projects=(), + service_project_name=_service_project_name, + ) + operational = state_from_indexed_owners( + selected, + project_names_from_state=_project_names, + source_run_ids=_source_run_ids, + ) + + assert operational is not None + assert {service.pid for service in operational.services.values()} == {101, 202} + assert {service.project for service in operational.services.values()} == {"Alpha", "Beta"} diff --git a/tests/python/state/test_state_repository_contract.py b/tests/python/state/test_state_repository_contract.py index 396d70dbf..f23b70a09 100644 --- a/tests/python/state/test_state_repository_contract.py +++ b/tests/python/state/test_state_repository_contract.py @@ -885,6 +885,50 @@ def test_load_all_filters_projects_subtracted_from_a_mixed_historical_owner(self }, ) + def test_load_all_uses_sole_index_owner_for_opaque_legacy_service(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + state = RunState( + run_id="run-alpha", + mode="trees", + services={ + "Opaque Runtime": ServiceRecord( + name="Opaque Runtime", + type="worker", + cwd="/unmapped/path", + ) + }, + ) + context = SimpleNamespace(name="Alpha", root=Path(tmpdir) / "alpha", ports={}) + repo.save_run( + state=state, + contexts=[context], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + latest = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["Alpha"], + ) + active = repo.load_all(mode="trees") + + self.assertIsNotNone(latest) + assert latest is not None + self.assertEqual(latest.services["Opaque Runtime"].project, "Alpha") + self.assertEqual(len(active), 1) + self.assertEqual(active[0].services["Opaque Runtime"].project, "Alpha") + def test_aggregate_and_subset_saves_keep_project_metadata_owned_by_live_projects(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: runtime_dir = Path(tmpdir) / "runtime" @@ -1274,10 +1318,15 @@ def test_resume_orchestrator_does_not_write_legacy_state_files_directly(self) -> self.assertNotIn('runtime_legacy_root / "runtime_map.json"', raw) def test_lifecycle_cleanup_orchestrator_does_not_write_legacy_state_files_directly(self) -> None: - cleanup_path = REPO_ROOT / "python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py" - raw = cleanup_path.read_text(encoding="utf-8") - self.assertNotIn('runtime_legacy_root / "run_state.json"', raw) - self.assertNotIn('runtime_legacy_root / "runtime_map.json"', raw) + cleanup_paths = ( + REPO_ROOT / "python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py", + REPO_ROOT / "python/envctl_engine/runtime/lifecycle_full_cleanup.py", + ) + for cleanup_path in cleanup_paths: + with self.subTest(path=cleanup_path.name): + raw = cleanup_path.read_text(encoding="utf-8") + self.assertNotIn('runtime_legacy_root / "run_state.json"', raw) + self.assertNotIn('runtime_legacy_root / "runtime_map.json"', raw) def test_engine_runtime_legacy_compat_helper_is_removed(self) -> None: runtime_path = REPO_ROOT / "python/envctl_engine/runtime/engine_runtime.py" @@ -2357,6 +2406,126 @@ def fail_first_run_alias(path: Path, text: str) -> None: self.assertTrue(run_alias.is_file()) self.assertEqual(load_state(str(run_alias)).run_id, "run-a") + def test_non_os_post_commit_failure_still_signals_durable_authority(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + committed: list[str] = [] + original_publish = repo._publish_revision_aliases + repo._publish_revision_aliases = ( # type: ignore[method-assign] + lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("alias logic failed")) + ) + + with self.assertRaisesRegex(RuntimeError, "alias logic failed"): + repo.save_run( + state=RunState(run_id="run-committed", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + on_commit=lambda: committed.append("run-committed"), + ) + + self.assertEqual(committed, ["run-committed"]) + repo._publish_revision_aliases = original_publish # type: ignore[method-assign] + loaded = repo.load_latest(mode="main", strict_mode_match=True) + self.assertIsNotNone(loaded) + self.assertEqual(loaded.run_id, "run-committed") + + def test_registry_publish_then_interrupt_retains_revision_and_signals_commit(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + committed: list[str] = [] + repo.run_index.initialize_empty() + original_atomic_write = _run_index.atomic_write_text + published = False + + def publish_backup_then_interrupt(path: Path, text: str) -> None: + nonlocal published + original_atomic_write(path, text) + if path == repo.run_index.backup_path and not published: + published = True + raise KeyboardInterrupt + + with ( + patch( + "envctl_engine.state.run_index.atomic_write_text", + side_effect=publish_backup_then_interrupt, + ), + self.assertRaises(KeyboardInterrupt), + ): + repo.save_run( + state=RunState(run_id="run-interrupt", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + on_commit=lambda: committed.append("run-interrupt"), + ) + + self.assertTrue(published) + self.assertEqual(committed, ["run-interrupt"]) + candidates = repo.run_index.candidates(StateSelector(mode="main", project_names=())) + self.assertEqual(len(candidates), 1) + self.assertEqual(candidates[0].run_id, "run-interrupt") + self.assertTrue(candidates[0].state_path.is_file()) + loaded = repo.load_latest(mode="main", strict_mode_match=True) + self.assertIsNotNone(loaded) + self.assertEqual(loaded.run_id, "run-interrupt") + + def test_registry_interrupt_before_publish_removes_uncommitted_revision(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + committed: list[str] = [] + repo.run_index.initialize_empty() + + with ( + patch( + "envctl_engine.state.run_index.atomic_write_text", + side_effect=KeyboardInterrupt, + ), + self.assertRaises(KeyboardInterrupt), + ): + repo.save_run( + state=RunState(run_id="run-pre-publish", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + on_commit=lambda: committed.append("run-pre-publish"), + ) + + self.assertEqual(committed, []) + self.assertEqual( + repo.run_index.candidates(StateSelector(mode="main", project_names=())), + [], + ) + revisions = repo.run_dir_path("run-pre-publish") / "revisions" + self.assertEqual(list(revisions.glob("*")), []) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/state/test_state_roundtrip.py b/tests/python/state/test_state_roundtrip.py index 7275f464d..05bb181fb 100644 --- a/tests/python/state/test_state_roundtrip.py +++ b/tests/python/state/test_state_roundtrip.py @@ -33,6 +33,8 @@ def test_json_state_roundtrip_preserves_services(self) -> None: container_id="sha256:abc", container_name="envctl-app-tree-alpha-backend", container_image="envctl/example-backend:dev", + container_launch_token="launch-token-123", + container_cleanup_pending_since=1_700_000_000.25, ) }, requirements={ @@ -73,8 +75,36 @@ def test_json_state_roundtrip_preserves_services(self) -> None: loaded.services["Tree Alpha Backend"].container_image, "envctl/example-backend:dev", ) + self.assertEqual( + loaded.services["Tree Alpha Backend"].container_launch_token, + "launch-token-123", + ) + self.assertEqual( + loaded.services["Tree Alpha Backend"].container_cleanup_pending_since, + 1_700_000_000.25, + ) self.assertEqual(loaded.metadata["source"], "python") + def test_requirement_storage_key_roundtrip_preserves_authoritative_owner_project(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) / "run_state.json" + state = RunState( + run_id="run-collision", + mode="main", + requirements={ + "Main Restart Collision": RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6381}, + ) + }, + ) + + dump_state(state, str(path)) + loaded = load_state(str(path), allowed_root=tmpdir) + + self.assertEqual(loaded.requirements["Main Restart Collision"].project, "Main") + self.assertEqual(loaded.requirements["Main Restart Collision"].redis["final"], 6381) + def test_legacy_shell_state_is_loaded_without_source_execution(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: legacy = Path(tmpdir) / "legacy.state" @@ -128,6 +158,32 @@ def test_merge_states_roundtrip_keeps_latest_ports(self) -> None: self.assertEqual(merged.run_id, "run-b") self.assertEqual(merged.services["Tree Alpha Frontend"].actual_port, 9002) + def test_merge_states_preserves_same_storage_name_owned_by_distinct_projects(self) -> None: + alpha = ServiceRecord( + name="Opaque Shared Runtime", + type="worker", + cwd="/alpha", + pid=101, + project="Alpha", + ) + beta = ServiceRecord( + name="Opaque Shared Runtime", + type="worker", + cwd="/beta", + pid=202, + project="Beta", + ) + + merged = merge_states( + [ + RunState(run_id="alpha", mode="trees", services={alpha.name: alpha}), + RunState(run_id="beta", mode="trees", services={beta.name: beta}), + ] + ) + + self.assertEqual({service.pid for service in merged.services.values()}, {101, 202}) + self.assertEqual(len(merged.services), 2) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/ui/test_dashboard_pr_link_rendering.py b/tests/python/ui/test_dashboard_pr_link_rendering.py index b52d1b3bf..7dcfde672 100644 --- a/tests/python/ui/test_dashboard_pr_link_rendering.py +++ b/tests/python/ui/test_dashboard_pr_link_rendering.py @@ -59,6 +59,19 @@ def test_project_root_prefers_metadata_then_service_component_parent(self) -> No ) }, ) + opaque_service_state = RunState( + run_id="run-opaque", + mode="main", + services={ + "Opaque API Process": ServiceRecord( + name="Opaque API Process", + type="backend", + cwd=str(backend), + project="Main", + service_slug="backend", + ) + }, + ) self.assertEqual( pr_link_rendering.dashboard_project_root(SimpleNamespace(), state=metadata_state, project="Main"), @@ -68,6 +81,14 @@ def test_project_root_prefers_metadata_then_service_component_parent(self) -> No pr_link_rendering.dashboard_project_root(SimpleNamespace(), state=service_state, project="Main"), repo, ) + self.assertEqual( + pr_link_rendering.dashboard_project_root( + SimpleNamespace(), + state=opaque_service_state, + project="Main", + ), + repo, + ) def test_lookup_pr_caches_missing_and_positive_results(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: diff --git a/tests/python/ui/test_dashboard_restart_selection_support.py b/tests/python/ui/test_dashboard_restart_selection_support.py index 09b157c91..777d0db2b 100644 --- a/tests/python/ui/test_dashboard_restart_selection_support.py +++ b/tests/python/ui/test_dashboard_restart_selection_support.py @@ -36,6 +36,19 @@ def test_dashboard_stopped_services_by_project_with_data(self) -> None: result = dashboard_stopped_services_by_project(state) self.assertEqual(result, {"p1": {"backend": "p1 Backend", "frontend": "p1 Frontend"}}) + def test_dashboard_stopped_services_by_project_includes_additional_services(self) -> None: + state = _make_state(metadata={ + "dashboard_stopped_services": [ + {"project": "p1", "type": "voice-runtime", "name": "opaque-agent"}, + {"project": "p1", "type": "worker", "name": ""}, + ] + }) + + self.assertEqual( + dashboard_stopped_services_by_project(state), + {"p1": {"voice-runtime": "opaque-agent", "worker": "p1 Worker"}}, + ) + def test_dashboard_project_configured_services_empty(self) -> None: state = _make_state() result = dashboard_project_configured_services(state) diff --git a/tests/python/ui/test_selection_support.py b/tests/python/ui/test_selection_support.py index a988856b0..62c026f5f 100644 --- a/tests/python/ui/test_selection_support.py +++ b/tests/python/ui/test_selection_support.py @@ -97,6 +97,30 @@ def test_services_from_selection_handles_all_projects_and_services(self) -> None {"Main Backend", "Main Frontend"}, ) + def test_multiword_additional_service_selection_uses_record_project_metadata(self) -> None: + service = ServiceRecord( + name="Opaque Runtime Process", + type="voice-runtime", + cwd="/tmp/customer-platform/voice", + project="Customer Platform", + service_slug="voice-runtime", + ) + state = RunState(run_id="run-extra", mode="trees", services={service.name: service}) + runtime = SimpleNamespace(_project_name_from_service=lambda _name: "Opaque Runtime") + + self.assertEqual( + [project.name for project in project_names_from_state(runtime, state)], + ["Customer Platform"], + ) + self.assertEqual( + services_from_selection( + runtime, + TargetSelection(project_names=["customer platform"]), + state, + ), + {service.name}, + ) + def test_no_target_selected_message_and_explicit_target_detection_match_route_context(self) -> None: runtime = SimpleNamespace(_selectors_from_passthrough=lambda args: {"svc"} if args else set())