diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c16a01e60..4e27655af 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,6 @@ name: Tests -# Runs the canonical contributor validation lane on every PR targeting `main`. +# Runs the canonical contributor validation lane on every PR targeting `dev` or `main`. # The lane matches `docs/developer/contributing.md`: # # uv run --extra dev pytest -q @@ -9,12 +9,12 @@ name: Tests # # `pull_request` already fires on PR open, reopen, and every push to the head # branch (the `synchronize` event), so a separate `push` trigger would just -# double the runs. Post-merge work on `main` is handled by `release.yml`. +# double the runs. Release publishing from `main` is handled explicitly by `release.yml`. # # Jobs run in parallel: # - pytest: the slow lane # - package: python -m build + release shipability gate -# - lint: ruff (non-blocking until main is green) +# - lint: ruff (non-blocking until the lint baseline is green) # # All three restore uv's cache through GitHub's official cache action keyed by # uv.lock so dependency installs reuse already-downloaded wheels without relying @@ -31,6 +31,7 @@ name: Tests on: pull_request: branches: + - dev - main permissions: diff --git a/contracts/python_runtime_gap_report.json b/contracts/python_runtime_gap_report.json index b135b3f8e..3a03a2972 100644 --- a/contracts/python_runtime_gap_report.json +++ b/contracts/python_runtime_gap_report.json @@ -1,8 +1,8 @@ { "version": 1, - "generated_at": "2026-07-06T17:49:55+00:00", - "matrix_generated_at": "2026-07-06T17:49:55+00:00", - "matrix_sha256": "19b3bb085a8392aa9e6f99856983555c06da3606d5e81a6d1f2be7d20022899d", + "generated_at": "2026-07-10T21:33:31+00:00", + "matrix_generated_at": "2026-07-10T21:30:49+00:00", + "matrix_sha256": "6ecf2e6088f7d7a8926cc75e72d4376423b3a171c47406c9f995a5c368e48027", "summary": { "feature_count": 59, "gap_count": 0, diff --git a/contracts/runtime_feature_matrix.json b/contracts/runtime_feature_matrix.json index a29441403..d1e99da6b 100644 --- a/contracts/runtime_feature_matrix.json +++ b/contracts/runtime_feature_matrix.json @@ -1,6 +1,6 @@ { "version": 1, - "generated_at": "2026-07-06T17:49:55+00:00", + "generated_at": "2026-07-10T21:30:49+00:00", "inventory_sources": { "python_supported_commands": [ "blast-all", @@ -327,6 +327,7 @@ "--deps", "--deps-parallel", "--deps-sequential", + "--docker", "--doctor", "--entire-system", "--failed", @@ -419,6 +420,7 @@ "python_source_of_truth": [ "python/envctl_engine/runtime/engine_runtime.py", "python/envctl_engine/runtime/engine_runtime_lifecycle_support.py", + "python/envctl_engine/runtime/lifecycle_full_cleanup.py", "python/envctl_engine/runtime/lifecycle_blast_support.py", "python/envctl_engine/runtime/lifecycle_blast_ports.py", "python/envctl_engine/runtime/lifecycle_blast_processes.py", @@ -1311,7 +1313,8 @@ "shell_source_of_truth": [], "python_source_of_truth": [ "python/envctl_engine/runtime/engine_runtime.py", - "python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py" + "python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py", + "python/envctl_engine/runtime/lifecycle_full_cleanup.py" ], "evidence_tests": [ "tests/python/runtime/test_lifecycle_parity_stop_health.py" diff --git a/docs/developer/contributing.md b/docs/developer/contributing.md index 649ba583e..02284bc23 100644 --- a/docs/developer/contributing.md +++ b/docs/developer/contributing.md @@ -8,7 +8,7 @@ This repository is Python-first at runtime. Most behavior changes therefore need - did the change accidentally break the supported Python runtime contract? ## Workflow -1. Create a branch from `main`. +1. Create a branch from `dev`. 2. Keep changes scoped to one objective. 3. Install the CLI in editable mode: @@ -35,7 +35,7 @@ uv run --extra dev python scripts/release_shipability_gate.py --repo . --check-t 6. Use narrower `unittest` targets only for focused local iteration; they are no longer the authoritative release-readiness lane. 7. Use conventional commits (`type(scope): subject`). -8. Open a PR with: +8. Open a PR targeting `dev` with: - `Summary` - `Validation` - `Impact` diff --git a/docs/reference/commands.md b/docs/reference/commands.md index b568e2d28..2df44f879 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -253,11 +253,16 @@ envctl pr-preview-controller --command start --pr-number 287 Behavior: - starts or reconciles a preview when the configured label is added +- treats an explicit `--command start` as a forced reconciliation: an existing + same-head runtime is stopped before the import/start path runs again, so + changed CI environment or secret inputs cannot be hidden by Git identity - redeploys labeled PRs when GitHub sends a new-commit event - loads a tracked `.envctl` from the exact PR head when present; otherwise it falls back to the controller-generated configuration - publishes explicitly listed `ENVCTL_PREVIEW_PUBLIC_SERVICES` on deterministic TLS hosts and projects matching HTTP/WebSocket source URLs into launch env +- rejects a routed service whose JSON health response explicitly reports + `"ready": false`, even when the endpoint itself returns HTTP 2xx - creates `ENVCTL_PREVIEW_GENERATED_SECRETS` ephemerally for each preview start, so shared service credentials do not need to be stored in GitHub - also redeploys during scheduled reconciliation if the saved preview head is stale @@ -271,6 +276,22 @@ Behavior: - records GitHub deployments and marks them inactive when previews stop - is intended for trusted CI runners with `gh`, `git`, Docker, and envctl available +Integration ownership: + +- envctl is the canonical owner of preview event parsing, branch-to-PR + matching, worktree lifecycle, runtime cleanup, comments, and deployments; + consuming repositories should invoke the installed command instead of + vendoring a controller copy +- the caller workflow should forward `pull_request_target`, feature-branch + `push`, `schedule`, and `workflow_dispatch` events through + `GITHUB_EVENT_NAME` and `GITHUB_EVENT_PATH` +- feature-branch `push` events are restricted to open same-repository PRs with + the configured preview label; deleted branches, tags, forks, and unmatched + branches are ignored +- install or select the desired envctl revision before invoking the command; + this keeps controller rollout/version policy in CI while behavior and tests + remain centralized in envctl + ## Runtime Resolution Events @@ -291,6 +312,10 @@ High-value command families: - `plan` - `import` - `test` + +When `envctl pr` or `envctl ship` opens a pull request without an explicit +`ENVCTL_PR_BASE`, it selects the first branch that exists in this order: +`dev`, `main`, then `master`. Both remote branches and local refs are checked. - `logs` - `health` - `errors` @@ -388,6 +413,14 @@ envctl kill --backend --headless # alias for stop --backend envctl kill-all --headless # alias for stop-all ``` +Plain dependency cleanup is intentionally storage-preserving: `stop --dependencies` +and plain `stop-all` release saved dependency records and scoped port locks, but +leave envctl-managed Docker dependency stacks and their storage available for a +later start to reuse. Use `stop-all --remove-volumes` to remove supported managed +containers and volumes, or `blast-all` for aggressive ecosystem cleanup. +Explicit `--main` and `--trees` cleanup is mode-scoped: it never stops or retires +the other mode's saved runtime. + For AI plan-agent launches, `--entire-system` starts configured or autodetected local app services. In a repo/worktree with no explicit backend, frontend, or additional-service configuration and no supported autodetectable app layout, envctl reports that no local app system is configured and continues without app services; the AI session still launches. @@ -522,6 +555,11 @@ Commit defaults: - write one complete next commit message in `.envctl-commit-message.md` rather than multiple fragmented summaries only when using the fallback ledger - envctl-local control artifacts (`.envctl*`, `.codegraph/`, `.serena/project.local.yml`, `MAIN_TASK.md`, `OLD_TASK_*.md`, `trees/`) stay local; if a broad `git add .` stages them, `envctl commit` unstages those protected paths before committing normal changes +PR defaults: + +- `envctl pr` and `envctl ship` target `dev` when that remote branch exists, otherwise `main`, otherwise `master`; use `--pr-base ` to override the selection explicitly +- an existing open PR for the current head branch is rediscovered before creation, so rerunning the command reports or updates the same PR instead of opening a duplicate + Optional plan-agent launch config for `--plan` and `--import`: `envctl ship` is the preferred AI handoff command. From inside the current @@ -534,7 +572,7 @@ If GitHub has attached the pushed head commit but still has no target test check `no_checks_reported` immediately instead of waiting for the full timeout. The command returns the structured JSON result by default including the PR URL, `pr_created`, `operation_statuses`, `checks_state`, `passed_checks`, `failing_checks`, `pending_checks`, and `checks_error`. `--json` remains accepted as a compatibility no-op; -use `--human` only when compact terminal output is preferred. The check wait timeout defaults to 2 minutes; pending +use `--human` only when compact terminal output is preferred. The check wait timeout defaults to 3 minutes; pending checks after that window are reported as `checks_pending_timeout` without failing the handoff command, while actual failed checks still return a non-zero exit. The timeout can be tuned with `ENVCTL_SHIP_CHECK_TIMEOUT_SECONDS` and diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 227ffd302..0a7d02c42 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -139,6 +139,13 @@ Only simple `${VAR}` placeholders are supported. Shell-style defaults, command s Frontend launch-env sections may use `ENVCTL_SOURCE_SUPABASE_URL` and `ENVCTL_SOURCE_SUPABASE_ANON_KEY`. They cannot reference `ENVCTL_SOURCE_SUPABASE_SERVICE_ROLE_KEY`; envctl rejects that mapping before launching the frontend process. +Every launched service also receives `ENVCTL_PYTHON_EXECUTABLE`, the absolute +interpreter path running envctl. Repository-owned shell launchers can use this +as a bootstrap interpreter on minimal hosts where a `python` command alias is +not installed. Applications should still create and execute their own project +environment rather than importing application dependencies from envctl's +environment. + ## Managed Supabase Ports Managed Supabase exposes two distinct local resources: @@ -371,6 +378,65 @@ The wizard saves accepted backend/frontend test suggestions to `ENVCTL_BACKEND_T | `ENVCTL_WORKTREE_CGC_DATABASE` | `kuzudb` | Backend passed as `--database ` to `cgc context create` for generated worktree contexts. | ## Execution Policy + +### Docker application runtime + +`envctl ... --docker` preserves normal project selection, port allocation, dependency startup, +service ordering, state, dashboard, logs, health, restart, and stop behavior, but launches every +selected app service in a managed container instead of as a host process. Managed dependencies such +as Postgres, Redis, Supabase, and n8n continue to use their existing Docker adapters. + +Settings resolve service-specific keys before global fallbacks. Additional service names are +uppercased and non-alphanumeric characters become underscores, so `voice-runtime` accepts the +standard `ENVCTL_SERVICE_VOICE_RUNTIME_*` prefix (and the shorter `ENVCTL_VOICE_RUNTIME_*` alias). + +| Variable | Default | Purpose | +| --- | --- | --- | +| `ENVCTL__DOCKER_IMAGE` / `ENVCTL_DOCKER_IMAGE` | unset | Existing image to run. Without a Dockerfile setting, no build runs. | +| `ENVCTL__DOCKERFILE` / `ENVCTL_DOCKERFILE` | discovered | Dockerfile to build; discovery checks the service directory and then project root. | +| `ENVCTL__DOCKER_CONTEXT` / `ENVCTL_DOCKER_CONTEXT` | Dockerfile directory | Build context relative to the project root. | +| `ENVCTL__DOCKER_TARGET` / `ENVCTL_DOCKER_TARGET` | unset | Optional multi-stage build target. | +| `ENVCTL__DOCKER_BUILD_ARGS` / `ENVCTL_DOCKER_BUILD_ARGS` | public frontend env keys | Comma-separated build-arg names or `NAME=value` entries. Named values come from the service env and are not placed on the Docker command line. | +| `ENVCTL__DOCKER_COMMAND` / `ENVCTL_DOCKER_COMMAND` | image command | Explicit command executed inside the container. | +| `ENVCTL__DOCKER_COMMAND_MODE` / `ENVCTL_DOCKER_COMMAND_MODE` | `image` | `image` keeps the image CMD/ENTRYPOINT; `service` translates and runs envctl's normal service command. | +| `ENVCTL__DOCKER_WORKDIR` / `ENVCTL_DOCKER_WORKDIR` | image default | Container working directory. | +| `ENVCTL__DOCKER_PORT` / `ENVCTL_DOCKER_PORT` | single exposed image port or allocated host port | Container-side listener port. | +| `ENVCTL__DOCKER_BUILD_POLICY` / `ENVCTL_DOCKER_BUILD_POLICY` | `cached` | `cached`, `missing`, or `never`. | +| `ENVCTL_DOCKER_BUILD_TIMEOUT_SECONDS` | `900` | Image build timeout. | +| `ENVCTL_DOCKER_RUN_TIMEOUT_SECONDS` | `120` | Container creation timeout. | + +The default `cached` policy invokes `docker build` with BuildKit enabled on every start. Docker +evaluates the context and reuses unchanged layers, while changed source cannot accidentally reuse a +stale whole image. Generated tags are stable for one checkout/worktree, preventing parallel trees +from racing on one tag. Use `missing` for immutable/prebuilt local tags and `never` when image +lifecycle is owned outside envctl. + +Containers publish only to `127.0.0.1`, carry `io.envctl.*` ownership labels, and persist their ID, +name, and image in runtime state. Internal URLs and host settings pointing at `localhost` map to +`host.docker.internal`; browser-facing `VITE_*`, `NEXT_PUBLIC_*`, `PUBLIC_*`, CORS, and frontend URL +values remain host-facing. Service env passes through a short-lived mode-`0600` file and is not +rendered into Docker command diagnostics. Normal logs are snapshotted from Docker on demand; +`logs --follow` creates followers only for that command and tears them down on exit, so application +runtime state does not depend on long-lived host helper processes. + +```dotenv +ENVCTL_BACKEND_DOCKER_IMAGE=acme/api-dev:latest +ENVCTL_BACKEND_DOCKER_COMMAND=uvicorn app.main:app --host 0.0.0.0 --port 8000 +ENVCTL_BACKEND_DOCKER_PORT=8000 + +ENVCTL_FRONTEND_DOCKERFILE=frontend/Dockerfile +ENVCTL_FRONTEND_DOCKER_CONTEXT=frontend +ENVCTL_FRONTEND_DOCKER_PORT=5173 +``` + +```bash +envctl start --trees --entire-system --docker --headless +envctl health --json +envctl logs --project feature-a-1 --service backend +envctl restart --project feature-a-1 --service backend --docker --headless +envctl stop --project feature-a-1 --entire-system +``` + | Variable | Default | Purpose | | --- | --- | --- | | `ENVCTL_SERVICE_ATTACH_PARALLEL` | `true` | Run backend+frontend service attach in parallel when both are selected. | diff --git a/docs/reference/important-flags.md b/docs/reference/important-flags.md index 4441f34f9..c1026dcc0 100644 --- a/docs/reference/important-flags.md +++ b/docs/reference/important-flags.md @@ -41,9 +41,9 @@ pass `--interactive` only when you want prompts. | `--frontend` | Runtime scope: run or stop frontend app services only. | | `--fullstack` | Runtime scope: run or stop backend and frontend app services. | | `--both` | Alias for `--fullstack`. | -| `--dependencies` | Runtime scope: run or stop dependencies only (DB/Redis/etc.), without app services. | +| `--dependencies` | Runtime scope: run dependencies only, or detach their saved records/port locks on stop without removing reusable managed Docker stacks. | | `--deps` | Alias for `--dependencies`. | -| `--entire-system` | Runtime scope: run or stop dependencies plus backend and frontend services. | +| `--entire-system` | Runtime scope: run dependencies plus app services; on stop, terminate app services and detach dependency records/locks while keeping managed Docker stacks reusable. | | `--only-frontend` | Startup/plan modifier: launch only the frontend app service; skip backend, managed dependencies, and dependency prep. | | `--only-backend` | Startup/plan modifier: launch only the backend app service; skip frontend, managed dependencies, and dependency prep. | | `--no-deps` | Startup/plan modifier: skip managed dependencies and plan-agent dependency prep. | @@ -89,12 +89,22 @@ Plan-agent handoff note: if `--plan --tmux/--omx --headless` starts the implemen | `--parallel-trees` | Enable parallel tree startup workers. | | `--parallel-trees-max ` | Max parallel tree startup workers. | | `--service-parallel` / `--service-sequential` | Run backend+frontend startup attach in parallel or sequential mode (default: parallel). | +| `--docker` | Build or reuse images and run backend, frontend, and additional app services as managed Docker containers. | | `--parallel` / `--sequential` | For `envctl test`, run backend/frontend test suites in parallel or sequential mode (default: parallel for multiple suites, with sequential fallback). Legacy aliases: `--test-parallel` / `--test-sequential`. | | `--test-parallel-max ` | Cap concurrent test suites in parallel mode and pytest-xdist workers when pytest-xdist is explicitly enabled (default suite cap: CPU-aware, max `4`; default pytest workers: current free CPU cores). | | `--clear-port-state` | Clear saved port reservations/state. | | `--force` | Free configured ports if needed. | | `ENVCTL_DOCKER_PREWARM=0\|1` | Enable/disable one-shot Docker daemon prewarm before requirements startup (default: `1`). | | `ENVCTL_DOCKER_PREWARM_TIMEOUT_SECONDS=` | Timeout for Docker prewarm command (default: `10`). | +| `ENVCTL__DOCKER_IMAGE=` | Use an existing image for one app service. `ENVCTL_DOCKER_IMAGE` is the global fallback. | +| `ENVCTL__DOCKERFILE=` | Build an app service image from a Dockerfile. `ENVCTL_DOCKERFILE` is the global fallback. | +| `ENVCTL__DOCKER_CONTEXT=` | Override the Docker build context; defaults to the Dockerfile directory. | +| `ENVCTL__DOCKER_BUILD_ARGS=` | Add comma-separated Docker build-arg names (or `NAME=value` entries); public frontend env keys are passed automatically. | +| `ENVCTL__DOCKER_COMMAND=` | Override the image command inside the container. | +| `ENVCTL__DOCKER_COMMAND_MODE=image\|service` | Keep image CMD/ENTRYPOINT (default) or run envctl's normal service command. | +| `ENVCTL__DOCKER_PORT=` | Override the container-side port while preserving envctl's allocated host port. | +| `ENVCTL__DOCKER_WORKDIR=` | Override the image working directory for the service command. | +| `ENVCTL__DOCKER_BUILD_POLICY=cached\|missing\|never` | `cached` (default) reuses Docker layers; `missing` builds only when absent; `never` requires an existing image. | | `ENVCTL_REQUIREMENTS_PORT_MISMATCH_POLICY=adopt_existing\|recreate` | On existing container port mismatch, adopt existing mapping (default) or force recreate. | ## Logs and Debugging diff --git a/docs/user/common-workflows.md b/docs/user/common-workflows.md index f9de37747..8c2b6f60c 100644 --- a/docs/user/common-workflows.md +++ b/docs/user/common-workflows.md @@ -68,6 +68,12 @@ envctl stop --entire-system --headless envctl kill-all --headless ``` +`stop --dependencies` and plain `stop-all` preserve managed Docker dependency +stacks and storage for reuse while releasing their saved records and port locks. +Use `stop-all --remove-volumes` or `blast-all` when those managed containers must +also be removed. Pass `--main` or `--trees` to clean only that saved runtime mode; +the other mode remains active. + When you invoke a specific action command directly, envctl stays non-interactive by default. For example, `envctl kill-all`, `envctl pr`, `envctl test`, `envctl logs`, and `envctl migrate` behave like their `--headless` forms. Add `--interactive` only when you want envctl to prompt @@ -212,6 +218,33 @@ This is especially useful: - before CI or agent automation - when debugging resume surprises +## Run the Same System in Docker + +Use `--docker` to preserve envctl's main/trees, selective-service, dependency, dashboard, health, +logs, restart, and stop workflows while replacing app host processes with managed containers: + +```bash +envctl start --main --entire-system --docker --headless +envctl start --trees --entire-system --isolated-deps --docker --headless +``` + +If a service directory contains a `Dockerfile`, envctl builds it with Docker's layer cache. Otherwise, +configure an existing image with `ENVCTL_BACKEND_DOCKER_IMAGE`, `ENVCTL_FRONTEND_DOCKER_IMAGE`, or +the corresponding prefix for an additional app service. Production-oriented images commonly need +`ENVCTL__DOCKER_COMMAND` and `ENVCTL__DOCKER_PORT` so the service listens on +`0.0.0.0` at the expected container port. + +```bash +envctl endpoints --json +envctl health --json +envctl logs --project Main --service backend +envctl restart --project Main --service backend --docker --headless +envctl stop --project Main --entire-system +``` + +See [Configuration: Docker application runtime](../reference/configuration.md#docker-application-runtime) +for image, Dockerfile, command, port, workdir, target, and cache-policy settings. + ## Debug a Bad Interactive Session Use this when the issue is in the dashboard, selector, spinner, or input flow. diff --git a/docs/user/getting-started.md b/docs/user/getting-started.md index ce08c790c..c96f8c627 100644 --- a/docs/user/getting-started.md +++ b/docs/user/getting-started.md @@ -188,7 +188,7 @@ Why this works well: - `explain-startup` shows what the runtime is about to do - `--resume` is the fastest normal start path after previous runs - `dashboard`, `logs`, and `test` cover the common daily loop -- `stop-all` gives you a clean shutdown +- `stop-all` cleanly shuts down tracked app services and releases dependency records/locks; managed Docker dependency stacks and storage remain reusable (use `stop-all --remove-volumes` or `blast-all` to remove them) If you want a fresh start instead of resume: diff --git a/python/envctl_engine/actions/action_command_execution_support.py b/python/envctl_engine/actions/action_command_execution_support.py index f18734d8f..51e394167 100644 --- a/python/envctl_engine/actions/action_command_execution_support.py +++ b/python/envctl_engine/actions/action_command_execution_support.py @@ -18,6 +18,8 @@ "migrate": "run_migrate_action", } +_BULK_MUTATING_COMMANDS = frozenset({"commit", "migrate", "pr", "ship"}) + class ActionSpinner(Protocol): def start(self) -> None: ... @@ -59,6 +61,12 @@ def execute(self) -> int: self._emit_finish(1, error=error) return 1 + bulk_confirmation_error = self._bulk_confirmation_error(targets) + if bulk_confirmation_error is not None: + print(bulk_confirmation_error) + self._emit_finish(1, error=bulk_confirmation_error) + return 1 + handler = self._handler() if handler is None: return self.runtime.unsupported_command(self.route.command) @@ -70,6 +78,15 @@ def execute(self) -> int: self._emit_finish(code) return code + def _bulk_confirmation_error(self, targets: list[object]) -> str | None: + if self.route.command not in _BULK_MUTATING_COMMANDS: + return None + if not bool(self.route.flags.get("all")) or len(targets) <= 1: + return None + if bool(self.route.flags.get("yes")) or bool(self.route.flags.get("force")): + return None + return f"{self.route.command} --all targets {len(targets)} projects and requires --yes or --force" + def _execute_special_worktree_command(self) -> int | None: if self.route.command == "self-destruct-worktree": code = self.orchestrator.run_self_destruct_worktree_action(self.route) diff --git a/python/envctl_engine/actions/action_commit_support.py b/python/envctl_engine/actions/action_commit_support.py index 3ccaa4e28..43335d25e 100644 --- a/python/envctl_engine/actions/action_commit_support.py +++ b/python/envctl_engine/actions/action_commit_support.py @@ -19,7 +19,7 @@ resolve_commit_message_request, write_commit_message_file as _write_commit_message_file, ) -from envctl_engine.actions.action_protected_artifacts import EnvctlProtectedPathPartition +from envctl_engine.actions.action_protected_artifacts import EnvctlProtectedPathPartition, unstaged_stageable_paths from envctl_engine.ui.path_links import render_paths_in_terminal_text RunGitFn = Callable[[Path, list[str]], subprocess.CompletedProcess[str]] @@ -88,6 +88,7 @@ def _stage_commit_candidates(self, git_root: Path) -> EnvctlProtectedPathPartiti return None partition = self.dependencies.partition_envctl_protected_paths(pre_stage_status.stdout) + paths_to_stage = unstaged_stageable_paths(pre_stage_status.stdout) unstaged_protected_paths = self._unstage_protected_paths(git_root, partition) if unstaged_protected_paths is None: return None @@ -97,7 +98,7 @@ def _stage_commit_candidates(self, git_root: Path) -> EnvctlProtectedPathPartiti return None partition = refreshed_partition - if not self._stage_paths(git_root, partition.stageable_paths): + if not self._stage_paths(git_root, paths_to_stage): return None self._print_skipped_protected_paths(unstaged_protected_paths, partition.protected_skipped_paths) return partition @@ -141,7 +142,7 @@ def _refreshed_partition_after_unstage(self, git_root: Path) -> EnvctlProtectedP def _stage_paths(self, git_root: Path, stageable_paths: list[str]) -> bool: if not stageable_paths: return True - add = self.dependencies.run_git(git_root, ["add", "--", *stageable_paths]) + add = self.dependencies.run_git(git_root, ["add", "--all", "--", *stageable_paths]) if add.returncode != 0: self.dependencies.print_error("git add failed", add) return False diff --git a/python/envctl_engine/actions/action_git_state_support.py b/python/envctl_engine/actions/action_git_state_support.py index d99a73ab5..63d92262e 100644 --- a/python/envctl_engine/actions/action_git_state_support.py +++ b/python/envctl_engine/actions/action_git_state_support.py @@ -1,10 +1,17 @@ from __future__ import annotations from dataclasses import dataclass +import json +import os from pathlib import Path import subprocess from typing import Callable +from envctl_engine.shared.git_branch_support import ( + PREFERRED_PR_BASE_BRANCHES, + detect_preferred_default_branch, +) + @dataclass(frozen=True, slots=True) class DirtyWorktreeReport: @@ -20,6 +27,12 @@ def dirty(self) -> bool: return self.staged or self.unstaged or self.untracked +@dataclass(frozen=True, slots=True) +class ExistingPullRequest: + url: str + base_branch: str + + def resolve_git_root(project_root: Path, repo_root: Path) -> Path: for candidate in (project_root, repo_root): if (candidate / ".git").exists(): @@ -71,53 +84,139 @@ def classify_dirty_porcelain(status_output: str) -> tuple[bool, bool, bool]: def detect_default_branch(git_root: Path, *, git_output: Callable[[Path, list[str]], str]) -> str: - ref = git_output(git_root, ["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]).strip() - if ref.startswith("origin/"): - return ref.split("origin/", 1)[1] - for candidate in ("main", "master"): - if git_output(git_root, ["rev-parse", "--verify", candidate]).strip(): - return candidate - return "main" + return detect_preferred_default_branch( + lambda args: git_output(git_root, list(args)), + probe_remote_heads=True, + ) -def existing_pr_url( +def detect_pr_base_branch(git_root: Path, *, git_output: Callable[[Path, list[str]], str]) -> str: + for branch in PREFERRED_PR_BASE_BRANCHES: + for ref in (f"refs/remotes/origin/{branch}", f"refs/heads/{branch}"): + if git_output(git_root, ["rev-parse", "--verify", ref]).strip(): + return branch + return detect_default_branch(git_root, git_output=git_output) + + +def existing_pull_request( git_root: Path, branch: str, *, gh_path: str | None, run_process: Callable[..., subprocess.CompletedProcess[str]], -) -> str: +) -> ExistingPullRequest | None: branch_name = branch.strip() if not branch_name or branch_name in {"HEAD", "unknown"}: - return "" + return None if gh_path is None: - return "" + return None listed = run_process( - [gh_path, "pr", "list", "--head", branch_name, "--state", "open", "--json", "url", "--jq", ".[0].url"], + [ + gh_path, + "pr", + "list", + "--head", + branch_name, + "--state", + "open", + "--limit", + "1", + "--json", + "url,baseRefName", + ], cwd=str(git_root), text=True, capture_output=True, check=False, ) if listed.returncode != 0: - return "" - return listed.stdout.strip() + return None + try: + payload = json.loads(listed.stdout or "[]") + except (TypeError, json.JSONDecodeError): + return None + if not isinstance(payload, list) or not payload or not isinstance(payload[0], dict): + return None + url = str(payload[0].get("url") or "").strip() + if not url: + return None + return ExistingPullRequest( + url=url, + base_branch=str(payload[0].get("baseRefName") or "").strip(), + ) -def run_git( +def existing_pr_url( git_root: Path, - args: list[str], + branch: str, *, + gh_path: str | None, + run_process: Callable[..., subprocess.CompletedProcess[str]], +) -> str: + existing = existing_pull_request( + git_root, + branch, + gh_path=gh_path, + run_process=run_process, + ) + return existing.url if existing is not None else "" + + +def update_pull_request_base( + git_root: Path, + pull_request: ExistingPullRequest, + base_branch: str, + *, + gh_path: str | None, run_process: Callable[..., subprocess.CompletedProcess[str]], ) -> subprocess.CompletedProcess[str]: + branch = base_branch.strip() + command = [*([gh_path] if gh_path else ["gh"]), "pr", "edit", pull_request.url, "--base", branch] + if gh_path is None: + return subprocess.CompletedProcess( + command, + 127, + stdout="", + stderr=( + "gh is required to update the existing pull request base. " + f"Install or authenticate gh, then run: gh pr edit {pull_request.url} --base {branch}" + ), + ) return run_process( - ["git", "-C", str(git_root), *args], + command, + cwd=str(git_root), text=True, capture_output=True, check=False, ) +def run_git( + git_root: Path, + args: list[str], + *, + run_process: Callable[..., subprocess.CompletedProcess[str]], +) -> subprocess.CompletedProcess[str]: + command = ["git", "-C", str(git_root), *args] + options: dict[str, object] = { + "text": True, + "capture_output": True, + "check": False, + } + if args[:1] == ["ls-remote"]: + options["timeout"] = 10 + options["env"] = {**os.environ, "GIT_TERMINAL_PROMPT": "0"} + try: + return run_process(command, **options) + except subprocess.TimeoutExpired as exc: + return subprocess.CompletedProcess( + command, + 124, + stdout=str(exc.stdout or ""), + stderr="Timed out while querying remote branches.", + ) + + def git_output( git_root: Path, args: list[str], diff --git a/python/envctl_engine/actions/action_pr_git_support.py b/python/envctl_engine/actions/action_pr_git_support.py new file mode 100644 index 000000000..bf20826a8 --- /dev/null +++ b/python/envctl_engine/actions/action_pr_git_support.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from pathlib import Path +import shutil +import subprocess + +from envctl_engine.actions.action_git_state_support import ( + ExistingPullRequest, + existing_pr_url as lookup_existing_pr_url, + existing_pull_request as lookup_existing_pull_request, + update_pull_request_base as update_existing_pull_request_base, +) + + +def existing_pr_url(git_root: Path, branch: str) -> str: + return lookup_existing_pr_url( + git_root, + branch, + gh_path=shutil.which("gh"), + run_process=subprocess.run, + ) + + +def existing_pull_request(git_root: Path, branch: str) -> ExistingPullRequest | None: + return lookup_existing_pull_request( + git_root, + branch, + gh_path=shutil.which("gh"), + run_process=subprocess.run, + ) + + +def update_pull_request_base( + git_root: Path, + pull_request: ExistingPullRequest, + base_branch: str, +) -> subprocess.CompletedProcess[str]: + return update_existing_pull_request_base( + git_root, + pull_request, + base_branch, + gh_path=shutil.which("gh"), + run_process=subprocess.run, + ) + + +__all__ = ["existing_pr_url", "existing_pull_request", "update_pull_request_base"] diff --git a/python/envctl_engine/actions/action_pr_workflow_support.py b/python/envctl_engine/actions/action_pr_workflow_support.py index 523b7897a..da6ecf87a 100644 --- a/python/envctl_engine/actions/action_pr_workflow_support.py +++ b/python/envctl_engine/actions/action_pr_workflow_support.py @@ -5,6 +5,12 @@ import subprocess from typing import Any, Callable +from envctl_engine.actions.action_git_state_support import ExistingPullRequest + + +ExistingPullRequestLookup = Callable[[Path, str], ExistingPullRequest | None] +PullRequestBaseUpdater = Callable[[Path, ExistingPullRequest, str], subprocess.CompletedProcess[str]] + def run_pr_workflow( context: Any, @@ -13,7 +19,8 @@ def run_pr_workflow( git_available: bool, git_output_fn: Callable[[Path, list[str]], str], resolve_base_branch_fn: Callable[[Any, Path], str], - existing_pr_url_fn: Callable[[Path, str], str], + existing_pull_request_fn: ExistingPullRequestLookup, + update_pull_request_base_fn: PullRequestBaseUpdater, probe_dirty_worktree_fn: Callable[[Path, Path, str], Any], run_commit_action_fn: Callable[[Any], int], pr_title_fn: Callable[[Any, Path, str], str], @@ -32,13 +39,31 @@ def run_pr_workflow( if head_branch in {"HEAD", "unknown"}: print(f"Skipping {context.project_name} (detached HEAD).") return 0 - base_branch = resolve_base_branch_fn(context, git_root) - existing_url = existing_pr_url_fn(git_root, head_branch) - if existing_url: - print(f"PR already exists: {existing_url}") + explicit_base = _explicit_pr_base(context) + existing = existing_pull_request_fn(git_root, head_branch) + if existing is not None: + if explicit_base and existing.base_branch != explicit_base: + if not _update_existing_pull_request_base( + git_root=git_root, + head_branch=head_branch, + existing=existing, + desired_base=explicit_base, + existing_pull_request_fn=existing_pull_request_fn, + update_pull_request_base_fn=update_pull_request_base_fn, + print_process_output_fn=print_process_output_fn, + ): + return 1 + print(f"PR already exists: {existing.url}") return 0 + base_branch = resolve_base_branch_fn(context, git_root) + if base_branch and head_branch == base_branch: + print( + f"Cannot open a pull request from {head_branch!r} into itself. Create or switch to a feature branch first." + ) + return 1 + dirty_report = probe_dirty_worktree_fn(context.project_root, context.repo_root, context.project_name) if dirty_report.dirty: print(f"Dirty worktree detected for {context.project_name}; committing and pushing before PR creation.") @@ -54,6 +79,7 @@ def run_pr_workflow( git_root=git_root, head_branch=head_branch, base_branch=base_branch, + existing_pull_request_fn=existing_pull_request_fn, print_process_output_fn=print_process_output_fn, run_process_fn=run_process_fn, ) @@ -70,11 +96,58 @@ def run_pr_workflow( pr_title_fn=pr_title_fn, pr_body_fn=pr_body_fn, write_pr_body_file_fn=write_pr_body_file_fn, + existing_pull_request_fn=existing_pull_request_fn, print_process_output_fn=print_process_output_fn, run_process_fn=run_process_fn, ) +def _explicit_pr_base(context: Any) -> str: + env = getattr(context, "env", {}) + try: + return str(env.get("ENVCTL_PR_BASE", "")).strip() + except AttributeError: + return "" + + +def _update_existing_pull_request_base( + *, + git_root: Path, + head_branch: str, + existing: ExistingPullRequest, + desired_base: str, + existing_pull_request_fn: ExistingPullRequestLookup, + update_pull_request_base_fn: PullRequestBaseUpdater, + print_process_output_fn: Callable[[subprocess.CompletedProcess[str]], None], +) -> bool: + updated = update_pull_request_base_fn(git_root, existing, desired_base) + if updated.returncode != 0: + refreshed = existing_pull_request_fn(git_root, head_branch) + if _pull_request_targets(refreshed, desired_base): + return True + print_process_output_fn(updated) + print( + f"Unable to update existing PR {existing.url} from base " + f"{existing.base_branch or ''!r} to {desired_base!r}." + ) + return False + + refreshed = existing_pull_request_fn(git_root, head_branch) + if not _pull_request_targets(refreshed, desired_base): + actual_base = refreshed.base_branch if refreshed is not None else "" + print( + f"Updated existing PR {existing.url}, but could not verify base {desired_base!r} " + f"(reported {actual_base!r})." + ) + return False + print(f"Updated PR base to {desired_base}: {refreshed.url}") + return True + + +def _pull_request_targets(pull_request: ExistingPullRequest | None, base_branch: str) -> bool: + return pull_request is not None and pull_request.base_branch == base_branch + + def _run_create_pr_helper( context: Any, *, @@ -82,6 +155,7 @@ def _run_create_pr_helper( git_root: Path, head_branch: str, base_branch: str, + existing_pull_request_fn: ExistingPullRequestLookup, print_process_output_fn: Callable[[subprocess.CompletedProcess[str]], None], run_process_fn: Callable[..., subprocess.CompletedProcess[str]], ) -> int: @@ -96,9 +170,17 @@ def _run_create_pr_helper( capture_output=True, check=False, ) - print_process_output_fn(created) if created.returncode != 0: + if _recover_concurrent_pull_request( + git_root=git_root, + head_branch=head_branch, + desired_base=base_branch, + existing_pull_request_fn=existing_pull_request_fn, + ): + return 0 + print_process_output_fn(created) return 1 + print_process_output_fn(created) return 0 @@ -112,6 +194,7 @@ def _run_gh_pr_create( pr_title_fn: Callable[[Any, Path, str], str], pr_body_fn: Callable[[Any, Path, str, str], str], write_pr_body_file_fn: Callable[[str], Path], + existing_pull_request_fn: ExistingPullRequestLookup, print_process_output_fn: Callable[[subprocess.CompletedProcess[str]], None], run_process_fn: Callable[..., subprocess.CompletedProcess[str]], ) -> int: @@ -123,12 +206,34 @@ def _run_gh_pr_create( args.extend(["--base", base_branch]) try: created = run_process_fn(args, cwd=str(git_root), text=True, capture_output=True, check=False) - print_process_output_fn(created) if created.returncode != 0: + if _recover_concurrent_pull_request( + git_root=git_root, + head_branch=head_branch, + desired_base=base_branch, + existing_pull_request_fn=existing_pull_request_fn, + ): + return 0 + print_process_output_fn(created) return 1 + print_process_output_fn(created) return 0 finally: try: body_file.unlink() except OSError: pass + + +def _recover_concurrent_pull_request( + *, + git_root: Path, + head_branch: str, + desired_base: str, + existing_pull_request_fn: ExistingPullRequestLookup, +) -> bool: + existing = existing_pull_request_fn(git_root, head_branch) + if not _pull_request_targets(existing, desired_base): + return False + print(f"PR already exists: {existing.url}") + return True diff --git a/python/envctl_engine/actions/action_protected_artifacts.py b/python/envctl_engine/actions/action_protected_artifacts.py index 90baf0201..901699982 100644 --- a/python/envctl_engine/actions/action_protected_artifacts.py +++ b/python/envctl_engine/actions/action_protected_artifacts.py @@ -63,6 +63,26 @@ def ordered_unique_paths(*path_groups: list[str]) -> list[str]: return paths +def unstaged_stageable_paths(status_output: str) -> list[str]: + paths: list[str] = [] + seen: set[str] = set() + for raw_line in str(status_output or "").splitlines(): + line = raw_line.rstrip("\n") + if len(line) < 4: + continue + is_untracked = line.startswith("?? ") + worktree_status = line[1] + if not is_untracked and worktree_status == " ": + continue + candidates = status_candidate_paths(line, include_rename_source=not is_untracked) + for candidate in candidates: + if is_envctl_local_artifact_path(candidate) or candidate in seen: + continue + paths.append(candidate) + seen.add(candidate) + return paths + + def status_candidate_path(line: str) -> str: if line.startswith("?? "): return line[3:].strip() @@ -72,3 +92,16 @@ def status_candidate_path(line: str) -> str: if " -> " in payload: return payload.split(" -> ", 1)[1].strip() return payload + + +def status_candidate_paths(line: str, *, include_rename_source: bool = False) -> list[str]: + candidate = status_candidate_path(line) + if not candidate: + return [] + if not include_rename_source: + return [candidate] + payload = line[3:].strip() + if " -> " not in payload: + return [candidate] + source = payload.split(" -> ", 1)[0].strip() + return [source, candidate] if source and source != candidate else [candidate] diff --git a/python/envctl_engine/actions/action_ship_checks.py b/python/envctl_engine/actions/action_ship_checks.py index 56bc6e953..26b0f1dd0 100644 --- a/python/envctl_engine/actions/action_ship_checks.py +++ b/python/envctl_engine/actions/action_ship_checks.py @@ -30,7 +30,7 @@ TERMINAL_SHIP_CHECK_STATES = {"checks_passed", "checks_failed", "gh_unavailable", "no_checks_reported"} SHIP_NO_CHECKS_GRACE_ENV = "ENVCTL_SHIP_NO_CHECKS_GRACE_SECONDS" -DEFAULT_CHECK_TIMEOUT_SECONDS = 120.0 +DEFAULT_CHECK_TIMEOUT_SECONDS = 180.0 DEFAULT_CHECK_POLL_INTERVAL_SECONDS = 5.0 DEFAULT_CHECK_PROGRESS_INTERVAL_SECONDS = 10.0 DEFAULT_NO_CHECKS_GRACE_SECONDS = 15.0 @@ -99,8 +99,13 @@ class ShipCheckPoller: def run(self) -> dict[str, object]: next_progress_at = self.timing.progress_interval_seconds + last_observed_checks: Mapping[str, object] | None = None while True: result = self._query() + if _result_reports_target_checks(result): + last_observed_checks = result + elif result.get("state") == "no_checks_reported" and last_observed_checks is not None: + result = _pending_after_transient_empty_rollup(result, last_observed_checks) if _ship_check_result_is_terminal(result): return result @@ -238,6 +243,42 @@ def _ship_check_result_is_terminal(result: Mapping[str, object]) -> bool: return not _failed_check_logs_are_retryable(result) +def _result_reports_target_checks(result: Mapping[str, object]) -> bool: + synthetic_waiters = {"github_checks", "github_head_ref"} + for key in ("failing_checks", "passed_checks", "pending_checks"): + checks = result.get(key) + if not isinstance(checks, list): + continue + for check in checks: + if not isinstance(check, Mapping): + continue + name = str(check.get("name") or "").strip() + if name and name not in synthetic_waiters: + return True + return False + + +def _pending_after_transient_empty_rollup( + result: Mapping[str, object], + last_observed: Mapping[str, object], +) -> dict[str, object]: + pending = last_observed.get("pending_checks") + pending_checks = list(pending) if isinstance(pending, list) and pending else [ + {"name": "github_checks", "state": "WAITING"} + ] + passed = last_observed.get("passed_checks") + pr_checks = last_observed.get("pr_checks") + return { + **result, + "state": "checks_pending_timeout", + "failing_checks": [], + "passed_checks": list(passed) if isinstance(passed, list) else [], + "pending_checks": pending_checks, + "pr_checks": list(pr_checks) if isinstance(pr_checks, list) else [], + "error": "GitHub temporarily returned an empty check rollup after target checks were already observed.", + } + + def _check_progress_message( result: Mapping[str, object], *, diff --git a/python/envctl_engine/actions/action_ship_checks_phase.py b/python/envctl_engine/actions/action_ship_checks_phase.py new file mode 100644 index 000000000..f9103492c --- /dev/null +++ b/python/envctl_engine/actions/action_ship_checks_phase.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from envctl_engine.actions.action_ship_contract import emit_ship_progress +from envctl_engine.actions.action_ship_phase_status import ( + callable_accepts_keyword, + check_phase_status, + ship_status_is_success, +) + +GithubPrChecks = Callable[..., dict[str, object]] +FinishShip = Callable[..., int] + + +def run_ship_checks_phase( + *, + state: Any, + checks_fn: GithubPrChecks, + finish: FinishShip, +) -> int: + check_kwargs: dict[str, object] = { + "branch": state.branch, + "pr_url": state.pr_url, + "expected_head_sha": state.after_sha, + } + if callable_accepts_keyword(checks_fn, "progress_callback"): + check_kwargs["progress_callback"] = emit_ship_progress + checks = checks_fn(state.git_root, **check_kwargs) + status = check_phase_status(checks) + state.step_statuses.append(status) + return finish( + state, + status=status, + ok=ship_status_is_success(status), + commit_sha=state.after_sha, + pushed=state.pushed, + pr_url=state.pr_url, + pr_created=state.pr_created, + checks=checks, + merge_conflicts=state.merge_conflicts, + ) + + +__all__ = ["run_ship_checks_phase"] diff --git a/python/envctl_engine/actions/action_ship_conflicts.py b/python/envctl_engine/actions/action_ship_conflicts.py index d0047d032..919210dd4 100644 --- a/python/envctl_engine/actions/action_ship_conflicts.py +++ b/python/envctl_engine/actions/action_ship_conflicts.py @@ -62,9 +62,10 @@ def predicted_merge_conflict_report( resolve_base_ref: ResolveBaseRef, run_git: RunGit, git_output: GitOutput, + base_branch_override: str = "", ) -> dict[str, object]: del branch - base_branch = resolve_base_branch(context, git_root) + base_branch = base_branch_override.strip() or resolve_base_branch(context, git_root) base_ref = resolve_base_ref(git_root, base_branch) if not base_ref: return { diff --git a/python/envctl_engine/actions/action_ship_pr_context.py b/python/envctl_engine/actions/action_ship_pr_context.py new file mode 100644 index 000000000..fdc36b6f0 --- /dev/null +++ b/python/envctl_engine/actions/action_ship_pr_context.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import Any, Protocol + +from envctl_engine.actions.action_git_state_support import ExistingPullRequest + +ExistingPullRequestLookup = Callable[[Path, str], ExistingPullRequest | None] + + +class ShipPullRequestState(Protocol): + git_root: Path + branch: str + pr_url: str + pr_base_branch: str + + +def explicit_pr_base(context: Any) -> str: + env = getattr(context, "env", {}) + try: + return str(env.get("ENVCTL_PR_BASE", "")).strip() + except AttributeError: + return "" + + +def refresh_existing_pull_request( + state: ShipPullRequestState, + lookup: ExistingPullRequestLookup | None, +) -> None: + if lookup is None: + return + existing = lookup(state.git_root, state.branch) + if existing is None: + return + state.pr_url = existing.url + state.pr_base_branch = existing.base_branch + + +def conflict_base_branch_override(context: Any, state: ShipPullRequestState) -> str: + return explicit_pr_base(context) or state.pr_base_branch + + +__all__ = [ + "ExistingPullRequestLookup", + "conflict_base_branch_override", + "explicit_pr_base", + "refresh_existing_pull_request", +] diff --git a/python/envctl_engine/actions/action_ship_pr_phase.py b/python/envctl_engine/actions/action_ship_pr_phase.py index f328c3604..68b693a48 100644 --- a/python/envctl_engine/actions/action_ship_pr_phase.py +++ b/python/envctl_engine/actions/action_ship_pr_phase.py @@ -4,6 +4,7 @@ from typing import Any, Callable from envctl_engine.actions.action_ship_contract import emit_ship_progress +from envctl_engine.actions.action_ship_pr_context import explicit_pr_base as _explicit_pr_base FinishShip = Callable[..., int] @@ -18,6 +19,19 @@ def run_ship_pr_phase( finish: FinishShip, ) -> int | None: if state.pr_url: + explicit_base = _explicit_pr_base(context) + if explicit_base: + pr_code = run_pr_action(context) + if pr_code != 0: + return finish( + state, + status="pr_failed", + ok=False, + commit_sha=state.after_sha, + pr_url=state.pr_url, + pr_created=False, + ) + state.pr_url = existing_pr_url(state.git_root, state.branch) or state.pr_url state.step_statuses.append("pr_exists") emit_ship_progress(f"ship: PR already exists for {context.project_name}: {state.pr_url}") return None diff --git a/python/envctl_engine/actions/action_ship_support.py b/python/envctl_engine/actions/action_ship_support.py index 87c96088e..407eb1dc9 100644 --- a/python/envctl_engine/actions/action_ship_support.py +++ b/python/envctl_engine/actions/action_ship_support.py @@ -9,6 +9,7 @@ from envctl_engine.actions.action_ship_check_results import normalize_github_pr_checks # noqa: F401 from envctl_engine.actions.action_ship_checks import github_pr_checks as default_github_pr_checks +from envctl_engine.actions.action_ship_checks_phase import run_ship_checks_phase from envctl_engine.actions.action_ship_conflicts import ( existing_merge_conflict_report, parse_merge_tree_conflicts, # noqa: F401 @@ -17,17 +18,16 @@ ) from envctl_engine.actions.action_ship_contract import ( emit_ship_commit_progress, - emit_ship_progress, parse_ship_json_output, print_ship_result, # noqa: F401 ship_payload, # noqa: F401 ship_protected_paths, ) from envctl_engine.actions.action_ship_finish import finish_ship_workflow -from envctl_engine.actions.action_ship_phase_status import ( - callable_accepts_keyword, - check_phase_status, - ship_status_is_success, +from envctl_engine.actions.action_ship_pr_context import ( + ExistingPullRequestLookup, + conflict_base_branch_override, + refresh_existing_pull_request, ) from envctl_engine.actions.action_ship_pr_phase import run_ship_pr_label_phase, run_ship_pr_phase from envctl_engine.actions.action_ship_push import run_ship_push_phase @@ -39,6 +39,7 @@ GithubPrChecks = Callable[..., dict[str, object]] github_pr_checks = default_github_pr_checks + @dataclass(slots=True) class ShipWorkflowState: git_root: Path @@ -51,6 +52,7 @@ class ShipWorkflowState: committed: bool = False pushed: bool = False pr_url: str = "" + pr_base_branch: str = "" pr_created: bool = False step_statuses: list[str] = field(default_factory=list) merge_conflicts: dict[str, object] = field(default_factory=dict) @@ -72,6 +74,7 @@ class ShipWorkflowDependencies: partition_envctl_protected_paths: Callable[[str], Any] ordered_unique_paths: Callable[..., list[str]] github_pr_checks: GithubPrChecks | None = None + existing_pull_request: ExistingPullRequestLookup | None = None @dataclass(slots=True) @@ -144,7 +147,10 @@ def _run_commit_phase(self, state: ShipWorkflowState) -> int | None: self.context.repo_root, project_name=self.context.project_name, ) - state.pr_url = self.dependencies.existing_pr_url(state.git_root, state.branch) + if self.dependencies.existing_pull_request is None: + state.pr_url = self.dependencies.existing_pr_url(state.git_root, state.branch) + else: + refresh_existing_pull_request(state, self.dependencies.existing_pull_request) commit_code = self.dependencies.run_commit_action(self.context) state.after_sha = self.dependencies.git_output(state.git_root, ["rev-parse", "HEAD"]).strip() @@ -159,13 +165,16 @@ def _run_commit_phase(self, state: ShipWorkflowState) -> int | None: return self._finish(state, status="commit_failed", ok=False, commit_sha=state.after_sha) def _run_pr_phase(self, state: ShipWorkflowState) -> int | None: - return run_ship_pr_phase( + result = run_ship_pr_phase( state=state, context=self.context, run_pr_action=self.dependencies.run_pr_action, existing_pr_url=self.dependencies.existing_pr_url, finish=self._finish, ) + if result is None: + refresh_existing_pull_request(state, self.dependencies.existing_pull_request) + return result def _run_pr_label_phase(self, state: ShipWorkflowState) -> int | None: return run_ship_pr_label_phase( @@ -184,6 +193,7 @@ def _reject_predicted_merge_conflicts(self, state: ShipWorkflowState) -> int | N resolve_base_ref=self.dependencies.resolve_base_ref, run_git=self.dependencies.run_git, git_output=self.dependencies.git_output, + base_branch_override=conflict_base_branch_override(self.context, state), ) if state.merge_conflicts.get("state") != "conflicts": return None @@ -228,27 +238,10 @@ def _run_push_phase(self, state: ShipWorkflowState) -> int | None: return None def _run_checks_phase(self, state: ShipWorkflowState) -> int: - checks_fn = self.dependencies.github_pr_checks or default_github_pr_checks - check_kwargs: dict[str, object] = { - "branch": state.branch, - "pr_url": state.pr_url, - "expected_head_sha": state.after_sha, - } - if callable_accepts_keyword(checks_fn, "progress_callback"): - check_kwargs["progress_callback"] = emit_ship_progress - checks = checks_fn(state.git_root, **check_kwargs) - status = check_phase_status(checks) - state.step_statuses.append(status) - return self._finish( - state, - status=status, - ok=ship_status_is_success(status), - commit_sha=state.after_sha, - pushed=state.pushed, - pr_url=state.pr_url, - pr_created=state.pr_created, - checks=checks, - merge_conflicts=state.merge_conflicts, + return run_ship_checks_phase( + state=state, + checks_fn=self.dependencies.github_pr_checks or default_github_pr_checks, + finish=self._finish, ) def _finish( @@ -277,8 +270,10 @@ def _finish( merge_conflicts=merge_conflicts, ) + def run_ship_workflow( - context: Any, *, + context: Any, + *, resolve_git_root: Callable[[Path, Path], Path], git_available: bool, git_output: GitOutput, @@ -292,7 +287,9 @@ def run_ship_workflow( existing_pr_url: Callable[[Path, str], str], partition_envctl_protected_paths: Callable[[str], Any], ordered_unique_paths: Callable[..., list[str]], - github_pr_checks: GithubPrChecks | None = None) -> int: + github_pr_checks: GithubPrChecks | None = None, + existing_pull_request: ExistingPullRequestLookup | None = None, +) -> int: return ShipWorkflowRunner( context=context, dependencies=ShipWorkflowDependencies( @@ -310,5 +307,6 @@ def run_ship_workflow( partition_envctl_protected_paths=partition_envctl_protected_paths, ordered_unique_paths=ordered_unique_paths, github_pr_checks=github_pr_checks, + existing_pull_request=existing_pull_request, ), ).run() diff --git a/python/envctl_engine/actions/action_target_support.py b/python/envctl_engine/actions/action_target_support.py index 7eba9d430..2a52b5b59 100644 --- a/python/envctl_engine/actions/action_target_support.py +++ b/python/envctl_engine/actions/action_target_support.py @@ -5,6 +5,7 @@ from typing import Any, Callable, Mapping, Sequence, cast from envctl_engine.runtime.command_router import Route +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.ui.selection_types import TargetSelection @@ -68,7 +69,8 @@ def action_target_names_with_roots(targets: Sequence[object]) -> list[str]: def build_action_target_contexts(targets: Sequence[object]) -> list[ActionTargetContext]: - total = max(len(targets), 1) + identities = action_target_identities(targets) + total = len(identities) return [ ActionTargetContext( name=identity.name, @@ -77,7 +79,7 @@ def build_action_target_contexts(targets: Sequence[object]) -> list[ActionTarget total=total, target_obj=identity.target_obj, ) - for index, identity in enumerate(action_target_identities(targets), start=1) + for index, identity in enumerate(identities, start=1) ] @@ -100,42 +102,257 @@ def emit_action_output( return True +def _normalized_service_targets(service_targets: list[object]) -> list[str]: + normalized: list[str] = [] + for target in service_targets: + value = str(target).strip().lower().removeprefix("service:") + if value: + normalized.append(value) + return normalized + + +def _projects_by_service_selector(runtime: Any, state: object | None) -> dict[str, list[str]]: + projects_by_selector: dict[str, list[str]] = {} + services = getattr(state, "services", {}) if state is not None else {} + for service_name, service in services.items(): + project = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime.project_name_from_service, + ) + if not project: + continue + service_type = str(getattr(service, "type", "") or "").strip().lower() + selectors = {service_name.lower(), service_type} - {""} + for selector in selectors: + projects_by_selector.setdefault(selector, []).append(project) + return projects_by_selector + + +def _deduplicate_project_names(projects: list[str]) -> list[str]: + deduped: list[str] = [] + seen: set[str] = set() + for project in projects: + lowered = project.lower() + if lowered not in seen: + seen.add(lowered) + deduped.append(project) + return deduped + + def projects_for_services(runtime: Any, service_targets: list[object]) -> list[str]: - normalized_targets = [ - str(target).strip().lower().removeprefix("service:") - for target in service_targets - if str(target).strip() - ] + projects, _unresolved = _resolve_projects_for_services(runtime, service_targets) + return projects + + +def _resolve_projects_for_services( + runtime: Any, + service_targets: list[object], +) -> tuple[list[str], tuple[str, ...]]: + normalized_targets = _normalized_service_targets(service_targets) if not normalized_targets: - return [] + return [], () state = runtime.load_existing_state(mode="trees") or runtime.load_existing_state(mode="main") + projects_by_selector = _projects_by_service_selector(runtime, state) resolved: list[str] = [] + unresolved: list[str] = [] for target in normalized_targets: - matched = False - if state is not None: - for service_name, service in state.services.items(): - service_type = str(getattr(service, "type", "") or "").strip().lower() - if service_name.lower() == target or service_type == target: - project = runtime.project_name_from_service(service_name) - if project: - resolved.append(project) - matched = True - if matched: - continue - project = runtime.project_name_from_service(target) - if project: + matched_projects = projects_by_selector.get(target, []) + if matched_projects: + resolved.extend(matched_projects) + elif project := runtime.project_name_from_service(target): resolved.append(project) + else: + unresolved.append(target) + return _deduplicate_project_names(resolved), tuple(unresolved) - deduped: list[str] = [] - seen: set[str] = set() - for project in resolved: - lowered = project.lower() - if lowered in seen: - continue - seen.add(lowered) - deduped.append(project) - return deduped + +def _is_configured_test_service_selector(runtime: Any, route: Route, selector: str) -> bool: + if route.command not in {"test", "test-focused"}: + return False + if selector in {"backend", "frontend"}: + return True + config = getattr(runtime, "config", None) + app_service_by_name = getattr(config, "app_service_by_name", None) + return callable(app_service_by_name) and app_service_by_name(selector) is not None + + +def _discover_action_candidates(runtime: Any, route: Route, *, trees_only: bool) -> list[object]: + mode = "trees" if trees_only else route.mode + candidates = list(runtime.discover_projects(mode=mode)) + if not candidates and not trees_only and route.mode == "main": + return list(runtime.discover_projects(mode="trees")) + return candidates + + +def _all_action_candidates(candidates: list[object]) -> tuple[list[object], str | None]: + if not candidates: + return [], "No projects discovered for selected mode." + return candidates, None + + +@dataclass(frozen=True) +class _ActionTargetRequest: + run_all: bool + untested_selected: bool + project_selectors: frozenset[str] + explicit_targeting: bool + unresolved_service_selectors: tuple[str, ...] + + +@dataclass +class _ActionTargetResolver: + runtime: Any + route: Route + trees_only: bool + resolve_current_worktree_target: Callable[..., object | None] + interactive_selection_allowed: Callable[[Route], bool] + no_target_selected_message: Callable[[Route], str] + + def resolve(self) -> tuple[list[object], str | None]: + request = self._request_from_route() + if current_result := self._current_worktree_result(request): + return current_result + candidates = _discover_action_candidates(self.runtime, self.route, trees_only=self.trees_only) + if request.run_all: + return _all_action_candidates(candidates) + if request.project_selectors: + return self._match_request(candidates, request) + return self._resolve_without_project_selectors(candidates, request) + + def _resolve_without_project_selectors( + self, + candidates: list[object], + request: _ActionTargetRequest, + ) -> tuple[list[object], str | None]: + if len(candidates) == 1 and not request.explicit_targeting: + return candidates, None + if not candidates and not request.explicit_targeting: + return _all_action_candidates(candidates) + if request.explicit_targeting: + return self._match_request(candidates, request) + interactive_request, error = self._request_from_interactive_selection(candidates) + if error is not None: + return [], error + return self._match_request(candidates, interactive_request) + + def _current_worktree_result( + self, + request: _ActionTargetRequest, + ) -> tuple[list[object], str | None] | None: + if self.trees_only and self.route.command == "self-destruct-worktree": + current = self.resolve_current_worktree_target() + if current is not None: + return [current], None + return [], "self-destruct-worktree must be run from inside a discovered worktree." + current_worktree_commands = {"commit", "pr", "ship", "test", "test-focused"} + if self.trees_only or self.route.command not in current_worktree_commands or request.explicit_targeting: + return None + current = self.resolve_current_worktree_target( + require_configured_main_root=self.route.mode == "main", + require_configured_root_match=True, + ) + if current is not None: + return [current], None + if self._execution_root_differs_from_configured_root(): + return [], self.no_target_selected_message(self.route) + return None + + def _execution_root_differs_from_configured_root(self) -> bool: + raw_runtime = getattr(self.runtime, "raw_runtime", self.runtime) + config = getattr(raw_runtime, "config", None) + base_dir = getattr(config, "base_dir", None) + execution_root = getattr(config, "execution_root", None) + if base_dir is None or execution_root is None: + return False + try: + return Path(str(base_dir)).expanduser().resolve() != Path(str(execution_root)).expanduser().resolve() + except OSError: + return True + + def _request_from_route(self) -> _ActionTargetRequest: + project_selectors = {name.lower() for name in self.route.projects} + project_selectors.update(self.runtime.selectors_from_passthrough(self.route.passthrough_args)) + services = self.route.flags.get("services") + unresolved_service_selectors: tuple[str, ...] = () + if isinstance(services, list): + project_target_services = _normalized_service_targets(services) + if self.route.command in {"test", "test-focused"}: + project_target_services = [ + selector for selector in project_target_services if selector not in {"backend", "frontend"} + ] + service_projects, unresolved_service_selectors = _resolve_projects_for_services( + self.runtime, + project_target_services, + ) + unresolved_service_selectors = tuple( + selector + for selector in unresolved_service_selectors + if not _is_configured_test_service_selector(self.runtime, self.route, selector) + ) + project_selectors.update(project.lower() for project in service_projects) + run_all = bool(self.route.flags.get("all")) + untested_selected = bool(self.route.flags.get("untested")) + return _ActionTargetRequest( + run_all=run_all, + untested_selected=untested_selected, + project_selectors=frozenset(project_selectors), + explicit_targeting=bool( + project_selectors or unresolved_service_selectors or run_all or untested_selected + ), + unresolved_service_selectors=unresolved_service_selectors, + ) + + def _request_from_interactive_selection( + self, + candidates: list[object], + ) -> tuple[_ActionTargetRequest, str | None]: + if not self.interactive_selection_allowed(self.route): + return self._request_from_route(), self.no_target_selected_message(self.route) + selection = cast( + TargetSelection, + self.runtime.select_project_targets( + prompt=f"Select {self.route.command} target", + projects=candidates, + allow_all=True, + allow_untested=self.route.command == "test", + multi=True, + ), + ) + if selection.cancelled: + return self._request_from_route(), self.no_target_selected_message(self.route) + selection.apply_to_route(self.route) + return self._request_from_route(), None + + def _match_request( + self, + candidates: list[object], + request: _ActionTargetRequest, + ) -> tuple[list[object], str | None]: + if request.run_all: + return _all_action_candidates(candidates) + if request.unresolved_service_selectors: + requested = ", ".join(request.unresolved_service_selectors) + return [], f"No matching targets found for: {requested}" + if not request.project_selectors: + return self._empty_request_result(request) + selected = [ + candidate + for candidate in candidates + if candidate.name.lower() in request.project_selectors + ] + selected_names = {candidate.name.lower() for candidate in selected} + missing_selectors = request.project_selectors - selected_names + if missing_selectors: + requested = ", ".join(sorted(missing_selectors)) + return [], f"No matching targets found for: {requested}" + return selected, None + + def _empty_request_result(self, request: _ActionTargetRequest) -> tuple[list[object], str | None]: + if self.route.command == "test" and request.untested_selected: + return [], None + return [], self.no_target_selected_message(self.route) def resolve_action_targets( @@ -147,77 +364,129 @@ def resolve_action_targets( interactive_selection_allowed: Callable[[Route], bool], no_target_selected_message: Callable[[Route], str], ) -> tuple[list[object], str | None]: - if trees_only and route.command == "self-destruct-worktree": - current = resolve_current_worktree_target() - if current is not None: - return [current], None - return [], "self-destruct-worktree must be run from inside a discovered worktree." - current_worktree_commands = {"commit", "pr", "ship", "test", "test-focused"} - if not trees_only and route.command in current_worktree_commands: - current = resolve_current_worktree_target( - require_configured_main_root=route.mode == "main", - require_configured_root_match=True, + return _ActionTargetResolver( + runtime=runtime, + route=route, + trees_only=trees_only, + resolve_current_worktree_target=resolve_current_worktree_target, + interactive_selection_allowed=interactive_selection_allowed, + no_target_selected_message=no_target_selected_message, + ).resolve() + + +def _exception_message(exc: Exception) -> str: + detail = str(exc).strip() + return f"{type(exc).__name__}: {detail}" if detail else type(exc).__name__ + + +def _completed_process_error(completed: Any) -> str | None: + if completed.returncode == 0: + return None + stdout = str(getattr(completed, "stdout", "") or "").strip() + stderr = str(getattr(completed, "stderr", "") or "").strip() + if stderr and stdout and stderr != stdout: + return f"{stderr}\n\nstdout:\n{stdout}" + return stderr or stdout or f"exit:{completed.returncode}" + + +def _run_action_target( + context: ActionTargetContext, + *, + resolve_command: Callable[[ActionTargetContext], ActionCommandResolution], + build_env: Callable[[ActionTargetContext], Mapping[str, str]], + process_run: Callable[[list[str], Path, Mapping[str, str]], Any], +) -> tuple[Any | None, str | None]: + try: + resolution = resolve_command(context) + if resolution.command is None or resolution.cwd is None: + return None, resolution.error or "no command resolved" + completed = process_run(resolution.command, resolution.cwd, build_env(context)) + return completed, _completed_process_error(completed) + except Exception as exc: # Keep independent targets running after one operational failure. + return None, _exception_message(exc) + + +def _report_action_failure( + context: ActionTargetContext, + error: str, + *, + command_name: str, + interactive_command: bool, + should_print: bool, + emit_status: Callable[[str], None], + printer: Callable[[str], None], + on_failure: Callable[[ActionTargetContext, str], None] | None, + failure_status_formatter: Callable[[ActionTargetContext, str], str] | None, +) -> None: + if on_failure is not None: + on_failure(context, error) + if should_print: + printer(f"{command_name} action failed for {context.name}: {error}") + if interactive_command: + status = ( + failure_status_formatter(context, error) + if failure_status_formatter is not None + else f"{command_name} failed for {context.name}: {error}" ) - if current is not None: - return [current], None - if trees_only: - candidates = runtime.discover_projects(mode="trees") - else: - candidates = runtime.discover_projects(mode=route.mode) - if not candidates and route.mode == "main": - candidates = runtime.discover_projects(mode="trees") - - run_all = bool(route.flags.get("all")) - untested_selected = bool(route.flags.get("untested")) - project_selectors = {name.lower() for name in route.projects} - project_selectors.update(runtime.selectors_from_passthrough(route.passthrough_args)) - - services = route.flags.get("services") - if isinstance(services, list): - for project in projects_for_services(runtime, services): - project_selectors.add(project.lower()) - - if run_all: - if not candidates: - return [], "No projects discovered for selected mode." - return candidates, None - - if not project_selectors and not run_all: - if len(candidates) == 1: - return candidates, None - if interactive_selection_allowed(route): - selection = cast( - TargetSelection, - runtime.select_project_targets( - prompt=f"Select {route.command} target", - projects=candidates, - allow_all=True, - allow_untested=route.command == "test", - multi=True, - ), - ) - if selection.cancelled: - return [], no_target_selected_message(route) - selection.apply_to_route(route) - run_all = bool(route.flags.get("all")) - project_selectors = {name.lower() for name in route.projects} - untested_selected = bool(route.flags.get("untested")) - else: - return [], no_target_selected_message(route) - - if not project_selectors and not run_all and not untested_selected: - return [], no_target_selected_message(route) - if run_all: - if not candidates: - return [], "No projects discovered for selected mode." - return candidates, None - selected = [candidate for candidate in candidates if candidate.name.lower() in project_selectors] - if not selected: - if route.command == "test" and untested_selected: - return [], None - requested = ", ".join(sorted(project_selectors)) - return [], f"No matching targets found for: {requested}" - return selected, None + emit_status(status) + + +def _report_action_success( + context: ActionTargetContext, + completed: Any, + *, + command_name: str, + interactive_command: bool, + emit_success_output: bool, + print_noninteractive_successes: bool, + emit_status: Callable[[str], None], + printer: Callable[[str], None], + on_success: Callable[[ActionTargetContext, Any], None] | None, + success_print_formatter: Callable[[ActionTargetContext, Any], str] | None, + success_status_formatter: Callable[[ActionTargetContext, Any], str] | None, +) -> None: + if emit_success_output: + emit_action_output(completed.stdout, emit_status=emit_status, printer=printer) + if on_success is not None: + on_success(context, completed) + if not interactive_command and print_noninteractive_successes: + message = ( + success_print_formatter(context, completed) + if success_print_formatter is not None + else f"{command_name} action succeeded for {context.name}." + ) + if message: + printer(message) + status = ( + success_status_formatter(context, completed) + if success_status_formatter is not None + else f"{command_name} succeeded for {context.name}" + ) + if status: + emit_status(status) + + +def _prepare_action_contexts( + targets: Sequence[object], + *, + command_name: str, + emit_status: Callable[[str], None], + printer: Callable[[str], None], +) -> tuple[list[ActionTargetContext], int]: + contexts = build_action_target_contexts(targets) + invalid_target_count = len(targets) - len(contexts) + if invalid_target_count: + message = ( + f"{command_name} action skipped {invalid_target_count} invalid target(s) " + "without both a name and root." + ) + printer(message) + emit_status(message) + elif not contexts: + message = f"{command_name} action failed: no targets were provided." + printer(message) + emit_status(message) + return contexts, invalid_target_count def execute_targeted_action( @@ -240,67 +509,53 @@ def execute_targeted_action( print_noninteractive_failures: bool = True, print_noninteractive_successes: bool = True, ) -> int: - def should_print_failure() -> bool: - return (not interactive_command and print_noninteractive_failures) or ( - interactive_command and interactive_print_failures - ) - - failures = 0 - for context in build_action_target_contexts(targets): + contexts, failures = _prepare_action_contexts( + targets, + command_name=command_name, + emit_status=emit_status, + printer=printer, + ) + if not contexts: + return 1 + + should_print_failure = (not interactive_command and print_noninteractive_failures) or ( + interactive_command and interactive_print_failures + ) + for context in contexts: emit_status(f"Running {command_name} for {context.name} ({context.index}/{context.total})...") - resolution = resolve_command(context) - if resolution.command is None or resolution.cwd is None: - raw_error = resolution.error or "no command resolved" - message = f"{command_name} action failed for {context.name}: {raw_error}" - if on_failure is not None: - on_failure(context, raw_error) - if should_print_failure(): - printer(message) - if interactive_command: - if failure_status_formatter is not None: - emit_status(failure_status_formatter(context, raw_error)) - else: - emit_status(f"{command_name} failed for {context.name}: {raw_error}") - failures += 1 - continue - - completed = process_run(resolution.command, resolution.cwd, build_env(context)) - if completed.returncode != 0: - stdout = str(getattr(completed, "stdout", "") or "").strip() - stderr = str(getattr(completed, "stderr", "") or "").strip() - if stderr and stdout and stderr != stdout: - error = f"{stderr}\n\nstdout:\n{stdout}" - else: - error = stderr or stdout or f"exit:{completed.returncode}" - message = f"{command_name} action failed for {context.name}: {error}" - if on_failure is not None: - on_failure(context, error) - if should_print_failure(): - printer(message) - if interactive_command: - if failure_status_formatter is not None: - emit_status(failure_status_formatter(context, error)) - else: - emit_status(f"{command_name} failed for {context.name}: {error}") + completed, error = _run_action_target( + context, + resolve_command=resolve_command, + build_env=build_env, + process_run=process_run, + ) + if error is not None: + _report_action_failure( + context, + error, + command_name=command_name, + interactive_command=interactive_command, + should_print=should_print_failure, + emit_status=emit_status, + printer=printer, + on_failure=on_failure, + failure_status_formatter=failure_status_formatter, + ) failures += 1 continue - - if emit_success_output: - emit_action_output(completed.stdout, emit_status=emit_status, printer=printer) - if on_success is not None: - on_success(context, completed) - if not interactive_command and print_noninteractive_successes: - if success_print_formatter is not None: - message = success_print_formatter(context, completed) - else: - message = f"{command_name} action succeeded for {context.name}." - if message: - printer(message) - if success_status_formatter is not None: - status_message = success_status_formatter(context, completed) - else: - status_message = f"{command_name} succeeded for {context.name}" - if status_message: - emit_status(status_message) + assert completed is not None + _report_action_success( + context, + completed, + command_name=command_name, + interactive_command=interactive_command, + emit_success_output=emit_success_output, + print_noninteractive_successes=print_noninteractive_successes, + emit_status=emit_status, + printer=printer, + on_success=on_success, + success_print_formatter=success_print_formatter, + success_status_formatter=success_status_formatter, + ) return 0 if failures == 0 else 1 diff --git a/python/envctl_engine/actions/action_test_execution_support.py b/python/envctl_engine/actions/action_test_execution_support.py index 13603e2df..9d040b225 100644 --- a/python/envctl_engine/actions/action_test_execution_support.py +++ b/python/envctl_engine/actions/action_test_execution_support.py @@ -31,6 +31,15 @@ def execution_mode(self) -> str: def multi_project(self) -> bool: return len(self.distinct_projects) > 1 + @property + def missing_execution_specs_message(self) -> str: + if self.untested and not self.target_contexts: + return ( + "No test command supports the all-untested scope. " + "Configure a test-all-trees.sh action command or select projects explicitly." + ) + return "No test command configured. Set Backend test command or Frontend test command in envctl config." + @dataclass(frozen=True, slots=True) class SuiteSpinnerDecision: @@ -55,7 +64,11 @@ def build(self) -> TestActionExecutionPlan: self._emit_scope_status() include_backend, include_frontend = self._selected_services() target_contexts = self._target_contexts() - _ensure_test_prereqs(self.orchestrator, target_contexts) + _ensure_test_prereqs( + self.orchestrator, + target_contexts, + aggregate_untested=self.untested and not target_contexts, + ) execution_specs = self._execution_specs( target_contexts=target_contexts, include_backend=include_backend, @@ -113,6 +126,8 @@ def _selected_services(self) -> tuple[bool, bool]: ) def _target_contexts(self) -> list[TestTargetContext]: + if self.untested and not self.targets: + return [] return self.orchestrator._test_target_contexts(self.targets) def _execution_specs( @@ -227,10 +242,18 @@ def print_test_execution_mode(orchestrator: Any, plan: TestActionExecutionPlan) print(orchestrator._colorize(text, fg=mode_color, bold=True)) -def _ensure_test_prereqs(orchestrator: Any, target_contexts: list[TestTargetContext]) -> None: +def _ensure_test_prereqs( + orchestrator: Any, + target_contexts: list[TestTargetContext], + *, + aggregate_untested: bool, +) -> None: + project_roots = [Path(context.project_root) for context in target_contexts] + if aggregate_untested: + project_roots.append(Path(orchestrator.runtime.config.base_dir)) seen_roots: set[Path] = set() - for context in target_contexts: - project_root = Path(context.project_root).resolve() + for root in project_roots: + project_root = root.resolve() if project_root in seen_roots: continue seen_roots.add(project_root) diff --git a/python/envctl_engine/actions/action_test_runner.py b/python/envctl_engine/actions/action_test_runner.py index 5540c15cd..eaad7658b 100644 --- a/python/envctl_engine/actions/action_test_runner.py +++ b/python/envctl_engine/actions/action_test_runner.py @@ -82,7 +82,7 @@ def run_test_action( execution_specs = plan.execution_specs interactive_command = plan.interactive_command if not execution_specs: - print("No test command configured. Set Backend test command or Frontend test command in envctl config.") + print(plan.missing_execution_specs_message) return 1 started_at = started_test_timer() diff --git a/python/envctl_engine/actions/action_worktree_target_resolution.py b/python/envctl_engine/actions/action_worktree_target_resolution.py index 543309386..9345c292f 100644 --- a/python/envctl_engine/actions/action_worktree_target_resolution.py +++ b/python/envctl_engine/actions/action_worktree_target_resolution.py @@ -33,9 +33,16 @@ def resolve(self) -> object | None: return None matches = self._matching_candidates(cwd, repo_root=repo_root, trees_dir_name=trees_dir_name) - if len(matches) != 1: + if len(matches) == 1: + return matches[0] + if matches: return None - return matches[0] + return self._linked_worktree_candidate( + cwd, + repo_root=repo_root, + configured_root_path=configured_root_path, + raw_runtime=raw_runtime, + ) def _current_working_directory(self) -> Path: invocation_cwd = str(getattr(self.runtime, "env", {}).get("ENVCTL_INVOCATION_CWD") or "").strip() @@ -57,6 +64,9 @@ def _repo_root(self, cwd: Path, *, configured_root_path: Path | None, trees_dir_ repo_root = repo_root_from_worktree_layout(cwd, trees_dir_name) if repo_root is None: repo_root = self.main_repo_root_for_linked_worktree(cwd) + if repo_root is None: + resolver = self.git_main_repo_root_for_worktree or main_repo_root_for_worktree + repo_root = resolver(worktree_root=cwd, runtime=self.runtime, trees_dir_name=trees_dir_name) if repo_root is None or repo_root.resolve() != configured_root_path: return None return repo_root @@ -84,6 +94,40 @@ def _matching_candidates(self, cwd: Path, *, repo_root: Path, trees_dir_name: st if _path_is_at_or_under(cwd, Path(str(getattr(candidate, "root"))).resolve()) ] + @staticmethod + def _linked_worktree_candidate( + cwd: Path, + *, + repo_root: Path, + configured_root_path: Path | None, + raw_runtime: Any, + ) -> object | None: + """Represent a linked checkout even when it is outside the configured trees directory. + + Source-checkout workflows commonly create sibling worktrees. Git proves + that such a checkout belongs to the configured repository, while the + top-level and configured-root comparisons prevent a tree-mode action + from silently targeting the main checkout. + """ + + top_level_raw = _git_output(raw_runtime, cwd, ["rev-parse", "--show-toplevel"]) + if not top_level_raw: + return None + try: + top_level = Path(top_level_raw).expanduser().resolve() + except OSError: + return None + if not _path_is_at_or_under(cwd, top_level): + return None + normalized_repo_root = repo_root.resolve() + if top_level == normalized_repo_root: + return None + if configured_root_path is not None and top_level == configured_root_path: + return None + + branch = _git_output(raw_runtime, top_level, ["branch", "--show-current"]) + return SimpleNamespace(name=branch or top_level.name, root=top_level) + def resolve_current_worktree_target( *, @@ -140,9 +184,7 @@ def main_repo_root_for_worktree(*, worktree_root: Path, runtime: Any, trees_dir_ return top_level common_dir_path = Path(common_dir_raw) common_dir = ( - (worktree_root / common_dir_path).resolve() - if not common_dir_path.is_absolute() - else common_dir_path.resolve() + (worktree_root / common_dir_path).resolve() if not common_dir_path.is_absolute() else common_dir_path.resolve() ) if common_dir.name == ".git": return common_dir.parent @@ -179,3 +221,18 @@ def repo_root_from_worktree_layout(worktree_root: Path, trees_dir_name: str) -> def _path_is_at_or_under(path: Path, root: Path) -> bool: return path == root or root in path.parents + + +def _git_output(runtime: object, cwd: Path, args: list[str]) -> str: + try: + process_runtime = resolve_process_runtime(runtime) + completed = process_runtime.run( + ["git", *args], + cwd=cwd, + timeout=10.0, + ) + except Exception: # noqa: BLE001 - target inference must fail closed + return "" + if getattr(completed, "returncode", 1) != 0: + return "" + return str(getattr(completed, "stdout", "") or "").strip() diff --git a/python/envctl_engine/actions/project_action_domain.py b/python/envctl_engine/actions/project_action_domain.py index 316f75a1f..0dea27444 100644 --- a/python/envctl_engine/actions/project_action_domain.py +++ b/python/envctl_engine/actions/project_action_domain.py @@ -10,6 +10,7 @@ import envctl_engine.actions.action_commit_support as commit_support import envctl_engine.actions.action_pr_label_support as pr_label_support import envctl_engine.actions.action_git_state_support as git_state_support +import envctl_engine.actions.action_pr_git_support as pr_git_support import envctl_engine.actions.action_pr_message_support as pr_message_support import envctl_engine.actions.action_review_artifact_support as review_artifact_support import envctl_engine.actions.action_review_base_support as review_base_support @@ -46,6 +47,7 @@ __all__ = [ "ActionProjectContext", "DirtyWorktreeReport", + "ExistingPullRequest", "EnvctlProtectedPathPartition", "OriginalPlanResolution", "ReviewBaseResolution", @@ -70,6 +72,7 @@ "_unmerged_stage_entries", "_write_pr_body_file", "detect_default_branch", + "existing_pull_request", "existing_pr_url", "probe_dirty_worktree", "resolve_git_root", @@ -78,6 +81,7 @@ "run_pr_action", "run_review_action", "run_ship_action", + "update_pull_request_base", ] @@ -97,6 +101,7 @@ def interactive(self) -> bool: ReviewBaseResolutionError = review_base_support.ReviewBaseResolutionError OriginalPlanResolution = review_original_plan_support.OriginalPlanResolution + def _workflow_runner() -> ProjectActionWorkflowRunner: return workflow_factory.ProjectActionWorkflowFactory( git=workflow_factory.ProjectActionWorkflowGitSources( @@ -115,6 +120,8 @@ def _workflow_runner() -> ProjectActionWorkflowRunner: pull_request=workflow_factory.ProjectActionWorkflowPullRequestSources( resolve_base_branch_fn=_resolve_pr_base_branch, existing_pr_url_fn=existing_pr_url, + existing_pull_request_fn=existing_pull_request, + update_pull_request_base_fn=update_pull_request_base, probe_dirty_worktree_source_fn=probe_dirty_worktree, run_commit_action_fn=run_commit_action, pr_title_fn=_pr_title, @@ -279,8 +286,12 @@ def run_review_action(context: ActionProjectContext) -> int: DirtyWorktreeReport = git_state_support.DirtyWorktreeReport +ExistingPullRequest = git_state_support.ExistingPullRequest resolve_git_root = git_state_support.resolve_git_root _classify_dirty_porcelain = git_state_support.classify_dirty_porcelain +existing_pr_url = pr_git_support.existing_pr_url +existing_pull_request = pr_git_support.existing_pull_request +update_pull_request_base = pr_git_support.update_pull_request_base def probe_dirty_worktree(project_root: Path, repo_root: Path, *, project_name: str = "") -> DirtyWorktreeReport: @@ -296,15 +307,6 @@ def detect_default_branch(git_root: Path) -> str: return git_state_support.detect_default_branch(git_root, git_output=_git_output) -def existing_pr_url(git_root: Path, branch: str) -> str: - return git_state_support.existing_pr_url( - git_root, - branch, - gh_path=shutil.which("gh"), - run_process=subprocess.run, - ) - - _resolve_original_plan = review_original_plan_support.resolve_original_plan _read_worktree_provenance = review_original_plan_support.read_worktree_provenance _resolve_plan_file_from_record = review_original_plan_support.resolve_plan_file_from_record @@ -335,7 +337,7 @@ def _resolve_pr_base_branch(context: ActionProjectContext, git_root: Path) -> st explicit = str(context.env.get("ENVCTL_PR_BASE", "")).strip() if explicit: return explicit - return detect_default_branch(git_root) + return git_state_support.detect_pr_base_branch(git_root, git_output=_git_output) _resolve_analyze_mode = review_iteration_support.resolve_analyze_mode diff --git a/python/envctl_engine/actions/project_action_workflow_factory.py b/python/envctl_engine/actions/project_action_workflow_factory.py index 0b5d2fe4e..7258c7226 100644 --- a/python/envctl_engine/actions/project_action_workflow_factory.py +++ b/python/envctl_engine/actions/project_action_workflow_factory.py @@ -7,7 +7,7 @@ from typing import Any import envctl_engine.actions.action_ship_support as ship_support -from envctl_engine.actions.action_git_state_support import DirtyWorktreeReport +from envctl_engine.actions.action_git_state_support import DirtyWorktreeReport, ExistingPullRequest from envctl_engine.actions.action_review_context import ReviewActionContext from envctl_engine.actions.action_review_base_support import ReviewBaseResolution from envctl_engine.actions.action_review_original_plan_support import OriginalPlanResolution @@ -41,6 +41,8 @@ class ProjectActionWorkflowCommitSources: class ProjectActionWorkflowPullRequestSources: resolve_base_branch_fn: Callable[[Any, Path], str] existing_pr_url_fn: Callable[[Path, str], str] + existing_pull_request_fn: Callable[[Path, str], ExistingPullRequest | None] + update_pull_request_base_fn: Callable[[Path, ExistingPullRequest, str], subprocess.CompletedProcess[str]] probe_dirty_worktree_source_fn: Callable[..., DirtyWorktreeReport] run_commit_action_fn: Callable[[Any], int] pr_title_fn: Callable[[Any, Path, str], str] @@ -89,6 +91,8 @@ def build(self) -> ProjectActionWorkflowRunner: pull_request=ProjectActionPullRequestWorkflowDependencies( resolve_base_branch_fn=self.pull_request.resolve_base_branch_fn, existing_pr_url_fn=self.pull_request.existing_pr_url_fn, + existing_pull_request_fn=self.pull_request.existing_pull_request_fn, + update_pull_request_base_fn=self.pull_request.update_pull_request_base_fn, probe_dirty_worktree_fn=self.probe_dirty_worktree, run_commit_action_fn=self.pull_request.run_commit_action_fn, pr_title_fn=self.pull_request.pr_title_fn, diff --git a/python/envctl_engine/actions/project_action_workflows.py b/python/envctl_engine/actions/project_action_workflows.py index e516c7d4c..9cf3d43e6 100644 --- a/python/envctl_engine/actions/project_action_workflows.py +++ b/python/envctl_engine/actions/project_action_workflows.py @@ -7,6 +7,7 @@ from typing import Any import envctl_engine.actions.action_commit_support as commit_support +from envctl_engine.actions.action_git_state_support import ExistingPullRequest import envctl_engine.actions.action_pr_workflow_support as pr_workflow_support import envctl_engine.actions.action_review_workflow_support as review_workflow_support import envctl_engine.actions.action_ship_support as ship_support @@ -45,6 +46,8 @@ class ProjectActionCommitWorkflowDependencies: class ProjectActionPullRequestWorkflowDependencies: resolve_base_branch_fn: Callable[[Any, Path], str] existing_pr_url_fn: Callable[[Path, str], str] + existing_pull_request_fn: Callable[[Path, str], ExistingPullRequest | None] + update_pull_request_base_fn: Callable[[Path, ExistingPullRequest, str], subprocess.CompletedProcess[str]] probe_dirty_worktree_fn: Callable[[Path, Path, str], DirtyWorktreeReport] run_commit_action_fn: Callable[[Any], int] pr_title_fn: Callable[[Any, Path, str], str] @@ -126,7 +129,8 @@ def run_pr_action(self, context: Any) -> int: git_available=self.git_available, git_output_fn=self.git.git_output_fn, resolve_base_branch_fn=self.pull_request.resolve_base_branch_fn, - existing_pr_url_fn=self.pull_request.existing_pr_url_fn, + existing_pull_request_fn=self.pull_request.existing_pull_request_fn, + update_pull_request_base_fn=self.pull_request.update_pull_request_base_fn, probe_dirty_worktree_fn=self.pull_request.probe_dirty_worktree_fn, run_commit_action_fn=self.pull_request.run_commit_action_fn, pr_title_fn=self.pull_request.pr_title_fn, @@ -151,6 +155,7 @@ def run_ship_action(self, context: Any) -> int: add_ship_pr_label=self.pull_request.add_ship_pr_label_fn, probe_dirty_worktree=self._probe_dirty_worktree_for_ship, existing_pr_url=self.pull_request.existing_pr_url_fn, + existing_pull_request=self.pull_request.existing_pull_request_fn, partition_envctl_protected_paths=self.commit.partition_envctl_protected_paths_fn, ordered_unique_paths=self.commit.ordered_unique_paths_fn, github_pr_checks=self.pull_request.github_pr_checks_fn, diff --git a/python/envctl_engine/config/__init__.py b/python/envctl_engine/config/__init__.py index 00399fbe7..5f8fb637d 100644 --- a/python/envctl_engine/config/__init__.py +++ b/python/envctl_engine/config/__init__.py @@ -220,9 +220,6 @@ def load_config(env: Mapping[str, str] | None = None) -> EngineConfig: runtime_scope_id = _runtime_scope_id(base_dir=base_dir, env=env, resolved=resolved) runtime_scope_dir = runtime_dir / "python-engine" / runtime_scope_id - runtime_dir.mkdir(parents=True, exist_ok=True) - runtime_scope_dir.mkdir(parents=True, exist_ok=True) - dependency_ports: dict[str, dict[str, int]] = {} for definition in _dependency_definitions(): resources: dict[str, int] = {} @@ -283,9 +280,11 @@ def load_config(env: Mapping[str, str] | None = None) -> EngineConfig: plan_agent_cmux_workspace = str(resolved.get("ENVCTL_PLAN_AGENT_CMUX_WORKSPACE", "") or "").strip() plan_agent_superset_project = str(resolved.get("ENVCTL_PLAN_AGENT_SUPERSET_PROJECT", "") or "").strip() plan_agent_superset_workspace = str(resolved.get("ENVCTL_PLAN_AGENT_SUPERSET_WORKSPACE", "") or "").strip() - plan_agent_terminals_enable = parse_bool(resolved.get("ENVCTL_PLAN_AGENT_TERMINALS_ENABLE"), False) or bool( - plan_agent_cmux_workspace - ) or bool(plan_agent_superset_project or plan_agent_superset_workspace) + plan_agent_terminals_enable = ( + parse_bool(resolved.get("ENVCTL_PLAN_AGENT_TERMINALS_ENABLE"), False) + or bool(plan_agent_cmux_workspace) + or bool(plan_agent_superset_project or plan_agent_superset_workspace) + ) additional_services, additional_service_errors = _parse_additional_services(resolved) supabase_auth_users, supabase_auth_user_errors = _parse_supabase_auth_users(resolved) @@ -358,8 +357,7 @@ def load_config(env: Mapping[str, str] | None = None) -> EngineConfig: plan_agent_cli_cmd=str(resolved.get("ENVCTL_PLAN_AGENT_CLI_CMD", "") or "").strip(), plan_agent_cmux_workspace=plan_agent_cmux_workspace, plan_agent_surface_transport=( - str(resolved.get("ENVCTL_PLAN_AGENT_SURFACE_TRANSPORT", "cmux") or "cmux").strip().lower() - or "cmux" + str(resolved.get("ENVCTL_PLAN_AGENT_SURFACE_TRANSPORT", "cmux") or "cmux").strip().lower() or "cmux" ), plan_agent_superset_project=plan_agent_superset_project, plan_agent_superset_workspace=plan_agent_superset_workspace, diff --git a/python/envctl_engine/debug/debug_utils.py b/python/envctl_engine/debug/debug_utils.py index 83ddcbbd2..7511cee73 100644 --- a/python/envctl_engine/debug/debug_utils.py +++ b/python/envctl_engine/debug/debug_utils.py @@ -84,11 +84,6 @@ def file_lock(path: Path, *, timeout: float) -> Iterator[None]: except OSError: pass handle.close() - try: - if path.exists() and path.stat().st_size == 0: - path.unlink() - except OSError: - pass def debug_env_value(env: dict[str, str], key: str) -> str | None: diff --git a/python/envctl_engine/planning/plan_agent/cmux_transport.py b/python/envctl_engine/planning/plan_agent/cmux_transport.py index d874b819f..dfd4a7f36 100644 --- a/python/envctl_engine/planning/plan_agent/cmux_transport.py +++ b/python/envctl_engine/planning/plan_agent/cmux_transport.py @@ -25,12 +25,14 @@ _print_launch_summary, _queue_failure_event_context, ) -from envctl_engine.planning.plan_agent.workflow import ( +from envctl_engine.planning.plan_agent.workflow_runtime_support import ( _codex_goal_text_for_worktree, _emit_codex_goal_event, + _surface_respawn_command, +) +from envctl_engine.planning.plan_agent.workflow_review_support import ( _review_original_plan_path, _review_prompt_arguments, - _surface_respawn_command, ) from envctl_engine.planning.plan_agent.workflow_build import ( _build_plan_agent_workflow, diff --git a/python/envctl_engine/planning/plan_agent/config.py b/python/envctl_engine/planning/plan_agent/config.py index ced72c8f5..9142b443c 100644 --- a/python/envctl_engine/planning/plan_agent/config.py +++ b/python/envctl_engine/planning/plan_agent/config.py @@ -2,7 +2,7 @@ # ruff: noqa: F401 import shutil -from typing import Any, Literal +from typing import Any from envctl_engine.config import EngineConfig, _apply_plan_agent_aliases @@ -18,7 +18,6 @@ normalize_plan_agent_surface_transport, parse_codex_cycles, uses_direct_submission, - workflow_mode_for_launch_config, ) from envctl_engine.planning.plan_agent.models import PlanAgentLaunchConfig @@ -39,14 +38,6 @@ def _launch_policy( ) -def _parse_codex_cycles(raw: object) -> tuple[int, str | None]: - return parse_codex_cycles(raw) - - -def _workflow_mode_for_launch_config(launch_config: PlanAgentLaunchConfig) -> str: - return workflow_mode_for_launch_config(launch_config) - - def _codex_tui_queue_workflow_supported(launch_config: PlanAgentLaunchConfig) -> bool: return codex_tui_queue_workflow_supported(launch_config) @@ -68,22 +59,6 @@ def resolve_plan_agent_launch_config( return _launch_policy(config, env, route=route).resolve_launch_config() -def _default_plan_agent_cli_command(cli: str, *, codex_yolo_enabled: bool = True) -> str: - return default_plan_agent_cli_command(cli, codex_yolo_enabled=codex_yolo_enabled) - - -def _default_plan_agent_surface_transport() -> Literal["cmux", "tmux"]: - return default_plan_agent_surface_transport(command_available=shutil.which) - - -def _resolve_available_plan_agent_surface_transport(raw: str) -> str: - return normalize_plan_agent_surface_transport(raw) - - -def _cmux_transport_explicitly_requested(config: EngineConfig, env: dict[str, str], route: object | None) -> bool: - return _launch_policy(config, env, route=route).cmux_transport_explicitly_requested() - - def _route_requests_ulw(route: object | None) -> bool: return bool(getattr(route, "flags", {}).get("ulw")) @@ -105,10 +80,6 @@ def plan_agent_launch_prereq_commands( return _launch_policy(config, env, route=route).prereq_commands() -def _command_executable(raw_command: str) -> str | None: - return command_executable(raw_command) - - def _missing_launch_commands(runtime: Any, launch_config: PlanAgentLaunchConfig) -> list[str]: return missing_launch_commands(runtime, launch_config) diff --git a/python/envctl_engine/planning/plan_agent/constants.py b/python/envctl_engine/planning/plan_agent/constants.py index aae7d6396..501084232 100644 --- a/python/envctl_engine/planning/plan_agent/constants.py +++ b/python/envctl_engine/planning/plan_agent/constants.py @@ -66,7 +66,6 @@ "queued follow-up messages", "follow-up inputs", ) -_OPENCODE_READY_PROMPT_RE = re.compile(r"^[ \t]*[>›❯»][ \t]*.*$") _OPENCODE_LOADING_MARKERS = ( "loading", "starting", diff --git a/python/envctl_engine/planning/plan_agent/launch_policy.py b/python/envctl_engine/planning/plan_agent/launch_policy.py index 093a91298..352a4ca40 100644 --- a/python/envctl_engine/planning/plan_agent/launch_policy.py +++ b/python/envctl_engine/planning/plan_agent/launch_policy.py @@ -16,8 +16,6 @@ _DEFAULT_PRESET, _DEFAULT_SHELL, _PLAN_AGENT_CODEX_CYCLE_CAP, - _PLAN_AGENT_WORKFLOW_CODEX_CYCLES, - _PLAN_AGENT_WORKFLOW_SINGLE_PROMPT, _SUPPORTED_PLAN_AGENT_CLIS, ) from envctl_engine.planning.plan_agent.models import PlanAgentLaunchConfig @@ -306,12 +304,6 @@ def parse_codex_cycles(raw: object) -> tuple[int, str | None]: return value, None -def workflow_mode_for_launch_config(launch_config: PlanAgentLaunchConfig) -> str: - if codex_tui_queue_workflow_supported(launch_config) and launch_config.codex_cycles > 0: - return _PLAN_AGENT_WORKFLOW_CODEX_CYCLES - return _PLAN_AGENT_WORKFLOW_SINGLE_PROMPT - - def codex_tui_queue_workflow_supported(launch_config: PlanAgentLaunchConfig) -> bool: return launch_config.cli == "codex" and launch_config.transport in {"cmux", "tmux", "omx"} diff --git a/python/envctl_engine/planning/plan_agent/superset_desktop_support.py b/python/envctl_engine/planning/plan_agent/superset_desktop_support.py index b1303edb7..f73d94c16 100644 --- a/python/envctl_engine/planning/plan_agent/superset_desktop_support.py +++ b/python/envctl_engine/planning/plan_agent/superset_desktop_support.py @@ -9,7 +9,7 @@ from envctl_engine.planning.plan_agent.models import CreatedPlanWorktree, PlanAgentLaunchConfig, PlanAgentLaunchOutcome from envctl_engine.planning.plan_agent.recovery import _print_launch_summary -from envctl_engine.planning.plan_agent.workflow import _tab_title_for_worktree +from envctl_engine.planning.plan_agent.workflow_build import _tab_title_for_worktree from envctl_engine.runtime.runtime_context import resolve_process_runtime diff --git a/python/envctl_engine/planning/plan_agent/superset_transport.py b/python/envctl_engine/planning/plan_agent/superset_transport.py index a82172a47..094b24ac7 100644 --- a/python/envctl_engine/planning/plan_agent/superset_transport.py +++ b/python/envctl_engine/planning/plan_agent/superset_transport.py @@ -31,11 +31,11 @@ write_superset_codex_goal_launcher, ) import envctl_engine.planning.plan_agent.superset_worktree_launch_support as superset_worktree_launch_support -from envctl_engine.planning.plan_agent.workflow import ( +from envctl_engine.planning.plan_agent.workflow_runtime_support import ( _codex_goal_text_for_worktree, _emit_codex_goal_event, - _workflow_step_prompt_text, ) +from envctl_engine.planning.plan_agent.workflow_prompt_support import _workflow_step_prompt_text _ensure_superset_codex_goal_agent = ensure_superset_codex_goal_agent diff --git a/python/envctl_engine/planning/plan_agent/tmux_identity_support.py b/python/envctl_engine/planning/plan_agent/tmux_identity_support.py index ae2a7da99..34a413d6d 100644 --- a/python/envctl_engine/planning/plan_agent/tmux_identity_support.py +++ b/python/envctl_engine/planning/plan_agent/tmux_identity_support.py @@ -9,7 +9,7 @@ ) from envctl_engine.planning.plan_agent.models import CreatedPlanWorktree -from envctl_engine.planning.plan_agent.workflow import _tab_title_for_worktree +from envctl_engine.planning.plan_agent.workflow_build import _tab_title_for_worktree def tmux_session_name_for_worktree(repo_root: Path, worktree: CreatedPlanWorktree, *, cli: str) -> str: diff --git a/python/envctl_engine/planning/plan_agent/tmux_transport.py b/python/envctl_engine/planning/plan_agent/tmux_transport.py index fde1eea4a..ecdfc7cbc 100644 --- a/python/envctl_engine/planning/plan_agent/tmux_transport.py +++ b/python/envctl_engine/planning/plan_agent/tmux_transport.py @@ -44,7 +44,7 @@ _prompt_existing_tmux_session_action, _should_prompt_existing_tmux_session, ) -from envctl_engine.planning.plan_agent.workflow import ( +from envctl_engine.planning.plan_agent.workflow_runtime_support import ( _codex_goal_text_for_worktree, _emit_codex_goal_event, _wrap_omx_initial_prompt_for_workflow, diff --git a/python/envctl_engine/planning/plan_agent/workflow.py b/python/envctl_engine/planning/plan_agent/workflow.py deleted file mode 100644 index d186f2529..000000000 --- a/python/envctl_engine/planning/plan_agent/workflow.py +++ /dev/null @@ -1,234 +0,0 @@ -from __future__ import annotations - -# ruff: noqa: F401 -from typing import Any - -from envctl_engine.planning.plan_agent.constants import ( - _AI_CLI_SHELL_FAILURE_MARKERS, - _ANSI_ESCAPE_RE, - _BROWSER_E2E_FOLLOWUP_TEMPLATE, - _CLI_READY_DELAY_SECONDS_BY_CLI, - _CLI_READY_POLL_INTERVAL_SECONDS, - _CODEX_BYPASS_FLAGS, - _CODEX_LOADING_MARKERS, - _CODEX_QUEUE_CONFIRMED_MARKERS, - _CODEX_QUEUE_MAX_TAB_ATTEMPTS, - _CODEX_QUEUE_READY_HINT, - _CODEX_QUEUE_READY_POLL_INTERVAL_SECONDS, - _CODEX_QUEUE_READY_TIMEOUT_SECONDS, - _CODEX_READY_MARKERS, - _CODEX_READY_PROMPT_RE, - _DEFAULT_CLI_READY_DELAY_SECONDS, - _DEFAULT_PRESET, - _DEFAULT_SHELL, - _DONE_PLANNING_ROOT, - _FINALIZATION_INSTRUCTION_TEMPLATE, - _FIRST_CYCLE_COMPLETION_TEMPLATE, - _INTERMEDIATE_CYCLE_COMPLETION_TEMPLATE, - _LOW_SIGNAL_TAB_TITLE_WORDS, - _OMX_SESSION_READY_POLL_INTERVAL_SECONDS, - _OMX_SESSION_READY_TIMEOUT_SECONDS, - _OMX_SESSION_STATE_RELATIVE_PATH, - _OMX_SPAWN_OUTPUT_EXCERPT_CHARS, - _OMX_TMUX_EXTENDED_KEYS_RELATIVE_PATH, - _OMX_TMUX_LOCK_STALE_SECONDS, - _OMX_WORKFLOW_KEYWORDS, - _OPENCODE_LOADING_MARKERS, - _OPENCODE_READY_MARKERS, - _OPENCODE_READY_PROMPT_RE, - _PLAN_AGENT_CODEX_CYCLE_CAP, - _PLAN_AGENT_TAB_TITLE_MAX_LEN, - _PLAN_AGENT_WORKFLOW_CODEX_CYCLES, - _PLAN_AGENT_WORKFLOW_SINGLE_PROMPT, - _PLANNING_ROOT, - _PR_REVIEW_COMMENTS_FOLLOWUP_TEMPLATE, - _PROMPT_PRE_SUBMIT_DELAY_SECONDS, - _PROMPT_SHAPING_COMMAND_TOKEN_RE, - _PROMPT_SUBMIT_READY_DELAY_SECONDS, - _PROMPT_SUBMIT_READY_POLL_INTERVAL_SECONDS, - _PROMPT_SUBMIT_READY_TIMEOUT_SECONDS, - _PROMPT_TEMPLATE_PACKAGE, - _READ_SCREEN_LINE_COUNT, - _REVIEW_WORKTREE_PRESET, - _SUPPORTED_PLAN_AGENT_CLIS, - _SURFACE_READY_DELAY_SECONDS, - _TMUX_WINDOW_READY_POLL_INTERVAL_SECONDS, - _TMUX_WINDOW_READY_TIMEOUT_SECONDS, - _WORKTREE_PROVENANCE_PATH, -) -from envctl_engine.planning.plan_agent.config import ( - PlanAgentLaunchPolicy, - _cli_ready_delay_seconds, - _cmux_transport_explicitly_requested, - _codex_tui_queue_workflow_supported, - _command_executable, - _default_plan_agent_cli_command, - _default_plan_agent_surface_transport, - _guidance_attach_command, - _launch_policy, - _missing_launch_commands, - _parse_codex_cycles, - _resolve_available_plan_agent_surface_transport, - _route_requests_ulw, - _ulw_route_supported, - _uses_direct_submission, - _workflow_mode_for_launch_config, - cli_ready_delay_seconds, - codex_tui_queue_workflow_supported, - command_executable, - default_plan_agent_cli_command, - default_plan_agent_surface_transport, - guidance_attach_command, - missing_launch_commands, - normalize_plan_agent_surface_transport, - parse_codex_cycles, - plan_agent_launch_prereq_commands, - resolve_plan_agent_launch_config, - uses_direct_submission, - workflow_mode_for_launch_config, -) -from envctl_engine.planning.plan_agent.models import ( - AgentTerminalLaunchResult, - AiCliReadyResult, - CreatedPlanWorktree, - PlanAgentAttachTarget, - PlanAgentAttachValidation, - PlanAgentLaunchConfig, - PlanAgentLaunchOutcome, - PlanAgentLaunchResult, - PlanSelectionResult, - PlanWorktreeSyncResult, - ReviewAgentLaunchReadiness, - _OmxSessionRecord, - _OmxSpawnProcessRecord, - _PlanAgentWorkflow, - _PlanAgentWorkflowStep, - _QueueFailure, - _WorkspaceLaunchTarget, -) -from envctl_engine.planning.plan_agent.terminal_screen import ( - _CODEX_GOAL_ACTIVE_MARKERS, - _codex_goal_screen_looks_active, - _codex_queue_message_needs_tab, - _codex_queue_screen_confirms_queued, - _codex_queue_screen_looks_ready, - _codex_queue_text_is_visible, - _format_ai_cli_ready_failure, - _normalized_screen_text, - _post_submit_screen_looks_accepted, - _prompt_picker_screen_looks_ready, - _prompt_submit_screen_looks_ready, - _screen_excerpt, - _screen_looks_active, - _screen_looks_ready, - _screen_tail_text, - _strip_ansi_sequences, -) -from envctl_engine.planning.plan_agent.workflow_build import ( - PlanAgentWorkflowBuilder, - _browser_e2e_instruction_text, - _build_plan_agent_workflow, - _finalization_instruction_text, - _first_cycle_completion_instruction_text, - _intermediate_cycle_completion_instruction_text, - _load_plan_agent_followup_prompt, - _pr_review_comments_instruction_text, - _slash_command, - _tab_title_for_worktree, -) -from envctl_engine.planning.plan_agent.workflow_prompt_support import ( - _append_runtime_addresses_for_preset, - _component_port, - _dependency_address, - _dependency_address_lines, - _dependency_label, - _int_or_none, - _latest_runtime_state, - _original_plan_file_path, - _original_task_source_prompt_section, - _resolve_preset_submission_text, - _runtime_addresses_prompt_section, - _service_address_lines, - _shape_prompt_text, - _shape_queue_message_text, - _state_project_matches_worktree, - _state_service_matches_worktree, - _workflow_step_prompt_text, - optional_state_repository, - resolve_codex_direct_prompt_body, - resolve_opencode_direct_prompt_body, -) -from envctl_engine.planning.plan_agent.workflow_review_support import ( - _active_plan_selector_for_path, - _cli_display_path, - _feature_name_from_project_name, - _infer_plan_file_from_feature, - _plan_matches_for_feature, - _recorded_plan_file_from_worktree, - _resolve_recorded_plan_file, - _review_original_plan_path, - _review_prompt_arguments, - resolve_plan_agent_launch_command, -) -import envctl_engine.runtime.runtime_context as runtime_context - - -def _surface_respawn_command(launch_config: PlanAgentLaunchConfig, worktree: CreatedPlanWorktree) -> str: - _ = worktree - return launch_config.shell - - -def _wrap_omx_initial_prompt_for_workflow(text: str, *, workflow: str) -> str: - normalized_workflow = str(workflow or "").strip().lower() - if normalized_workflow not in _OMX_WORKFLOW_KEYWORDS: - return text - stripped = str(text).lstrip() - prefix = f"${normalized_workflow}" - if stripped == prefix or stripped.startswith(f"{prefix} ") or stripped.startswith(f"{prefix}\n"): - return text - return f"{prefix}\n\n{text}" - - -def _codex_goal_text_for_worktree( - *, - worktree: CreatedPlanWorktree, - preset: str, - workflow_mode: str, - omx_workflow: str, -) -> str: - _ = (worktree, preset, workflow_mode) - lines = [ - "Implement MAIN_TASK.md end-to-end in this worktree.", - "Read it first, update code/tests, run focused validation, then ship with `envctl ship -m \"\"`.", - ] - normalized_omx = str(omx_workflow or "").strip().lower() - if normalized_omx: - lines.append(f"Keep ${normalized_omx} completion contract active.") - return " ".join(lines) - - -def _emit_codex_goal_event( - runtime: Any, - event: str, - *, - cli: str, - workflow: _PlanAgentWorkflow, - transport: str, - worktree: CreatedPlanWorktree, - reason: str | None = None, - **target: object, -) -> None: - payload: dict[str, object] = { - **target, - "worktree": worktree.name, - "cli": cli, - "workflow_mode": workflow.mode, - "codex_cycles": workflow.codex_cycles, - "transport": transport, - } - if reason is not None: - payload["reason"] = reason - runtime._emit(event, **payload) - - -__all__ = tuple(name for name in globals() if not name.startswith("__")) diff --git a/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py b/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py index 70343d00e..396ea959d 100644 --- a/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py +++ b/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py @@ -7,6 +7,7 @@ from envctl_engine.planning.plan_agent.models import CreatedPlanWorktree from envctl_engine.runtime.runtime_context import optional_state_repository +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RunState @@ -16,7 +17,7 @@ class RuntimeAddressPromptBuilder: worktree: CreatedPlanWorktree | None = None def prompt_section(self) -> str: - state = _latest_runtime_state(self.runtime) + state = _latest_runtime_state(self.runtime, worktree=self.worktree) if state is None: return "" lines = [ @@ -56,31 +57,49 @@ def _runtime_addresses_prompt_section(runtime: Any, *, worktree: CreatedPlanWork return RuntimeAddressPromptBuilder(runtime, worktree=worktree).prompt_section() -def _latest_runtime_state(runtime: Any) -> RunState | None: +def _latest_runtime_state(runtime: Any, *, worktree: CreatedPlanWorktree | None = None) -> RunState | None: + project_names = [worktree.name] if worktree is not None and str(worktree.name).strip() else None state_repository = optional_state_repository(runtime) if state_repository is not None and hasattr(state_repository, "load_latest"): - try: - state = state_repository.load_latest() - except Exception: - state = None - if isinstance(state, RunState): - return state - try_loader = getattr(runtime, "_try_load_existing_state", None) - if callable(try_loader): - for mode in ("trees", "main"): + for selected_projects in _state_lookup_project_attempts(project_names): try: - state = try_loader(mode=mode, strict_mode_match=True) + state = call_state_loader( + state_repository.load_latest, + project_names=selected_projects, + ) except Exception: state = None if isinstance(state, RunState): return state + try_loader = getattr(runtime, "_try_load_existing_state", None) + if callable(try_loader): + for mode in ("trees", "main"): + for selected_projects in _state_lookup_project_attempts(project_names): + try: + state = call_state_loader( + try_loader, + mode=mode, + strict_mode_match=True, + project_names=selected_projects, + ) + except Exception: + state = None + if isinstance(state, RunState): + return state return None +def _state_lookup_project_attempts(project_names: list[str] | None) -> tuple[list[str] | None, ...]: + if project_names is None: + return (None,) + return (project_names, None) + + def _dependency_address_lines(state: RunState, *, worktree: CreatedPlanWorktree | None = None) -> list[str]: rows: list[str] = [] seen: set[tuple[str, int]] = set() - for project_name, requirements in state.requirements.items(): + for storage_key, requirements in state.requirements.items(): + project_name = str(getattr(requirements, "project", "") or storage_key).strip() if not _state_project_matches_worktree(project_name, worktree): continue for dependency_id in ("postgres", "redis", "supabase", "n8n"): @@ -128,6 +147,9 @@ def _state_project_matches_worktree(project_name: object, worktree: CreatedPlanW def _state_service_matches_worktree(service: object, worktree: CreatedPlanWorktree | None) -> bool: if worktree is None: return True + project = str(getattr(service, "project", "") or "").strip() + if project and _state_project_matches_worktree(project, worktree): + return True service_name = str(getattr(service, "name", "") or "").strip() worktree_name = str(worktree.name or "").strip() if worktree_name and service_name.casefold().startswith(f"{worktree_name.casefold()} "): diff --git a/python/envctl_engine/planning/plan_agent/workflow_runtime_support.py b/python/envctl_engine/planning/plan_agent/workflow_runtime_support.py new file mode 100644 index 000000000..3962ee7f2 --- /dev/null +++ b/python/envctl_engine/planning/plan_agent/workflow_runtime_support.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from typing import Any + +from envctl_engine.planning.plan_agent.constants import _OMX_WORKFLOW_KEYWORDS +from envctl_engine.planning.plan_agent.models import ( + CreatedPlanWorktree, + PlanAgentLaunchConfig, + _PlanAgentWorkflow, +) + + +def _surface_respawn_command(launch_config: PlanAgentLaunchConfig, worktree: CreatedPlanWorktree) -> str: + _ = worktree + return launch_config.shell + + +def _wrap_omx_initial_prompt_for_workflow(text: str, *, workflow: str) -> str: + normalized_workflow = str(workflow or "").strip().lower() + if normalized_workflow not in _OMX_WORKFLOW_KEYWORDS: + return text + stripped = str(text).lstrip() + prefix = f"${normalized_workflow}" + if stripped == prefix or stripped.startswith(f"{prefix} ") or stripped.startswith(f"{prefix}\n"): + return text + return f"{prefix}\n\n{text}" + + +def _codex_goal_text_for_worktree( + *, + worktree: CreatedPlanWorktree, + preset: str, + workflow_mode: str, + omx_workflow: str, +) -> str: + _ = (worktree, preset, workflow_mode) + lines = [ + "Implement MAIN_TASK.md end-to-end in this worktree.", + "Read it first, update code/tests, run focused validation, then ship with `envctl ship -m \"\"`.", + ] + normalized_omx = str(omx_workflow or "").strip().lower() + if normalized_omx: + lines.append(f"Keep ${normalized_omx} completion contract active.") + return " ".join(lines) + + +def _emit_codex_goal_event( + runtime: Any, + event: str, + *, + cli: str, + workflow: _PlanAgentWorkflow, + transport: str, + worktree: CreatedPlanWorktree, + reason: str | None = None, + **target: object, +) -> None: + payload: dict[str, object] = { + **target, + "worktree": worktree.name, + "cli": cli, + "workflow_mode": workflow.mode, + "codex_cycles": workflow.codex_cycles, + "transport": transport, + } + if reason is not None: + payload["reason"] = reason + runtime._emit(event, **payload) diff --git a/python/envctl_engine/planning/plan_agent_launch_support.py b/python/envctl_engine/planning/plan_agent_launch_support.py index d6e36cedf..1bf85de21 100644 --- a/python/envctl_engine/planning/plan_agent_launch_support.py +++ b/python/envctl_engine/planning/plan_agent_launch_support.py @@ -28,7 +28,7 @@ from envctl_engine.planning.plan_agent.omx_transport import validate_plan_agent_attach_target from envctl_engine.planning.plan_agent.recovery import plan_agent_native_recovery_command from envctl_engine.planning.plan_agent.tmux_transport import attach_plan_agent_terminal -from envctl_engine.planning.plan_agent.workflow import resolve_plan_agent_launch_command +from envctl_engine.planning.plan_agent.workflow_review_support import resolve_plan_agent_launch_command __all__ = ( "AgentTerminalLaunchResult", diff --git a/python/envctl_engine/planning/worktree_domain.py b/python/envctl_engine/planning/worktree_domain.py index 93a746d87..9c61281b1 100644 --- a/python/envctl_engine/planning/worktree_domain.py +++ b/python/envctl_engine/planning/worktree_domain.py @@ -44,12 +44,6 @@ feature_project_candidates as _feature_project_candidates_impl, project_sort_key_for_feature as _project_sort_key_for_feature_impl, ) -from envctl_engine.planning.worktree_selection_memory import ( - initial_plan_selected_counts as _initial_plan_selected_counts_impl, - load_plan_selection_memory as _load_plan_selection_memory_impl, - plan_selection_memory_path as _plan_selection_memory_path_impl, - save_plan_selection_memory as _save_plan_selection_memory_impl, -) from envctl_engine.planning.worktree_shared_artifacts import ( link_repo_local_shared_artifacts as _link_repo_local_shared_artifacts_impl, ) @@ -303,13 +297,10 @@ def _prompt_planning_selection( self: Any, planning_files: list[str], raw_projects: list[tuple[str, Path]], - *, - persist_memory: bool = True, ) -> dict[str, int] | None: return _planning_runtime_bridge(self).prompt_planning_selection( planning_files=planning_files, raw_projects=raw_projects, - persist_memory=persist_memory, ) @@ -330,20 +321,6 @@ def _adjust_plan_counts_for_fresh_ai_launch( ) -def _initial_plan_selected_counts( - self: Any, - *, - planning_files: list[str], - existing_counts: dict[str, int], -) -> dict[str, int]: - remembered = self._load_plan_selection_memory() - return _initial_plan_selected_counts_impl( - planning_files=planning_files, - existing_counts=existing_counts, - remembered_counts=remembered, - ) - - def _run_planning_selection_menu( self: Any, *, @@ -430,10 +407,6 @@ def _resolve_planning_selection_target( ) -def _plan_selection_memory_path(self: Any) -> Path: - return _plan_selection_memory_path_impl(runtime_root=self.runtime_root) - - def _planning_root(self: Any) -> Path: return _planning_root_impl(planning_dir=self.config.planning_dir) @@ -442,21 +415,6 @@ def _planning_done_root(self: Any) -> Path: return _planning_done_root_impl(planning_dir=self.config.planning_dir) -def _load_plan_selection_memory(self: Any) -> dict[str, int]: - return _load_plan_selection_memory_impl( - runtime_root=self.runtime_root, - runtime_legacy_root=self.runtime_legacy_root, - ) - - -def _save_plan_selection_memory(self: Any, selected_counts: dict[str, int]) -> None: - _save_plan_selection_memory_impl( - runtime_root=self.runtime_root, - runtime_legacy_root=self.runtime_legacy_root, - selected_counts=selected_counts, - ) - - def _planning_keep_plan_enabled(self: Any, route: Route) -> bool: return _planning_keep_plan_enabled_impl(route=route, env=self.env, config_raw=self.config.raw) diff --git a/python/envctl_engine/planning/worktree_orchestrator.py b/python/envctl_engine/planning/worktree_orchestrator.py index 4e3d69869..faabe8c22 100644 --- a/python/envctl_engine/planning/worktree_orchestrator.py +++ b/python/envctl_engine/planning/worktree_orchestrator.py @@ -7,12 +7,9 @@ from envctl_engine.planning.plan_agent.models import PlanSelectionResult, PlanWorktreeSyncResult from envctl_engine.planning.worktree_domain import ( _apply_setup_worktree_selection as domain_apply_setup_worktree_selection, - _initial_plan_selected_counts as domain_initial_plan_selected_counts, - _load_plan_selection_memory as domain_load_plan_selection_memory, _planning_keep_plan_enabled as domain_planning_keep_plan_enabled, _prompt_planning_selection as domain_prompt_planning_selection, _run_planning_selection_menu as domain_run_planning_selection_menu, - _save_plan_selection_memory as domain_save_plan_selection_memory, _select_plan_projects as domain_select_plan_projects, _sync_plan_worktrees_from_plan_counts as domain_sync_plan_worktrees_from_plan_counts, ) @@ -25,9 +22,6 @@ class PlanningWorktreeOrchestrator: _apply_setup_worktree_selection = domain_apply_setup_worktree_selection _select_plan_projects = domain_select_plan_projects _prompt_planning_selection = domain_prompt_planning_selection - _initial_plan_selected_counts = domain_initial_plan_selected_counts - _load_plan_selection_memory = domain_load_plan_selection_memory - _save_plan_selection_memory = domain_save_plan_selection_memory _planning_keep_plan_enabled = domain_planning_keep_plan_enabled _sync_plan_worktrees_from_plan_counts = domain_sync_plan_worktrees_from_plan_counts _import_remote_branch_worktree = domain_import_remote_branch_worktree @@ -66,17 +60,6 @@ def prompt_planning_selection( ) -> dict[str, int] | None: return self._prompt_planning_selection(planning_files, raw_projects) - def initial_plan_selected_counts( - self, - *, - planning_files: list[str], - existing_counts: dict[str, int], - ) -> dict[str, int]: - return self._initial_plan_selected_counts( - planning_files=planning_files, - existing_counts=existing_counts, - ) - def run_planning_selection_menu( self, *, @@ -111,12 +94,6 @@ def _run_planning_selection_menu( existing_counts=existing_counts, ) - def load_plan_selection_memory(self) -> dict[str, int]: - return self._load_plan_selection_memory() - - def save_plan_selection_memory(self, selected_counts: dict[str, int]) -> None: - self._save_plan_selection_memory(selected_counts) - def planning_keep_plan_enabled(self, route: Route) -> bool: return self._planning_keep_plan_enabled(route) diff --git a/python/envctl_engine/planning/worktree_plan_project_selection.py b/python/envctl_engine/planning/worktree_plan_project_selection.py index 8a6ce8ea1..f84f1fdb6 100644 --- a/python/envctl_engine/planning/worktree_plan_project_selection.py +++ b/python/envctl_engine/planning/worktree_plan_project_selection.py @@ -190,7 +190,7 @@ def select_plan_projects( return PlanSelectionResult(raw_projects=raw_projects, selected_contexts=[], error="No TTY available.") dry_run = bool(getattr(route, "flags", {}).get("dry_run")) - plan_counts = prompt_planning_selection(planning_files, raw_projects, persist_memory=not dry_run) + plan_counts = prompt_planning_selection(planning_files, raw_projects) if plan_counts is None: output("Planning selection cancelled.") return PlanSelectionResult( diff --git a/python/envctl_engine/planning/worktree_prompt_selection.py b/python/envctl_engine/planning/worktree_prompt_selection.py index 4de1215c0..83c83b00c 100644 --- a/python/envctl_engine/planning/worktree_prompt_selection.py +++ b/python/envctl_engine/planning/worktree_prompt_selection.py @@ -10,21 +10,15 @@ def prompt_planning_selection( *, planning_files: list[str], raw_projects: list[tuple[str, Path]], - initial_plan_selected_counts: Callable[..., dict[str, int]], run_planning_selection_menu: Callable[..., dict[str, int] | None], - save_plan_selection_memory: Callable[[dict[str, int]], None], - persist_memory: bool = True, ) -> dict[str, int] | None: existing_counts = planning_existing_counts(projects=raw_projects, planning_files=planning_files) - selected_counts = initial_plan_selected_counts( - planning_files=planning_files, - existing_counts=existing_counts, - ) - chosen = run_planning_selection_menu( + selected_counts = { + plan_file: max(int(existing_counts.get(plan_file, 0)), 0) + for plan_file in planning_files + } + return run_planning_selection_menu( planning_files=planning_files, selected_counts=selected_counts, existing_counts=existing_counts, ) - if chosen and persist_memory: - save_plan_selection_memory(chosen) - return chosen diff --git a/python/envctl_engine/planning/worktree_provenance.py b/python/envctl_engine/planning/worktree_provenance.py index aaf59fce2..837aa1420 100644 --- a/python/envctl_engine/planning/worktree_provenance.py +++ b/python/envctl_engine/planning/worktree_provenance.py @@ -8,6 +8,7 @@ from envctl_engine.planning.worktree_code_intelligence_files import ensure_worktree_git_excludes from envctl_engine.runtime.codex_tmux_support import _tmux_session_exists from envctl_engine.runtime.runtime_context import resolve_process_runtime +from envctl_engine.shared.git_branch_support import detect_preferred_default_branch from envctl_engine.shared.parsing import parse_bool @@ -165,13 +166,10 @@ def resolve_branch_ref(self: Any, *, source_branch: str) -> str: def detect_default_branch(self: Any) -> str: - ref = git_command_output(self, ["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]).strip() - if ref.startswith("origin/"): - return ref.split("origin/", 1)[1] - for candidate in ("main", "master"): - if git_command_output(self, ["rev-parse", "--verify", candidate]).strip(): - return candidate - return "main" + return detect_preferred_default_branch( + lambda args: git_command_output(self, list(args)), + probe_remote_heads=True, + ) def git_command_output(self: Any, args: list[str]) -> str: @@ -179,11 +177,16 @@ def git_command_output(self: Any, args: list[str]) -> str: def git_command_output_at(self: Any, cwd: Path, args: list[str]) -> str: + command_env = dict(self._command_env(port=0)) + timeout = 30.0 + if args[:1] == ["ls-remote"]: + command_env["GIT_TERMINAL_PROMPT"] = "0" + timeout = 10.0 result = resolve_process_runtime(self).run( ["git", "-C", str(cwd), *args], cwd=cwd, - env=self._command_env(port=0), - timeout=30.0, + env=command_env, + timeout=timeout, ) if getattr(result, "returncode", 1) != 0: return "" diff --git a/python/envctl_engine/planning/worktree_runtime_bridge.py b/python/envctl_engine/planning/worktree_runtime_bridge.py index 621ad5407..28036805b 100644 --- a/python/envctl_engine/planning/worktree_runtime_bridge.py +++ b/python/envctl_engine/planning/worktree_runtime_bridge.py @@ -201,13 +201,10 @@ def prompt_planning_selection( self, planning_files: list[str], raw_projects: list[tuple[str, Path]], - *, - persist_memory: bool = True, ) -> dict[str, int] | None: return self.selection_bridge().prompt_planning_selection( planning_files=planning_files, raw_projects=raw_projects, - persist_memory=persist_memory, ) def run_planning_selection_menu( diff --git a/python/envctl_engine/planning/worktree_selection_memory.py b/python/envctl_engine/planning/worktree_selection_memory.py deleted file mode 100644 index 67520bc5f..000000000 --- a/python/envctl_engine/planning/worktree_selection_memory.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import annotations - -import json -from datetime import UTC, datetime -from pathlib import Path - - -def plan_selection_memory_path(*, runtime_root: Path) -> Path: - return runtime_root / "planning_selection.json" - - -def load_plan_selection_memory(*, runtime_root: Path, runtime_legacy_root: Path) -> dict[str, int]: - path = plan_selection_memory_path(runtime_root=runtime_root) - legacy_path = runtime_legacy_root / "planning_selection.json" - for candidate in (path, legacy_path): - if not candidate.is_file(): - continue - try: - payload = json.loads(candidate.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - continue - if not isinstance(payload, dict): - continue - remembered = payload.get("selected_counts", {}) - if not isinstance(remembered, dict): - continue - result: dict[str, int] = {} - for key, value in remembered.items(): - try: - parsed = int(value) - except (TypeError, ValueError): - continue - if parsed >= 0: - result[str(key)] = parsed - if result: - return result - return {} - - -def save_plan_selection_memory( - *, - runtime_root: Path, - runtime_legacy_root: Path, - selected_counts: dict[str, int], -) -> None: - path = plan_selection_memory_path(runtime_root=runtime_root) - payload = { - "selected_counts": dict(sorted({k: int(v) for k, v in selected_counts.items() if int(v) > 0}.items())), - "saved_at": datetime.now(UTC).isoformat(), - } - text = json.dumps(payload, indent=2, sort_keys=True) - path.write_text(text, encoding="utf-8") - runtime_legacy_root.mkdir(parents=True, exist_ok=True) - (runtime_legacy_root / "planning_selection.json").write_text(text, encoding="utf-8") - - -def initial_plan_selected_counts( - *, - planning_files: list[str], - existing_counts: dict[str, int], - remembered_counts: dict[str, int], -) -> dict[str, int]: - selected_counts: dict[str, int] = {} - for plan_file in planning_files: - existing = int(existing_counts.get(plan_file, 0)) - selected_counts[plan_file] = max(existing, 0) - return selected_counts diff --git a/python/envctl_engine/planning/worktree_selection_runtime_bridge.py b/python/envctl_engine/planning/worktree_selection_runtime_bridge.py index a67435c7a..05b19e7d5 100644 --- a/python/envctl_engine/planning/worktree_selection_runtime_bridge.py +++ b/python/envctl_engine/planning/worktree_selection_runtime_bridge.py @@ -45,17 +45,12 @@ def prompt_planning_selection( self, planning_files: list[str], raw_projects: list[tuple[str, Path]], - *, - persist_memory: bool = True, ) -> dict[str, int] | None: runtime = self.runtime return prompt_planning_selection( planning_files=planning_files, raw_projects=raw_projects, - initial_plan_selected_counts=runtime._initial_plan_selected_counts, run_planning_selection_menu=runtime._run_planning_selection_menu, - save_plan_selection_memory=runtime._save_plan_selection_memory, - persist_memory=persist_memory, ) def run_planning_selection_menu( diff --git a/python/envctl_engine/planning/worktree_setup_coordinator.py b/python/envctl_engine/planning/worktree_setup_coordinator.py index 21bc724fb..92d2292f2 100644 --- a/python/envctl_engine/planning/worktree_setup_coordinator.py +++ b/python/envctl_engine/planning/worktree_setup_coordinator.py @@ -28,9 +28,6 @@ def apply_setup_worktree_selection( ) -> list[ProjectContextLike]: if not setup_worktree_requested(route): return project_contexts - if bool(route.flags.get("docker")): - raise RuntimeError("setup-worktrees is not supported in Docker mode.") - multi_entries = coerce_setup_entries( route=route, flag_name="setup_worktrees", diff --git a/python/envctl_engine/pr_preview_controller.py b/python/envctl_engine/pr_preview_controller.py index 95df9c72e..3cee69855 100644 --- a/python/envctl_engine/pr_preview_controller.py +++ b/python/envctl_engine/pr_preview_controller.py @@ -6,6 +6,7 @@ import argparse from dataclasses import asdict, dataclass, field, fields, replace from datetime import UTC, datetime, timedelta +import hashlib import json import os from pathlib import Path @@ -99,6 +100,7 @@ PUBLIC_ROUTE_HEALTH_DEADLINE_SECONDS = 30.0 PUBLIC_ROUTE_HEALTH_INTERVAL_SECONDS = 2.0 PUBLIC_ROUTE_REQUEST_TIMEOUT_SECONDS = 5.0 +PUBLIC_ROUTE_RESPONSE_BODY_LIMIT_BYTES = 64 * 1024 PLAN_AGENT_BOOLEAN_CONFIG_KEYS = { "ENVCTL_PLAN_AGENT_TERMINALS_ENABLE", "CMUX", @@ -652,6 +654,8 @@ def probe_public_route_url(url: str) -> str | None: timeout=PUBLIC_ROUTE_REQUEST_TIMEOUT_SECONDS, ) as response: status = int(response.status) + read = getattr(response, "read", None) + body = read(PUBLIC_ROUTE_RESPONSE_BODY_LIMIT_BYTES) if callable(read) else b"" except urllib.error.HTTPError as exc: return f"{url} returned HTTP {exc.code}" except urllib.error.URLError as exc: @@ -661,10 +665,35 @@ def probe_public_route_url(url: str) -> str | None: except OSError as exc: return f"{url} failed: {exc}" if 200 <= status < 400: + readiness_failure = semantic_readiness_failure(url, body) + if readiness_failure: + return readiness_failure return None return f"{url} returned HTTP {status}" +def semantic_readiness_failure(url: str, body: object) -> str | None: + if not isinstance(body, (bytes, bytearray)) or not body: + return None + try: + payload = json.loads(bytes(body).decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return None + if not isinstance(payload, dict) or "ready" not in payload: + return None + if payload["ready"] is True: + return None + + reason = payload.get("reason") + safe_reason = "" + if isinstance(reason, str): + safe_reason = re.sub(r"[^A-Za-z0-9_.:-]+", "_", reason).strip("_")[:80] + suffix = f" (reason={safe_reason})" if safe_reason else "" + if payload["ready"] is False: + return f"{url} reported ready=false{suffix}" + return f"{url} returned a non-boolean readiness value{suffix}" + + def public_route_health(endpoints: dict[str, Any]) -> PublicRouteHealth: routed_services = tuple( name @@ -829,6 +858,91 @@ def stable_system_python_bin() -> str | None: return shutil.which("python3") +def prepare_python_command_shim( + env: dict[str, str], + *, + runtime_root: Path, +) -> Path: + """Expose the selected Python 3 interpreter under the legacy ``python`` name. + + Imported PR branches can contain launch scripts written before envctl + propagated ``PYTHON_BIN``. Self-hosted runners commonly provide only + ``python3``. A controller-owned, private PATH shim lets those scripts use + the selected interpreter without modifying the imported worktree. + """ + + configured = str(env.get("PYTHON_BIN") or stable_system_python_bin() or "").strip() + if not configured: + raise RuntimeError("No Python 3 interpreter is available for PR preview startup") + candidate = Path(configured).expanduser() + if not candidate.is_absolute(): + resolved_command = shutil.which(configured, path=env.get("PATH")) + if not resolved_command: + raise RuntimeError(f"Configured preview Python interpreter is not executable: {configured}") + candidate = Path(resolved_command) + try: + python_bin = candidate.resolve(strict=True) + except OSError as exc: + raise RuntimeError(f"Configured preview Python interpreter does not exist: {configured}") from exc + if not python_bin.is_file() or not os.access(python_bin, os.X_OK): + raise RuntimeError(f"Configured preview Python interpreter is not executable: {python_bin}") + + try: + probe = subprocess.run( + [ + str(python_bin), + "-c", + "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ], + check=False, + capture_output=True, + text=True, + timeout=10.0, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise RuntimeError(f"Configured preview interpreter could not be validated: {python_bin}") from exc + reported_version = str(probe.stdout or "").strip() + if probe.returncode != 0 or re.fullmatch(r"3\.\d+", reported_version) is None: + raise RuntimeError(f"Configured preview interpreter is not Python 3: {python_bin}") + + _ensure_real_directory(runtime_root) + shim_root = runtime_root / "python-command-shims" + _ensure_real_directory(shim_root, private=True) + interpreter_key = hashlib.sha256(str(python_bin).encode("utf-8")).hexdigest()[:20] + shim_dir = shim_root / interpreter_key + _ensure_real_directory(shim_dir, private=True) + shim_path = shim_dir / "python" + temporary = shim_dir / f".python-{os.getpid()}-{secrets.token_hex(8)}" + try: + temporary.symlink_to(python_bin) + os.replace(temporary, shim_path) + finally: + temporary.unlink(missing_ok=True) + + env["PYTHON_BIN"] = str(python_bin) + env["ENVCTL_PYTHON_EXECUTABLE"] = str(python_bin) + existing_path = str(env.get("PATH") or "") + legacy_shim_dir = runtime_root / "python-command-shim" + path_parts = [ + part + for part in existing_path.split(os.pathsep) + if part and Path(part) != legacy_shim_dir and Path(part).parent != shim_root + ] + env["PATH"] = os.pathsep.join((str(shim_dir), *path_parts)) + return shim_path + + +def _ensure_real_directory(path: Path, *, private: bool = False) -> None: + try: + path.mkdir(parents=True, exist_ok=False, mode=0o700 if private else 0o755) + except FileExistsError: + pass + if path.is_symlink() or not path.is_dir(): + raise RuntimeError(f"PR preview runtime path is not a real directory: {path}") + if private: + path.chmod(0o700) + + def normalize_envctl_config_value(raw: str) -> str: value = raw.strip() if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: @@ -1623,7 +1737,12 @@ def run_command(self, command: str, pr_number: int | None) -> int: raise SystemExit(f"command {command!r} requires --pr-number") pr = self.get_pr(pr_number) if command == "start": - return self.start(pr, reason="manual workflow dispatch", refresh_ttl=True) + return self.start( + pr, + reason="manual workflow dispatch", + refresh_ttl=True, + force_restart=True, + ) if command == "stop": return self.stop(pr, reason="manual workflow dispatch", comment=True) if command == "delete": @@ -1742,13 +1861,17 @@ def start( *, reason: str, refresh_ttl: bool = False, + force_restart: bool = False, ) -> int: self.validate_same_repo(pr) label_added_at = self.label_active_since(pr.number) or utc_now() if refresh_ttl: label_added_at = utc_now() existing_state = self.load_state(pr.number) - if self.current_head_preview_is_running(pr, existing_state): + if not force_restart and self.current_head_preview_is_running( + pr, + existing_state, + ): assert existing_state is not None updated_state = self.refresh_running_preview_state( existing_state, @@ -1952,7 +2075,16 @@ def run_import() -> CommandResult: start_env = self.envctl_command_env() start_env.update(pr_preview_start_env_overrides()) start_env.update(self.external_dependency_start_env(external_dependencies)) - self.reset_incompatible_backend_venv(project, start_env) + python_setup_error = "" + try: + prepare_python_command_shim( + start_env, + runtime_root=self.config.preview_root / "runtime", + ) + except (OSError, RuntimeError) as exc: + python_setup_error = str(exc) + if not python_setup_error: + self.reset_incompatible_backend_venv(project, start_env) public_urls = self.public_route_urls(pr.number) if public_urls.get("backend"): start_env["ENVCTL_BACKEND_ENV__FRONTEND_BASE_URL"] = public_urls["frontend"] @@ -1968,21 +2100,31 @@ def run_import() -> CommandResult: start_env["ENVCTL_FRONTEND_ENV__VITE_SUPABASE_URL"] = public_urls["supabase"] self.apply_repository_preview_env(start_env, pr_number=pr.number) - start_result = self.runner.run( - [ - self.config.envctl_bin, - "start", - "--trees", - "--project", - str(project["name"]), - "--headless", - "--entire-system", - "--isolated-deps", - "--copy-db-storage", - ], - cwd=self.config.control_repo, - check=False, - env=start_env, + start_argv = [ + self.config.envctl_bin, + "start", + "--trees", + "--project", + str(project["name"]), + "--headless", + "--entire-system", + "--isolated-deps", + "--copy-db-storage", + ] + start_result = ( + CommandResult( + argv=start_argv, + returncode=1, + stdout="", + stderr=f"PR preview Python compatibility setup failed: {python_setup_error}", + ) + if python_setup_error + else self.runner.run( + start_argv, + cwd=self.config.control_repo, + check=False, + env=start_env, + ) ) if start_result.returncode != 0: state = PreviewState( diff --git a/python/envctl_engine/runtime/browser_runtime_diagnostics.py b/python/envctl_engine/runtime/browser_runtime_diagnostics.py index 974778e6a..9ca96a759 100644 --- a/python/envctl_engine/runtime/browser_runtime_diagnostics.py +++ b/python/envctl_engine/runtime/browser_runtime_diagnostics.py @@ -79,15 +79,25 @@ def build_runtime_diagnostics( def project_names(state: RunState, *, runtime: Any | None) -> list[str]: - names: set[str] = set(state.requirements) + names: dict[str, str] = {} + + def add(raw_name: object) -> None: + name = str(raw_name or "").strip() + if name: + names.setdefault(name.casefold(), name) + + for storage_key, requirements in state.requirements.items(): + add(getattr(requirements, "project", "") or storage_key) roots = state.metadata.get("project_roots") if isinstance(roots, dict): - names.update(str(name) for name in roots) + for name in roots: + add(name) for service_name, service in state.services.items(): - names.add(service_project_name(service_name, service, runtime=runtime)) + add(service_project_name(service_name, service, runtime=runtime)) launch = launch_metadata(state) - names.update(launch) - return sorted(name for name in names if name) + for name in launch: + add(name) + return sorted(names.values(), key=lambda name: (name.casefold(), name)) def service_payload(service: ServiceRecord | None) -> dict[str, object]: @@ -159,6 +169,10 @@ def safe_launch_env(raw_launch: object, *, service_name: str) -> dict[str, str]: def requirements_for_project(state: RunState, project: str) -> RequirementsResult | None: if project in state.requirements: return state.requirements[project] + target = str(project).strip().casefold() + for storage_key, requirements in state.requirements.items(): + if str(requirements.project or storage_key).strip().casefold() == target: + return requirements if len(state.requirements) == 1: return next(iter(state.requirements.values())) return None diff --git a/python/envctl_engine/runtime/cli.py b/python/envctl_engine/runtime/cli.py index 9ee0d43af..92831ea00 100644 --- a/python/envctl_engine/runtime/cli.py +++ b/python/envctl_engine/runtime/cli.py @@ -80,6 +80,8 @@ def _effective_prereq_mode(route: Route) -> str: def _requires_docker(mode: str, config: EngineConfig, *, route: Route | None = None) -> bool: + if route is not None and bool(route.flags.get("docker")): + return True if route is not None and route.flags.get("launch_dependencies") is False: return False if effective_dependency_scope(route, mode) == "shared" and mode == "trees": diff --git a/python/envctl_engine/runtime/command_flag_storage.py b/python/envctl_engine/runtime/command_flag_storage.py index ddafa71b6..26463a55b 100644 --- a/python/envctl_engine/runtime/command_flag_storage.py +++ b/python/envctl_engine/runtime/command_flag_storage.py @@ -41,6 +41,7 @@ def boolean_flag_name(token: str) -> str: "--test-parallel": "test_parallel", "--service-parallel": "service_parallel", "--service-prep-parallel": "service_prep_parallel", + "--ignore-service-deps": "ignore_service_deps", "--refresh-cache": "refresh_cache", "--fast": "fast", "--fast-startup": "fast", diff --git a/python/envctl_engine/runtime/command_special_flags.py b/python/envctl_engine/runtime/command_special_flags.py index b232c950a..9056f0e7b 100644 --- a/python/envctl_engine/runtime/command_special_flags.py +++ b/python/envctl_engine/runtime/command_special_flags.py @@ -64,8 +64,6 @@ def handle_special_flag(flags: dict[str, object], token: str) -> None: "--no-requirements-parallel", }: flags["requirements_parallel"] = False - elif token == "--ignore-service-deps": - flags["ignore_service_deps"] = True elif token == "--only-backend": flags["launch_backend"] = True flags["launch_frontend"] = False diff --git a/python/envctl_engine/runtime/docker_service_runtime.py b/python/envctl_engine/runtime/docker_service_runtime.py new file mode 100644 index 000000000..c85f10773 --- /dev/null +++ b/python/envctl_engine/runtime/docker_service_runtime.py @@ -0,0 +1,1179 @@ +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import shlex +import tempfile +import threading +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +from envctl_engine.shared.parsing import parse_bool + + +_HOST_ENV_KEYS = { + "HOME", + "OLDPWD", + "PATH", + "PWD", + "PYTHONHOME", + "PYTHONPATH", + "SHELL", + "SHLVL", + "TMPDIR", + "VIRTUAL_ENV", + "VIRTUAL_ENV_PROMPT", + "ZDOTDIR", + "_", +} +_PUBLIC_ENV_PREFIXES = ("VITE_", "NEXT_PUBLIC_", "PUBLIC_") +_PUBLIC_ENV_KEYS = { + "BACKEND_CORS_ORIGINS", + "CORS_ORIGINS", + "CORS_ORIGINS_RAW", + "ENVCTL_SOURCE_FRONTEND_URL", + "FRONTEND_BASE_URL", + "FRONTEND_URL", + "PUBLIC_URL", +} +_DOCKER_INFO_LOCK = threading.Lock() +_RUNTIME_SCOPE_LABEL = "io.envctl.runtime-scope" +_LAUNCH_TOKEN_LABEL = "io.envctl.launch-token" +_ENV_FILE_PID_PATTERN = re.compile( + r"^\.docker-env-[0-9a-f]{12}-p(?P[1-9][0-9]*)-[A-Za-z0-9_]+$" +) +_SECRET_SCRUB_FAILED = "secret scrub failed" + + +def _docker_name(value: str, *, limit: int) -> str: + normalized = re.sub(r"[^a-z0-9_.-]+", "-", value.strip().lower()).strip("-._") + return (normalized or "service")[:limit] + + +def docker_service_container_name(*, project_name: str, project_root: Path, service_name: str) -> str: + digest = hashlib.sha256(str(project_root.resolve()).encode()).hexdigest()[:10] + prefix = _docker_name(f"envctl-app-{project_name}-{service_name}", limit=52) + return f"{prefix}-{digest}" + + +@dataclass(frozen=True, slots=True) +class DockerServiceLaunch: + container_id: str + container_name: str + image: str + log_path: str + launch_token: str = "" + cleanup_pending_since: float | None = None + + +class DockerServiceLaunchCleanupError(RuntimeError): + """A failed launch whose container cleanup could not be confirmed.""" + + def __init__(self, message: str, launch: DockerServiceLaunch) -> None: + super().__init__(message) + self.launch = launch + + +def docker_service_mode_enabled(runtime: Any) -> bool: + env = getattr(runtime, "env", {}) + return parse_bool(str(env.get("DOCKER_MODE", "")), False) + + +def docker_service_container_command_source( + runtime: Any, + service_name: str, + *, + docker_mode: bool | None = None, +) -> str | None: + if not (docker_service_mode_enabled(runtime) if docker_mode is None else docker_mode): + return None + env = getattr(runtime, "env", {}) + raw = getattr(getattr(runtime, "config", None), "raw", {}) + suffix = re.sub(r"[^A-Z0-9]+", "_", service_name.upper()).strip("_") or "SERVICE" + + def setting(name: str) -> str: + for key in (f"ENVCTL_SERVICE_{suffix}_{name}", f"ENVCTL_{suffix}_{name}", f"ENVCTL_{name}"): + value = str(env.get(key) or raw.get(key) or "").strip() + if value: + return value + return "" + + if setting("DOCKER_COMMAND"): + return "docker_command" + if (setting("DOCKER_COMMAND_MODE") or "image").lower() in {"image", "default"}: + return "docker_image" + return None + + +def docker_service_uses_image_command( + runtime: Any, + service_name: str, + *, + docker_mode: bool | None = None, +) -> bool: + return docker_service_container_command_source( + runtime, + service_name, + docker_mode=docker_mode, + ) == "docker_image" + + +def state_uses_docker_services(state: object) -> bool: + metadata = getattr(state, "metadata", {}) + if isinstance(metadata, dict) and str(metadata.get("application_runtime", "")).lower() == "docker": + return True + services = getattr(state, "services", {}) + return isinstance(services, dict) and any( + str(getattr(service, "runtime_kind", "process") or "process").lower() == "docker" + for service in services.values() + ) + + +class DockerServiceRuntime: + """Build and run application services as managed Docker containers.""" + + def __init__(self, runtime: Any, process_runtime: Any, *, refresh_cache: bool = False) -> None: + self.runtime = runtime + self.process_runtime = process_runtime + self.refresh_cache = refresh_cache + + def launch( + self, + *, + project_name: str, + project_root: Path, + service_name: str, + cwd: Path, + command: Sequence[str], + env: Mapping[str, str], + host_port: int, + container_port: int, + listener_expected: bool, + log_path: str, + on_confirmed: Callable[[DockerServiceLaunch], None] | None = None, + ) -> DockerServiceLaunch: + self._require_docker() + image = self._resolve_image( + project_root=project_root, + service_name=service_name, + cwd=cwd, + build_env=env, + ) + container_name = self._container_name( + project_name=project_name, + project_root=project_root, + service_name=service_name, + ) + self._prepare_container_name(container_name) + effective_container_port = self._container_port(service_name, container_port, image=image) + container_env = self._container_env(env) + if listener_expected and effective_container_port > 0: + container_env["PORT"] = str(effective_container_port) + env_file = self._write_env_file( + project_name=project_name, + service_name=service_name, + values=container_env, + ) + launch_token = uuid.uuid4().hex + result: object + run_error: BaseException | None = None + docker_run_invoked = False + try: + docker_command = [ + "docker", + "run", + "--detach", + "--name", + container_name, + "--label", + "io.envctl.managed=true", + "--label", + f"io.envctl.project={project_name}", + "--label", + f"io.envctl.service={service_name}", + "--label", + f"{_RUNTIME_SCOPE_LABEL}={self._runtime_scope_identity()}", + "--label", + f"{_LAUNCH_TOKEN_LABEL}={launch_token}", + "--add-host", + "host.docker.internal:host-gateway", + "--env-file", + str(env_file), + ] + if listener_expected and host_port > 0: + docker_command.extend(["--publish", f"127.0.0.1:{host_port}:{effective_container_port}"]) + workdir = self._setting(service_name, "DOCKER_WORKDIR") + if workdir: + docker_command.extend(["--workdir", workdir]) + docker_command.append(image) + docker_command.extend( + self._container_command( + service_name, + command, + project_root=project_root, + cwd=cwd, + host_port=host_port, + container_port=effective_container_port, + ) + ) + docker_run_invoked = True + result = self.process_runtime.run( + docker_command, + cwd=project_root, + env={**os.environ, "DOCKER_BUILDKIT": "1"}, + timeout=self._timeout("ENVCTL_DOCKER_RUN_TIMEOUT_SECONDS", 120.0), + ) + except BaseException as exc: # noqa: BLE001 - cancellation still requires token cleanup + run_error = exc + result = None + finally: + env_cleanup_error = self._scrub_env_file_safely(env_file) + if env_cleanup_error: + self._emit_best_effort( + "service.container.env_file_cleanup_warning", + project=project_name, + service=service_name, + detail=env_cleanup_error, + ) + provisional_launch = DockerServiceLaunch( + container_id=container_name, + container_name=container_name, + image=image, + log_path=log_path, + launch_token=launch_token, + cleanup_pending_since=time.time(), + ) + if env_cleanup_error and _SECRET_SCRUB_FAILED in env_cleanup_error: + cleanup_detail = f"Docker env file cleanup failed: {env_cleanup_error}" + if not docker_run_invoked: + if run_error is not None: + raise RuntimeError(f"{run_error}; {cleanup_detail}") from run_error + raise RuntimeError(cleanup_detail) + self._raise_launch_error_after_cleanup( + cleanup_detail, + provisional_launch, + absence_can_confirm=self._definite_docker_rejection(result), + ) + if run_error is not None: + if not docker_run_invoked: + if isinstance(run_error, Exception): + raise RuntimeError(str(run_error)) from run_error + raise run_error + run_error_detail = str(run_error).strip() or type(run_error).__name__ + self._raise_launch_error_after_cleanup( + run_error_detail, + provisional_launch, + cause=run_error, + ) + if result is None: + self._raise_launch_error_after_cleanup("docker run returned no result", provisional_launch) + returncode = getattr(result, "returncode", None) + if not isinstance(returncode, int): + self._raise_launch_error_after_cleanup("docker run returned an invalid result", provisional_launch) + if returncode != 0: + detail = str( + getattr(result, "stderr", "") + or getattr(result, "stdout", "") + or "docker run failed" + ).strip() + self._raise_launch_error_after_cleanup( + detail, + provisional_launch, + absence_can_confirm=self._definite_docker_rejection(result), + ) + try: + output_lines = [ + line.strip() + for line in str(getattr(result, "stdout", "") or "").splitlines() + if line.strip() + ] + output_identity = output_lines[-1] if output_lines else "" + identity_ref = ( + output_identity + if re.fullmatch(r"[0-9a-fA-F]{64}", output_identity) + else container_name + ) + ownership, container_id = self._container_label_identity( + identity_ref, + label=_LAUNCH_TOKEN_LABEL, + expected_value=launch_token, + ) + except BaseException as exc: # noqa: BLE001 - post-run authority is still transactional + self._raise_launch_error_after_cleanup( + str(exc).strip() or type(exc).__name__, + provisional_launch, + cause=exc, + ) + if ownership != "owned" or container_id is None: + self._raise_launch_error_after_cleanup( + "docker run completed but the launched container identity could not be confirmed", + provisional_launch, + ) + try: + launch = DockerServiceLaunch( + container_id=container_id, + container_name=container_name, + image=image, + log_path=log_path, + launch_token=launch_token, + ) + if on_confirmed is not None: + on_confirmed(launch) + self._emit_best_effort( + "service.container.start", + project=project_name, + service=service_name, + container_name=container_name, + image=image, + port=host_port if listener_expected else None, + ) + return launch + except BaseException as exc: # noqa: BLE001 - confirmed identity was not handed off + self._raise_launch_error_after_cleanup( + str(exc).strip() or type(exc).__name__, + provisional_launch, + cause=exc, + ) + + def _raise_launch_error_after_cleanup( + self, + detail: str, + launch: DockerServiceLaunch, + *, + cause: BaseException | None = None, + absence_can_confirm: bool = False, + ) -> None: + cleanup_error: BaseException | None = None + try: + cleanup_confirmed = self._cleanup_failed_launch( + launch, + absence_can_confirm=absence_can_confirm, + ) + except BaseException as exc: # noqa: BLE001 - preserve authority across cleanup cancellation + cleanup_confirmed = False + cleanup_error = exc + if cleanup_confirmed: + if cause is not None: + if not isinstance(cause, Exception): + raise cause + raise RuntimeError(detail) from cause + raise RuntimeError(detail) + error = DockerServiceLaunchCleanupError( + f"{detail}; Docker cleanup could not confirm removal of {launch.container_name}", + launch, + ) + chained_cause = cause if cause is not None else cleanup_error + if chained_cause is not None: + raise error from chained_cause + raise error + + def _cleanup_failed_launch( + self, + launch: DockerServiceLaunch, + *, + absence_can_confirm: bool, + remove: bool = True, + ) -> bool: + deadline = time.monotonic() + self._timeout("ENVCTL_DOCKER_CLEANUP_DISCOVERY_SECONDS", 1.0) + while True: + try: + token_cleanup = self._cleanup_launch_token_matches( + launch.launch_token, + remove=remove, + ) + except BaseException: # noqa: BLE001 - cancellation cannot abandon authority + return False + if token_cleanup == "unknown": + return False + if token_cleanup == "removed": + return True + + try: + ownership, container_id = self._container_label_identity( + launch.container_name, + label=_LAUNCH_TOKEN_LABEL, + expected_value=launch.launch_token, + ) + except BaseException: # noqa: BLE001 - cancellation cannot abandon authority + return False + if ownership == "owned" and container_id is not None: + try: + return self._remove_container_identity(container_id, remove=remove) + except BaseException: # noqa: BLE001 - cancellation cannot abandon authority + return False + if ownership == "unknown": + return False + + remaining = deadline - time.monotonic() + if remaining <= 0: + # A token-filtered daemon query plus repeated name probes stayed + # absent for the whole discovery window. This covers definite + # Docker failures without losing a timeout-accepted container + # that becomes visible shortly after its client is terminated. + return absence_can_confirm + time.sleep(min(0.1, remaining)) + + @staticmethod + def _definite_docker_rejection(result: object | None) -> bool: + returncode = getattr(result, "returncode", None) + if not isinstance(returncode, int) or returncode in {0, 124}: + return False + detail = str(getattr(result, "stderr", "") or getattr(result, "stdout", "")).lower() + return "timed out" not in detail and "timeout" not in detail + + def _cleanup_launch_token_matches(self, launch_token: str, *, remove: bool) -> str: + matching_ids = self._container_ids_for_launch_token(launch_token) + if matching_ids is None: + return "unknown" + if not matching_ids: + return "absent" + removal_results: list[bool] = [] + for candidate_id in matching_ids: + ownership, confirmed_id = self._container_label_identity( + candidate_id, + label=_LAUNCH_TOKEN_LABEL, + expected_value=launch_token, + ) + if ownership == "absent": + removal_results.append(True) + elif ownership == "owned" and confirmed_id == candidate_id: + removal_results.append( + self._remove_container_identity(confirmed_id, remove=remove) + ) + else: + removal_results.append(False) + return "removed" if all(removal_results) else "unknown" + + def _container_ids_for_launch_token(self, launch_token: str) -> tuple[str, ...] | None: + if not launch_token: + return None + result = self.process_runtime.run( + [ + "docker", + "container", + "ls", + "--all", + "--quiet", + "--no-trunc", + "--filter", + f"label={_LAUNCH_TOKEN_LABEL}={launch_token}", + ], + timeout=10.0, + ) + if result.returncode != 0: + return None + identities = tuple( + dict.fromkeys( + line.strip() + for line in str(result.stdout or "").splitlines() + if line.strip() + ) + ) + if any(re.fullmatch(r"[0-9a-fA-F]{64}", identity) is None for identity in identities): + return None + return identities + + def wait_until_ready(self, launch: DockerServiceLaunch, *, port: int, listener_expected: bool) -> bool: + if not self.container_running(launch.container_id): + self.snapshot_logs(launch.container_id, launch.log_path, tail=200) + return False + if not listener_expected: + return True + ready = bool( + self.process_runtime.wait_for_port( + port, + timeout=self._timeout("ENVCTL_SERVICE_LISTENER_TIMEOUT_SECONDS", 30.0), + ) + ) + if not ready: + self.snapshot_logs(launch.container_id, launch.log_path, tail=200) + return ready + + def snapshot_logs(self, container: str, log_path: str, *, tail: int) -> bool: + command = ["docker", "logs"] + since_marker = Path(f"{log_path}.docker-since") + if since_marker.is_file(): + try: + command.extend(["--since", str(since_marker.stat().st_mtime)]) + except OSError: + pass + command.extend(["--tail", str(max(tail, 0)), container]) + result = self.process_runtime.run( + command, + timeout=30.0, + ) + if result.returncode != 0: + return False + stdout = str(result.stdout or "") + stderr = str(result.stderr or "") + content = stdout + (("\n" if stdout and stderr and not stdout.endswith("\n") else "") + stderr) + path = Path(log_path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return True + + def follow_logs(self, container: str, log_path: str) -> int | None: + process = self.process_runtime.start_background( + ["docker", "logs", "--follow", "--since", "0s", container], + stdout_path=log_path, + stderr_path=log_path, + ) + pid = getattr(process, "pid", None) + return pid if isinstance(pid, int) and pid > 0 else None + + def stop( + self, + container: str, + *, + remove: bool = True, + verify_ownership: bool = False, + expected_launch_token: str | None = None, + pending_cleanup_since: float | None = None, + ) -> bool: + if not container: + return True + target = container + if expected_launch_token: + ownership, container_id = self._container_label_identity( + container, + label=_LAUNCH_TOKEN_LABEL, + expected_value=expected_launch_token, + ) + if ownership == "owned" and container_id is not None: + target = container_id + elif ownership == "unknown": + return False + else: + token_cleanup = self._cleanup_launch_token_matches( + expected_launch_token, + remove=remove, + ) + if token_cleanup == "removed": + return True + if token_cleanup == "unknown" or ownership == "foreign": + return False + if pending_cleanup_since is None: + return True + try: + pending_timestamp = float(pending_cleanup_since) + except (TypeError, ValueError): + return False + if not math.isfinite(pending_timestamp) or pending_timestamp <= 0: + return False + grace = self._timeout("ENVCTL_DOCKER_PENDING_CLEANUP_GRACE_SECONDS", 30.0) + if time.time() - pending_timestamp < grace: + return False + provisional = DockerServiceLaunch( + container_id=container, + container_name=container, + image="", + log_path="", + launch_token=expected_launch_token, + cleanup_pending_since=pending_timestamp, + ) + return self._cleanup_failed_launch( + provisional, + absence_can_confirm=True, + remove=remove, + ) + elif verify_ownership: + ownership, container_id = self._container_label_identity( + container, + label=_RUNTIME_SCOPE_LABEL, + expected_value=self._runtime_scope_identity(), + ) + if ownership == "absent": + return True + if ownership != "owned" or container_id is None: + return False + target = container_id + return self._remove_container_identity(target, remove=remove) + + def _remove_container_identity(self, container_id: str, *, remove: bool) -> bool: + command = ["docker", "rm", "--force", container_id] if remove else ["docker", "stop", container_id] + result = self.process_runtime.run(command, timeout=30.0) + return result.returncode == 0 or "No such container" in str(result.stderr or "") + + def _container_label_identity( + self, + container: str, + *, + label: str, + expected_value: str, + ) -> tuple[str, str | None]: + if not container or not expected_value: + return "unknown", None + inspected = self.process_runtime.run( + [ + "docker", + "container", + "inspect", + "--format", + f"{{{{.Id}}}}|{{{{index .Config.Labels {json.dumps(label)}}}}}", + container, + ], + timeout=10.0, + ) + if inspected.returncode != 0: + return ("absent", None) if "No such" in str(inspected.stderr or "") else ("unknown", None) + raw_id, separator, actual_value = str(inspected.stdout or "").strip().partition("|") + container_id = raw_id.strip() + if not separator or re.fullmatch(r"[0-9a-fA-F]{64}", container_id) is None: + return "unknown", None + return ( + ("owned", container_id) + if actual_value.strip() == expected_value + else ("foreign", container_id) + ) + + def container_running(self, container: str) -> bool: + if not container: + return False + result = self.process_runtime.run( + ["docker", "inspect", "--format", "{{.State.Running}}", container], + timeout=10.0, + ) + return result.returncode == 0 and str(result.stdout or "").strip().lower() == "true" + + def _resolve_image( + self, + *, + project_root: Path, + service_name: str, + cwd: Path, + build_env: Mapping[str, str], + ) -> str: + configured_image = self._setting(service_name, "DOCKER_IMAGE") + dockerfile_setting = self._setting(service_name, "DOCKERFILE") + dockerfile = self._resolve_path(dockerfile_setting, project_root) if dockerfile_setting else None + if dockerfile is None and not configured_image: + dockerfile = next( + (path for path in (cwd / "Dockerfile", project_root / "Dockerfile") if path.is_file()), + None, + ) + if dockerfile is None: + if configured_image: + return configured_image + raise RuntimeError( + f"Docker mode needs ENVCTL_{self._service_key(service_name)}_DOCKER_IMAGE, " + "ENVCTL_DOCKER_IMAGE, or a Dockerfile" + ) + if not dockerfile.is_file(): + raise RuntimeError(f"Configured Dockerfile does not exist: {dockerfile}") + context_setting = self._setting(service_name, "DOCKER_CONTEXT") + context = self._resolve_path(context_setting, project_root) if context_setting else dockerfile.parent + if context is None or not context.is_dir(): + raise RuntimeError(f"Configured Docker build context does not exist: {context}") + image = configured_image or self._generated_image_name(project_root, service_name) + self._build_image( + image=image, + dockerfile=dockerfile, + context=context, + service_name=service_name, + build_env=build_env, + ) + return image + + def _build_image( + self, + *, + image: str, + dockerfile: Path, + context: Path, + service_name: str, + build_env: Mapping[str, str], + ) -> None: + policy = (self._setting(service_name, "DOCKER_BUILD_POLICY") or "cached").strip().lower() + if policy in {"never", "image", "existing"}: + if not self._image_exists(image): + raise RuntimeError(f"Docker image {image!r} does not exist and build policy is {policy!r}") + return + if policy == "missing" and self._image_exists(image): + self.runtime._emit("service.container.build", service=service_name, image=image, cache="image") + return + command = ["docker", "build", "--file", str(dockerfile), "--tag", image] + if self.refresh_cache or parse_bool( + str(getattr(self.runtime, "env", {}).get("RUN_SH_REFRESH_CACHE", "")), + False, + ): + command.append("--pull") + target = self._setting(service_name, "DOCKER_TARGET") + if target: + command.extend(["--target", target]) + build_arguments = self._docker_build_arguments(service_name, build_env) + for key in sorted(build_arguments): + command.extend(["--build-arg", key]) + command.append(str(context)) + self.runtime._emit( + "service.container.build", + service=service_name, + image=image, + dockerfile=str(dockerfile), + context=str(context), + cache="docker_layer_cache", + ) + result = self.process_runtime.run( + command, + cwd=context, + env={**os.environ, **build_arguments, "DOCKER_BUILDKIT": "1"}, + timeout=self._timeout("ENVCTL_DOCKER_BUILD_TIMEOUT_SECONDS", 900.0), + ) + if result.returncode != 0: + detail = str(result.stderr or result.stdout or "docker build failed").strip() + raise RuntimeError(detail) + + def _image_exists(self, image: str) -> bool: + return self.process_runtime.run(["docker", "image", "inspect", image], timeout=10.0).returncode == 0 + + def _prepare_container_name(self, name: str) -> None: + inspect = self.process_runtime.run( + [ + "docker", + "container", + "inspect", + "--format", + ( + f"{{{{.Id}}}}|{{{{.State.Running}}}}|" + f"{{{{index .Config.Labels {json.dumps(_RUNTIME_SCOPE_LABEL)}}}}}" + ), + name, + ], + timeout=10.0, + ) + if inspect.returncode != 0: + if "No such" in str(inspect.stderr or "") or "not found" in str(inspect.stderr or "").lower(): + return + raise RuntimeError(str(inspect.stderr or inspect.stdout or f"failed to inspect {name}").strip()) + container_id, separator, remainder = str(inspect.stdout or "").strip().partition("|") + running_text, owner_separator, owner = remainder.partition("|") + if ( + not separator + or not owner_separator + or re.fullmatch(r"[0-9a-fA-F]{64}", container_id.strip()) is None + ): + raise RuntimeError(f"Docker returned an invalid identity while inspecting {name!r}") + expected_owner = self._runtime_scope_identity() + if owner != expected_owner: + raise RuntimeError( + f"Docker container {name!r} is managed by another envctl runtime scope; " + "stop it from that runtime before starting this one" + ) + normalized_running = running_text.strip().lower() + if normalized_running not in {"true", "false"}: + raise RuntimeError(f"Docker returned an invalid running state while inspecting {name!r}") + if normalized_running == "true": + raise RuntimeError( + f"Docker container {name!r} is already running but was not resumed from current state; " + "run envctl stop for this project before starting it again" + ) + self._remove_container(container_id.strip()) + + def _remove_container(self, name: str) -> None: + result = self.process_runtime.run(["docker", "rm", "--force", name], timeout=30.0) + if result.returncode != 0 and "No such container" not in str(result.stderr or ""): + raise RuntimeError(str(result.stderr or result.stdout or f"failed to remove {name}").strip()) + + def _runtime_scope_identity(self) -> str: + config = getattr(self.runtime, "config", None) + scope = getattr(config, "runtime_scope_dir", None) + if scope is None: + scope = Path(self.runtime._run_dir_path("docker-scope")).parent + return hashlib.sha256(str(Path(scope).resolve()).encode()).hexdigest()[:16] + + def _require_docker(self) -> None: + command_exists = getattr(self.runtime, "_command_exists", None) + if callable(command_exists) and not command_exists("docker"): + raise RuntimeError("Docker mode requires the docker CLI") + with _DOCKER_INFO_LOCK: + if bool(getattr(self.runtime, "_docker_application_daemon_checked", False)): + return + result = self.process_runtime.run( + ["docker", "info", "--format", "{{.ServerVersion}}"], + timeout=15.0, + ) + if result.returncode != 0: + raise RuntimeError(str(result.stderr or "Docker daemon is unavailable").strip()) + setattr(self.runtime, "_docker_application_daemon_checked", True) + + def _container_command( + self, + service_name: str, + command: Sequence[str], + *, + project_root: Path, + cwd: Path, + host_port: int, + container_port: int, + ) -> list[str]: + override = self._setting(service_name, "DOCKER_COMMAND") + command_mode = (self._setting(service_name, "DOCKER_COMMAND_MODE") or "image").lower() + if override: + resolved = shlex.split(override) + elif command_mode in {"service", "envctl", "host"}: + resolved = [str(part) for part in command] + elif command_mode in {"image", "default"}: + return [] + else: + raise RuntimeError( + f"Invalid Docker command mode for {service_name}: {command_mode!r}; expected image or service" + ) + if not resolved: + return [] + executable = Path(resolved[0]) + if executable.is_absolute(): + try: + resolved[0] = str(executable.relative_to(cwd)) + except ValueError: + try: + resolved[0] = str(executable.relative_to(project_root)) + except ValueError: + pass + normalized: list[str] = [] + for part in resolved: + if part in {"127.0.0.1", "localhost"}: + normalized.append("0.0.0.0") + continue + if host_port > 0 and container_port > 0: + if part == str(host_port): + normalized.append(str(container_port)) + continue + part = re.sub( + rf"^(--port=){re.escape(str(host_port))}$", + rf"\g<1>{container_port}", + part, + ) + part = re.sub(r"^(--host=)(localhost|127\.0\.0\.1)$", r"\g<1>0.0.0.0", part) + normalized.append(part) + return normalized + + def _docker_build_arguments( + self, + service_name: str, + build_env: Mapping[str, str], + ) -> dict[str, str]: + arguments = { + str(key): str(value) + for key, value in build_env.items() + if str(key).startswith(_PUBLIC_ENV_PREFIXES) + } + configured = self._setting(service_name, "DOCKER_BUILD_ARGS") + for token in configured.split(","): + item = token.strip() + if not item: + continue + key, separator, configured_value = item.partition("=") + key = key.strip() + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + raise RuntimeError(f"Invalid Docker build argument name: {key!r}") + if separator: + arguments[key] = configured_value + elif key in build_env: + arguments[key] = str(build_env[key]) + else: + raise RuntimeError(f"Docker build argument {key!r} is not present in the service environment") + return arguments + + def _container_env(self, values: Mapping[str, str]) -> dict[str, str]: + result: dict[str, str] = {} + browser_facing_keys = self._browser_facing_env_keys() + for raw_key, raw_value in values.items(): + key = str(raw_key) + if key in _HOST_ENV_KEYS or key.startswith("TERM_") or key.startswith("LC_"): + continue + value = str(raw_value) + if key in {"HOST", "BIND_HOST", "LISTEN_HOST", "SERVER_HOST"} and value in { + "localhost", + "127.0.0.1", + }: + result[key] = "0.0.0.0" + continue + if key not in browser_facing_keys and not key.startswith(_PUBLIC_ENV_PREFIXES): + value = re.sub( + r"(://(?:[^/@\s]+@)?)(localhost|127\.0\.0\.1)(?=[:/]|$)", + r"\1host.docker.internal", + value, + ) + if key.endswith("_HOST") and value in {"localhost", "127.0.0.1"}: + value = "host.docker.internal" + result[key] = value + result["ENVCTL_CONTAINER_RUNTIME"] = "docker" + result["ENVCTL_HOST_GATEWAY"] = "host.docker.internal" + return result + + def _browser_facing_env_keys(self) -> set[str]: + keys = set(_PUBLIC_ENV_KEYS) + runtime_env = getattr(self.runtime, "env", {}) + config_raw = getattr(getattr(self.runtime, "config", None), "raw", {}) + cors_key = str( + runtime_env.get( + "ENVCTL_BACKEND_CORS_ENV_KEY", + config_raw.get("ENVCTL_BACKEND_CORS_ENV_KEY", ""), + ) + or "" + ).strip() + if cors_key: + keys.add(cors_key) + return keys + + def _write_env_file(self, *, project_name: str, service_name: str, values: Mapping[str, str]) -> Path: + run_dir = Path(self.runtime._run_dir_path(getattr(self.runtime, "_active_run_id", None) or "docker")) + run_dir.mkdir(parents=True, exist_ok=True) + stale_errors = self._scrub_stale_env_files(run_dir) + unsafe_errors = [detail for detail in stale_errors if _SECRET_SCRUB_FAILED in detail] + if unsafe_errors: + raise RuntimeError("Stale Docker env file cleanup failed: " + "; ".join(unsafe_errors)) + for detail in stale_errors: + self._emit_best_effort("service.container.env_file_cleanup_warning", detail=detail, stale=True) + digest = hashlib.sha256(f"{project_name}\0{service_name}".encode()).hexdigest()[:12] + descriptor, raw_path = tempfile.mkstemp( + prefix=f".docker-env-{digest}-p{os.getpid()}-", + dir=run_dir, + ) + path = Path(raw_path) + try: + lines = [ + f"{key}={str(value).replace(chr(10), '')}" + for key, value in sorted(values.items()) + ] + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + descriptor = -1 + handle.write("\n".join(lines) + "\n") + return path + except BaseException as write_error: + if descriptor >= 0: + try: + os.close(descriptor) + except BaseException: # noqa: BLE001 - continue to pathname scrub + pass + cleanup_error = self._scrub_env_file_safely(path) + if cleanup_error: + self._emit_best_effort( + "service.container.env_file_cleanup_warning", + project=project_name, + service=service_name, + detail=cleanup_error, + write_failed=True, + ) + raise RuntimeError( + f"Docker env file write failed: {write_error}; cleanup failed: {cleanup_error}" + ) from write_error + raise + + def _scrub_stale_env_files(self, run_dir: Path) -> list[str]: + errors: list[str] = [] + legacy_grace = self._timeout("ENVCTL_DOCKER_ENV_STALE_SECONDS", 300.0) + now = time.time() + for path in tuple(run_dir.glob(".docker-env-*")): + match = _ENV_FILE_PID_PATTERN.fullmatch(path.name) + if match is not None: + pid = int(match.group("pid")) + if pid == os.getpid() or self._pid_is_running(pid): + continue + else: + try: + if now - path.lstat().st_mtime < legacy_grace: + continue + except OSError as exc: + errors.append(f"{path}: {_SECRET_SCRUB_FAILED}: {exc}") + continue + detail = self._scrub_env_file_safely(path) + if detail: + errors.append(f"{path}: {detail}") + return errors + + @staticmethod + def _pid_is_running(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except (PermissionError, OSError): + return True + return True + + @staticmethod + def _scrub_env_file(path: Path) -> str | None: + try: + path.unlink(missing_ok=True) + return None + except OSError as unlink_error: + if not path.exists() and not path.is_symlink(): + return None + try: + flags = os.O_WRONLY | os.O_TRUNC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags) + os.close(descriptor) + except OSError as scrub_error: + return f"{unlink_error}; secret scrub failed: {scrub_error}" + try: + path.unlink(missing_ok=True) + return None + except OSError as retry_error: + return f"{unlink_error}; file was scrubbed but unlink retry failed: {retry_error}" + + @staticmethod + def _scrub_env_file_safely(path: Path) -> str | None: + try: + return DockerServiceRuntime._scrub_env_file(path) + except BaseException as exc: # noqa: BLE001 - secret cleanup must never be masked + return f"{_SECRET_SCRUB_FAILED}: {exc}" + + def _emit_best_effort(self, event: str, **payload: object) -> None: + try: + self.runtime._emit(event, **payload) + except BaseException: # noqa: BLE001 - diagnostics cannot interrupt authority handoff + pass + + def _setting(self, service_name: str, suffix: str) -> str: + env = getattr(self.runtime, "env", {}) + raw = getattr(getattr(self.runtime, "config", None), "raw", {}) + service_key = f"ENVCTL_{self._service_key(service_name)}_{suffix}" + additional_service_key = f"ENVCTL_SERVICE_{self._service_key(service_name)}_{suffix}" + global_key = f"ENVCTL_{suffix}" + for key in (additional_service_key, service_key, global_key): + value = str(env.get(key) or raw.get(key) or "").strip() + if value: + return value + return "" + + def _container_port(self, service_name: str, host_port: int, *, image: str) -> int: + value = self._setting(service_name, "DOCKER_PORT") + if not value: + self._ensure_image_available(image) + exposed = self._single_exposed_port(image) + return exposed if exposed is not None else host_port + try: + port = int(value) + except ValueError as exc: + raise RuntimeError(f"Invalid Docker container port for {service_name}: {value!r}") from exc + if port <= 0 or port > 65535: + raise RuntimeError(f"Invalid Docker container port for {service_name}: {value!r}") + return port + + def _ensure_image_available(self, image: str) -> None: + if self._image_exists(image): + return + self.runtime._emit("service.container.pull", image=image) + pulled = self.process_runtime.run( + ["docker", "pull", image], + timeout=self._timeout("ENVCTL_DOCKER_PULL_TIMEOUT_SECONDS", 300.0), + ) + if pulled.returncode != 0: + detail = str(pulled.stderr or pulled.stdout or f"failed to pull Docker image {image}").strip() + raise RuntimeError(detail) + + def _single_exposed_port(self, image: str) -> int | None: + result = self.process_runtime.run( + ["docker", "image", "inspect", "--format", "{{json .Config.ExposedPorts}}", image], + timeout=10.0, + ) + if result.returncode != 0: + return None + try: + payload = json.loads(str(result.stdout or "null")) + except json.JSONDecodeError: + return None + if not isinstance(payload, dict) or len(payload) != 1: + return None + raw_port = str(next(iter(payload))).partition("/")[0] + try: + port = int(raw_port) + except ValueError: + return None + return port if 0 < port <= 65535 else None + + def _container_name(self, *, project_name: str, project_root: Path, service_name: str) -> str: + return docker_service_container_name( + project_name=project_name, + project_root=project_root, + service_name=service_name, + ) + + def _generated_image_name(self, project_root: Path, service_name: str) -> str: + identity = self._repository_identity(project_root) + digest = hashlib.sha256(str(project_root.resolve()).encode()).hexdigest()[:10] + repo = _docker_name(identity.name, limit=32) + service = _docker_name(service_name, limit=24) + return f"envctl/{repo}-{service}:{digest}" + + @staticmethod + def _repository_identity(project_root: Path) -> Path: + dot_git = project_root / ".git" + if dot_git.is_file(): + try: + gitdir = dot_git.read_text(encoding="utf-8").strip().removeprefix("gitdir:").strip() + resolved = (project_root / gitdir).resolve() + if resolved.parent.name == "worktrees": + return resolved.parent.parent.parent + except OSError: + pass + return project_root.resolve() + + @staticmethod + def _service_key(service_name: str) -> str: + return re.sub(r"[^A-Z0-9]+", "_", service_name.upper()).strip("_") or "SERVICE" + + @staticmethod + def _resolve_path(value: str, project_root: Path) -> Path | None: + if not value: + return None + path = Path(value).expanduser() + return path.resolve() if path.is_absolute() else (project_root / path).resolve() + + def _timeout(self, key: str, default: float) -> float: + value = str(getattr(self.runtime, "env", {}).get(key, "") or "").strip() + try: + return max(float(value), 1.0) if value else default + except ValueError: + return default + + +def refresh_container_log_snapshots( + runtime: Any, + state: object, + *, + tail: int, + follow: bool, +) -> list[int]: + docker_services = [ + service + for service in getattr(state, "services", {}).values() + if str(getattr(service, "runtime_kind", "process") or "process").lower() == "docker" + ] + if not docker_services: + return [] + process_runner = getattr(runtime, "process_runner", None) + if process_runner is None: + return [] + docker_runtime = DockerServiceRuntime(runtime, process_runner) + follower_pids: list[int] = [] + for service in docker_services: + container = str( + getattr(service, "container_id", "") or getattr(service, "container_name", "") or "" + ).strip() + log_path = str(getattr(service, "log_path", "") or "").strip() + if not container or not log_path: + continue + docker_runtime.snapshot_logs(container, log_path, tail=tail) + if follow: + pid = docker_runtime.follow_logs(container, log_path) + if pid is not None: + follower_pids.append(pid) + return follower_pids + + +def stop_container_log_followers(runtime: Any, pids: Sequence[int]) -> None: + for pid in pids: + try: + runtime.process_runner.terminate_process_group(pid, term_timeout=1.0, kill_timeout=1.0) + except Exception: # noqa: BLE001 + continue diff --git a/python/envctl_engine/runtime/endpoints_command_support.py b/python/envctl_engine/runtime/endpoints_command_support.py index c2c9ae51b..8b863e570 100644 --- a/python/envctl_engine/runtime/endpoints_command_support.py +++ b/python/envctl_engine/runtime/endpoints_command_support.py @@ -189,6 +189,10 @@ def _service_project_name(service_name: str, service: ServiceRecord, *, runtime: def _requirements_for_project(state: RunState, project: str) -> RequirementsResult | None: if project in state.requirements: return state.requirements[project] + target = str(project).strip().casefold() + for storage_key, requirements in state.requirements.items(): + if str(requirements.project or storage_key).strip().casefold() == target: + return requirements if len(state.requirements) == 1: return next(iter(state.requirements.values())) return None diff --git a/python/envctl_engine/runtime/engine_runtime_artifacts.py b/python/envctl_engine/runtime/engine_runtime_artifacts.py index aaf7d9310..21cec76f4 100644 --- a/python/envctl_engine/runtime/engine_runtime_artifacts.py +++ b/python/envctl_engine/runtime/engine_runtime_artifacts.py @@ -6,8 +6,9 @@ import time from collections.abc import Mapping, Sequence from pathlib import Path -from typing import Any, Protocol +from typing import Any, Callable, Protocol +from envctl_engine.runtime.runtime_context import optional_state_repository from envctl_engine.runtime.runtime_readiness import ( FEATURE_MATRIX_RELATIVE_PATH, GAP_REPORT_RELATIVE_PATH, @@ -27,7 +28,14 @@ class _SummaryContext(Protocol): ports: Mapping[str, _FinalPort] -def write_artifacts(runtime: Any, state: object, contexts: list[object], *, errors: list[str]) -> None: +def write_artifacts( + runtime: Any, + state: object, + contexts: list[object], + *, + errors: list[str], + on_commit: Callable[[], None] | None = None, +) -> None: started = time.monotonic() cached = _cached_runtime_readiness_payload(runtime) paths = runtime.state_repository.save_run( @@ -42,6 +50,7 @@ def write_artifacts(runtime: Any, state: object, contexts: list[object], *, erro if cached is not None else (lambda run_dir: _write_pending_runtime_readiness_payload(runtime, run_dir=run_dir)) ), + on_commit=on_commit, ) runtime._emit( "artifacts.write", @@ -60,36 +69,37 @@ def write_runtime_readiness_report( *, run_dir: Path | None = None, readiness_result: RuntimeReadinessResult | None = None, -) -> None: +) -> bool: started = time.monotonic() emit = getattr(runtime, "_emit", None) cached = _cached_runtime_readiness_payload(runtime) if readiness_result is None and cached is not None: - _write_runtime_readiness_payload(runtime, report_text=cached, run_dir=run_dir) + persisted = _write_runtime_readiness_payload(runtime, report_text=cached, run_dir=run_dir) if callable(emit): emit( "artifacts.runtime_readiness_report", duration_ms=round((time.monotonic() - started) * 1000.0, 2), used_cached_contract=True, run_dir=str(run_dir) if run_dir is not None else None, + persisted=persisted, ) - return + return persisted readiness = ( - readiness_result - if readiness_result is not None - else evaluate_runtime_readiness(runtime.config.base_dir) + readiness_result if readiness_result is not None else evaluate_runtime_readiness(runtime.config.base_dir) ) report_payload = build_runtime_readiness_report(readiness) report_text = json.dumps(report_payload, indent=2, sort_keys=True) - _write_runtime_readiness_payload(runtime, report_text=report_text, run_dir=run_dir) + persisted = _write_runtime_readiness_payload(runtime, report_text=report_text, run_dir=run_dir) if callable(emit): emit( "artifacts.runtime_readiness_report", duration_ms=round((time.monotonic() - started) * 1000.0, 2), used_cached_contract=readiness_result is not None, run_dir=str(run_dir) if run_dir is not None else None, + persisted=persisted, ) + return persisted def _write_runtime_readiness_payload( @@ -97,12 +107,24 @@ def _write_runtime_readiness_payload( *, report_text: str, run_dir: Path | None, -) -> None: +) -> bool: + repository = optional_state_repository(runtime) + repository_writer = getattr(repository, "write_runtime_artifact", None) + if callable(repository_writer): + return bool( + repository_writer( + run_id=run_dir.name if run_dir is not None else None, + artifact_name="runtime_readiness_report.json", + text=report_text, + ) + ) + (runtime.runtime_root / "runtime_readiness_report.json").write_text(report_text, encoding="utf-8") runtime.runtime_legacy_root.mkdir(parents=True, exist_ok=True) (runtime.runtime_legacy_root / "runtime_readiness_report.json").write_text(report_text, encoding="utf-8") if run_dir is not None: (run_dir / "runtime_readiness_report.json").write_text(report_text, encoding="utf-8") + return True def _write_cached_runtime_readiness_payload( @@ -110,9 +132,9 @@ def _write_cached_runtime_readiness_payload( *, run_dir: Path, cached: str, -) -> None: +) -> bool: started = time.monotonic() - _write_runtime_readiness_payload(runtime, report_text=cached, run_dir=run_dir) + persisted = _write_runtime_readiness_payload(runtime, report_text=cached, run_dir=run_dir) emit = getattr(runtime, "_emit", None) if callable(emit): emit( @@ -120,10 +142,12 @@ def _write_cached_runtime_readiness_payload( duration_ms=round((time.monotonic() - started) * 1000.0, 2), used_cached_contract=True, run_dir=str(run_dir), + persisted=persisted, ) + return persisted -def _write_pending_runtime_readiness_payload(runtime: Any, *, run_dir: Path) -> None: +def _write_pending_runtime_readiness_payload(runtime: Any, *, run_dir: Path) -> bool: report = { "passed": None, "pending": True, @@ -139,23 +163,29 @@ def _write_pending_runtime_readiness_payload(runtime: Any, *, run_dir: Path) -> "errors": [], "warnings": [], } - _write_runtime_readiness_payload( + persisted = _write_runtime_readiness_payload( runtime, report_text=json.dumps(report, indent=2, sort_keys=True), run_dir=run_dir, ) emit = getattr(runtime, "_emit", None) if callable(emit): - emit("artifacts.runtime_readiness_report.pending", run_dir=str(run_dir)) + emit("artifacts.runtime_readiness_report.pending", run_dir=str(run_dir), persisted=persisted) + return persisted def _start_background_runtime_readiness_report(runtime: Any, *, run_dir: Path) -> None: def _worker() -> None: try: - write_runtime_readiness_report(runtime, run_dir=run_dir) + persisted = write_runtime_readiness_report(runtime, run_dir=run_dir) emit = getattr(runtime, "_emit", None) if callable(emit): - emit("artifacts.runtime_readiness_report.complete", run_dir=str(run_dir)) + event = ( + "artifacts.runtime_readiness_report.complete" + if persisted + else "artifacts.runtime_readiness_report.discarded" + ) + emit(event, run_dir=str(run_dir)) except Exception as exc: # pragma: no cover emit = getattr(runtime, "_emit", None) if callable(emit): @@ -221,12 +251,28 @@ def _runtime_readiness_async_enabled(runtime: Any) -> bool: def print_summary(_runtime: Any, state: object, contexts: Sequence[_SummaryContext]) -> None: - _ = state print("envctl Python engine run summary") for context in contexts: - backend = context.ports["backend"].final - frontend = context.ports["frontend"].final + backend = _summary_service_port(state, context.name, "backend") or context.ports["backend"].final + frontend = _summary_service_port(state, context.name, "frontend") or context.ports["frontend"].final db = context.ports["db"].final redis = context.ports["redis"].final n8n = context.ports["n8n"].final print(f"- {context.name}: backend={backend} frontend={frontend} db={db} redis={redis} n8n={n8n}") + + +def _summary_service_port(state: object, project_name: str, service_type: str) -> int | None: + for service_name, service in getattr(state, "services", {}).items(): + project = str(getattr(service, "project", "") or "").strip() + if not project: + suffix = f" {service_type}".lower() + rendered_name = str(service_name) + project = rendered_name[: -len(suffix)] if rendered_name.lower().endswith(suffix) else "" + if project.casefold() != project_name.casefold(): + continue + if str(getattr(service, "type", "") or "").casefold() != service_type.casefold(): + continue + port = getattr(service, "actual_port", None) or getattr(service, "requested_port", None) + if isinstance(port, int) and port > 0: + return port + return None diff --git a/python/envctl_engine/runtime/engine_runtime_commands.py b/python/envctl_engine/runtime/engine_runtime_commands.py index 49fbddf58..996e84dd3 100644 --- a/python/envctl_engine/runtime/engine_runtime_commands.py +++ b/python/envctl_engine/runtime/engine_runtime_commands.py @@ -4,9 +4,11 @@ from pathlib import Path import shlex import shutil +import sys from typing import Any, Mapping from envctl_engine.runtime.command_resolution import resolve_requirement_start_command, resolve_service_start_command +from envctl_engine.runtime.docker_service_runtime import docker_service_container_command_source SERVICE_LAUNCH_ENV_REMOVALS = { "ACTIONS_CACHE_URL", @@ -105,6 +107,9 @@ def service_start_command_resolved( project_root: Path | None = None, port: int = 0, ) -> tuple[list[str], str]: + container_command_source = docker_service_container_command_source(runtime, service_name) + if container_command_source is not None: + return [], container_command_source result = resolve_service_start_command( service_name=service_name, project_root=(project_root or runtime.config.base_dir), @@ -150,14 +155,14 @@ def split_command( def _command_exists_for_cwd(runtime: Any, executable: str, *, cwd: str | Path | None) -> bool: - if runtime._command_exists(executable): - return True - if cwd is None or ("/" not in executable and "\\" not in executable): - return False + if "/" not in executable and "\\" not in executable: + return bool(runtime._command_exists(executable)) candidate = Path(executable).expanduser() - if candidate.is_absolute(): - return candidate.exists() - return (Path(cwd) / candidate).exists() + if not candidate.is_absolute(): + if cwd is None: + return False + candidate = Path(cwd) / candidate + return candidate.is_file() and os.access(candidate, os.X_OK) def command_env(runtime: Any, *, port: int, extra: Mapping[str, str] | None = None) -> dict[str, str]: @@ -166,6 +171,15 @@ def command_env(runtime: Any, *, port: int, extra: Mapping[str, str] | None = No env["PORT"] = str(port) if extra: env.update(extra) + # Every envctl process necessarily has one known-good Python executable, + # even on minimal hosts that intentionally provide no ``python`` shim. + # Repository-owned launchers can use this explicit path to bootstrap their + # own isolated environments without guessing which command name exists. + python_executable = str(sys.executable or "").strip() + if python_executable: + env["ENVCTL_PYTHON_EXECUTABLE"] = python_executable + else: + env.pop("ENVCTL_PYTHON_EXECUTABLE", None) for key in SERVICE_LAUNCH_ENV_REMOVALS: env.pop(key, None) return env diff --git a/python/envctl_engine/runtime/engine_runtime_construction.py b/python/envctl_engine/runtime/engine_runtime_construction.py index 70531d228..881773307 100644 --- a/python/envctl_engine/runtime/engine_runtime_construction.py +++ b/python/envctl_engine/runtime/engine_runtime_construction.py @@ -30,9 +30,23 @@ def initialize_runtime_construction(runtime: Any, config: EngineConfig, *, env: runtime.env = dict(env or {}) runtime.runtime_legacy_root = config.runtime_dir / "python-engine" runtime.runtime_root = config.runtime_scope_dir - runtime.runtime_legacy_root.mkdir(parents=True, exist_ok=True) - runtime.runtime_root.mkdir(parents=True, exist_ok=True) - runtime._ensure_legacy_lock_view() + compat_mode = runtime._state_compat_mode() + runtime.state_repository = RuntimeStateRepository( + runtime_root=runtime.runtime_root, + runtime_legacy_root=runtime.runtime_legacy_root, + runtime_dir=runtime.config.runtime_dir, + runtime_scope_id=runtime.config.runtime_scope_id, + compat_mode=compat_mode, + ) + # The repository canonicalizes ancestor symlinks before applying its + # containment and ownership checks. Keep every runtime consumer on that + # same canonical identity so persisted artifact paths and caller-visible + # paths never disagree (for example, macOS /var versus /private/var). + runtime.runtime_root = runtime.state_repository.runtime_root + runtime.runtime_legacy_root = runtime.state_repository.runtime_legacy_root + runtime.state_repository.ensure_runtime_roots() + if compat_mode == RuntimeStateRepository.COMPAT_READ_WRITE: + runtime._ensure_legacy_lock_view() runtime.port_planner = PortPlanner( backend_base=config.backend_port_base, frontend_base=config.frontend_port_base, @@ -61,7 +75,6 @@ def initialize_runtime_construction(runtime: Any, config: EngineConfig, *, env: ), ) runtime.requirements = RequirementsOrchestrator() - runtime.services = ServiceManager() runtime.events = [] runtime._emit_lock = threading.Lock() runtime._emit_listeners = [] @@ -69,9 +82,11 @@ def initialize_runtime_construction(runtime: Any, config: EngineConfig, *, env: runtime._startup_warnings_by_project = {} runtime._debug_hash_salt = uuid.uuid4().hex runtime._debug_recorder = None + runtime._debug_recorder_failure = None runtime._active_command_id = None runtime._last_debug_bundle_path = None runtime.process_runner = ProcessRunner(emit=runtime._emit) + runtime.services = ServiceManager(process_runner=runtime.process_runner) probe_backend_name = "psutil" if runtime._probe_psutil_enabled() else "shell" probe_backend = runtime._build_process_probe_backend() runtime.process_probe = ProcessProbe(probe_backend) @@ -89,13 +104,6 @@ def initialize_runtime_construction(runtime: Any, config: EngineConfig, *, env: "backend": runtime._conflict_count("BACKEND"), "frontend": runtime._conflict_count("FRONTEND"), } - runtime.state_repository = RuntimeStateRepository( - runtime_root=runtime.runtime_root, - runtime_legacy_root=runtime.runtime_legacy_root, - runtime_dir=runtime.config.runtime_dir, - runtime_scope_id=runtime.config.runtime_scope_id, - compat_mode=runtime._state_compat_mode(), - ) runtime.runtime_context = RuntimeContext( config=runtime.config, env=runtime.env, diff --git a/python/envctl_engine/runtime/engine_runtime_dispatch.py b/python/envctl_engine/runtime/engine_runtime_dispatch.py index 3254549a3..7d36cf34a 100644 --- a/python/envctl_engine/runtime/engine_runtime_dispatch.py +++ b/python/envctl_engine/runtime/engine_runtime_dispatch.py @@ -2,13 +2,35 @@ from typing import Any -from .command_policy import dispatch_family_for_command +from .command_policy import ( + LIFECYCLE_CLEANUP_COMMANDS, + STARTUP_COMMANDS, + dispatch_family_for_command, +) from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.inspection_support import dispatch_direct_inspection +from envctl_engine.runtime.lifecycle_operation_lease import run_exclusive_lifecycle_operation from envctl_engine.runtime.utility_command_support import dispatch_utility_command from envctl_engine.shared.process_probe import ProcessProbe +_EXCLUSIVE_OPERATION_COMMANDS = frozenset( + { + *LIFECYCLE_CLEANUP_COMMANDS, + *STARTUP_COMMANDS, + "blast-worktree", + "delete-worktree", + "ensure-worktree", + "resume", + "self-destruct-worktree", + } +) + + +def _requires_exclusive_operation(command: str) -> bool: + return command in _EXCLUSIVE_OPERATION_COMMANDS + + def dispatch(runtime: Any, route: Route) -> int: runtime.process_probe = ProcessProbe(runtime._build_process_probe_backend()) effective_mode = getattr(route, "mode", "") @@ -28,7 +50,14 @@ def dispatch(runtime: Any, route: Route) -> int: effective_mode=effective_mode, command=getattr(route, "command", ""), ) - return dispatch_command(runtime, route) + command = str(getattr(route, "command", "")).strip() + if not _requires_exclusive_operation(command): + return dispatch_command(runtime, route) + return run_exclusive_lifecycle_operation( + runtime, + command=command, + operation=lambda: dispatch_command(runtime, route), + ) def dispatch_command(runtime: Any, route: Route) -> int: diff --git a/python/envctl_engine/runtime/engine_runtime_event_support.py b/python/envctl_engine/runtime/engine_runtime_event_support.py index 53e2a9685..7c8bbce0b 100644 --- a/python/envctl_engine/runtime/engine_runtime_event_support.py +++ b/python/envctl_engine/runtime/engine_runtime_event_support.py @@ -6,7 +6,7 @@ from typing import Any, Mapping from envctl_engine.runtime.command_router import Route -from envctl_engine.runtime.runtime_context import run_dir_path +from envctl_engine.runtime.runtime_context import optional_state_repository, run_dir_path from envctl_engine.debug.debug_contract import apply_debug_event_contract from envctl_engine.debug.debug_utils import debug_env_value, hash_command, hash_value from envctl_engine.shared.parsing import parse_bool, parse_int @@ -70,7 +70,14 @@ def emit(runtime: Any, event_name: str, **payload: object) -> None: recorder_payload = dict(safe_payload) component_value = recorder_payload.pop("component", None) component: str | None = component_value if isinstance(component_value, str) else None - runtime._debug_recorder.record(event_name, component=component, **recorder_payload) + try: + runtime._debug_recorder.record(event_name, component=component, **recorder_payload) + except Exception as exc: # noqa: BLE001 + # Debug telemetry must never interrupt lifecycle ownership changes. + # Disable a failed recorder so a read-only diagnostic sink cannot + # repeatedly destabilize the command that it is observing. + runtime._debug_recorder_failure = str(exc) + runtime._debug_recorder = None for listener in list(runtime._emit_listeners): try: listener(event_name, dict(safe_payload)) @@ -112,6 +119,16 @@ def auto_debug_pack(runtime: Any, *, reason: str) -> None: def persist_events_snapshot(runtime: Any) -> None: events_text = "".join(json.dumps(event, sort_keys=True) + "\n" for event in runtime.events) + repository = optional_state_repository(runtime) + repository_writer = getattr(repository, "write_runtime_artifact", None) + if callable(repository_writer): + repository_writer( + run_id=_current_run_id(runtime), + artifact_name="events.jsonl", + text=events_text, + ) + return + (runtime.runtime_root / "events.jsonl").write_text(events_text, encoding="utf-8") runtime.runtime_legacy_root.mkdir(parents=True, exist_ok=True) (runtime.runtime_legacy_root / "events.jsonl").write_text(events_text, encoding="utf-8") @@ -121,6 +138,17 @@ def persist_events_snapshot(runtime: Any) -> None: def _current_run_dir(runtime: Any) -> Path | None: + run_id = _current_run_id(runtime) + if run_id is None: + return None + try: + candidate = run_dir_path(runtime, run_id) + except Exception: + return None + return candidate + + +def _current_run_id(runtime: Any) -> str | None: env = getattr(runtime, "env", None) if not isinstance(env, dict): return None @@ -130,11 +158,7 @@ def _current_run_dir(runtime: Any) -> Path | None: run_id = raw_run_id.strip() if not run_id: return None - try: - candidate = run_dir_path(runtime, run_id) - except Exception: - return None - return candidate + return run_id def current_session_id(runtime: Any) -> str | None: diff --git a/python/envctl_engine/runtime/engine_runtime_hooks.py b/python/envctl_engine/runtime/engine_runtime_hooks.py index 2085ec347..1eeb78a7e 100644 --- a/python/envctl_engine/runtime/engine_runtime_hooks.py +++ b/python/envctl_engine/runtime/engine_runtime_hooks.py @@ -7,7 +7,7 @@ from envctl_engine.shared.dependency_compose_assets import materialize_dependency_compose, supabase_managed_env from envctl_engine.shared.hooks import HookInvocationResult, legacy_shell_hook_issue, run_envctl_hook from envctl_engine.state.models import RequirementsResult, ServiceRecord -from envctl_engine.shared.parsing import parse_bool, parse_int +from envctl_engine.shared.parsing import parse_bool, parse_float_or_none, parse_int from envctl_engine.requirements.supabase import build_supabase_project_name from envctl_engine.requirements.core import dependency_definitions @@ -148,6 +148,16 @@ def services_from_hook_payload( actual_port=(actual_port if actual_port > 0 else None), log_path=(str(log_path) if log_path else None), status=status, + project=str(context.name), + service_slug=service_type.strip().lower(), + runtime_kind=str(raw.get("runtime_kind") or "process"), + container_id=str(raw.get("container_id") or "") or None, + container_name=str(raw.get("container_name") or "") or None, + container_image=str(raw.get("container_image") or "") or None, + container_launch_token=str(raw.get("container_launch_token") or "") or None, + container_cleanup_pending_since=parse_float_or_none( + raw.get("container_cleanup_pending_since") + ), ) records[name] = record if service_type == "backend" and actual_port > 0: diff --git a/python/envctl_engine/runtime/engine_runtime_lifecycle_facade.py b/python/envctl_engine/runtime/engine_runtime_lifecycle_facade.py index 2764266c0..44f0984a1 100644 --- a/python/envctl_engine/runtime/engine_runtime_lifecycle_facade.py +++ b/python/envctl_engine/runtime/engine_runtime_lifecycle_facade.py @@ -27,8 +27,8 @@ class RuntimeLifecycleFacadeMixin: def _release_port_session(self) -> None: runtime_release_port_session(self) - def _clear_runtime_state(self, *, command: str, aggressive: bool = False, route: Route | None = None) -> None: - self.lifecycle_cleanup_orchestrator.clear_runtime_state( + def _clear_runtime_state(self, *, command: str, aggressive: bool = False, route: Route | None = None) -> bool: + return self.lifecycle_cleanup_orchestrator.clear_runtime_state( command=command, aggressive=aggressive, route=route, @@ -54,8 +54,8 @@ def _blast_all_kill_orchestrator_processes(self) -> None: def _blast_all_docker_cleanup(self, *, route: Route | None) -> int: return self.lifecycle_cleanup_orchestrator.blast_all_docker_cleanup(route=route) - def _terminate_started_services(self, services: dict[str, object]) -> None: - runtime_terminate_started_services(self, services) + def _terminate_started_services(self, services: dict[str, object]) -> set[str]: + return runtime_terminate_started_services(self, services) def _terminate_services_from_state( self, @@ -64,8 +64,8 @@ def _terminate_services_from_state( selected_services: set[str] | None, aggressive: bool, verify_ownership: bool, - ) -> None: - runtime_terminate_services_from_state( + ) -> set[str]: + return runtime_terminate_services_from_state( self, state, selected_services=selected_services, diff --git a/python/envctl_engine/runtime/engine_runtime_planning_facade.py b/python/envctl_engine/runtime/engine_runtime_planning_facade.py index 46b587fa2..bdc61e245 100644 --- a/python/envctl_engine/runtime/engine_runtime_planning_facade.py +++ b/python/envctl_engine/runtime/engine_runtime_planning_facade.py @@ -16,7 +16,6 @@ _planning_done_root as domain_planning_done_root, _planning_menu_apply_key as domain_planning_menu_apply_key, _planning_root as domain_planning_root, - _plan_selection_memory_path as domain_plan_selection_memory_path, _preferred_tree_root_for_feature as domain_preferred_tree_root_for_feature, _project_sort_key_for_feature as domain_project_sort_key_for_feature, _read_planning_menu_escape_sequence as domain_read_planning_menu_escape_sequence, @@ -54,7 +53,6 @@ class RuntimePlanningFacadeMixin: _resolve_planning_selection_target = domain_resolve_planning_selection_target _planning_root = domain_planning_root _planning_done_root = domain_planning_done_root - _plan_selection_memory_path = domain_plan_selection_memory_path _create_feature_worktrees = domain_create_feature_worktrees _worktree_add_failure = domain_worktree_add_failure _setup_worktree_placeholder_fallback_enabled = domain_setup_worktree_placeholder_fallback_enabled @@ -79,17 +77,6 @@ def _prompt_planning_selection( ) -> dict[str, int] | None: return _planning_orchestrator(self).prompt_planning_selection(planning_files, raw_projects) - def _initial_plan_selected_counts( - self, - *, - planning_files: list[str], - existing_counts: dict[str, int], - ) -> dict[str, int]: - return _planning_orchestrator(self).initial_plan_selected_counts( - planning_files=planning_files, - existing_counts=existing_counts, - ) - def _run_planning_selection_menu( self, *, @@ -103,12 +90,6 @@ def _run_planning_selection_menu( existing_counts=existing_counts, ) - def _load_plan_selection_memory(self) -> dict[str, int]: - return _planning_orchestrator(self).load_plan_selection_memory() - - def _save_plan_selection_memory(self, selected_counts: dict[str, int]) -> None: - _planning_orchestrator(self).save_plan_selection_memory(selected_counts) - def _planning_keep_plan_enabled(self, route: Route) -> bool: return _planning_orchestrator(self).planning_keep_plan_enabled(route) diff --git a/python/envctl_engine/runtime/engine_runtime_service_truth.py b/python/envctl_engine/runtime/engine_runtime_service_truth.py index 1ada806a5..0df85d2d1 100644 --- a/python/envctl_engine/runtime/engine_runtime_service_truth.py +++ b/python/envctl_engine/runtime/engine_runtime_service_truth.py @@ -28,6 +28,7 @@ STARTUP_FAILURE_TOKENS, STARTUP_PROGRESS_TOKENS, command_result_error_text, + service_listener_failure_class, service_listener_failure_detail, tail_log_error_line, tail_log_startup_progress_line, @@ -50,6 +51,7 @@ "rebind_stale_service_pid", "refresh_service_listener_pids", "service_display_name", + "service_listener_failure_class", "service_listener_failure_detail", "service_startup_progress_timeout", "service_truth_discovery", diff --git a/python/envctl_engine/runtime/engine_runtime_startup_facade.py b/python/envctl_engine/runtime/engine_runtime_startup_facade.py index bd84ea62d..889de8dbe 100644 --- a/python/envctl_engine/runtime/engine_runtime_startup_facade.py +++ b/python/envctl_engine/runtime/engine_runtime_startup_facade.py @@ -1,7 +1,8 @@ from __future__ import annotations +from collections.abc import Sequence from pathlib import Path -from typing import Any, Mapping, cast +from typing import Any, Callable, Mapping, cast from envctl_engine.runtime.engine_runtime_artifacts import ( print_summary as runtime_print_summary, @@ -197,8 +198,15 @@ def _start_project_services( route=route, ) - def _write_artifacts(self, state: RunState, contexts: list[Any], *, errors: list[str]) -> None: - runtime_write_artifacts(self, state, contexts, errors=errors) + def _write_artifacts( + self, + state: RunState, + contexts: list[Any], + *, + errors: list[str], + on_commit: Callable[[], None] | None = None, + ) -> None: + runtime_write_artifacts(self, state, contexts, errors=errors, on_commit=on_commit) def _write_runtime_readiness_report( self, @@ -208,10 +216,21 @@ def _write_runtime_readiness_report( ) -> None: runtime_write_runtime_readiness_report(self, run_dir=run_dir, readiness_result=readiness_result) - def _try_load_existing_state(self, *, mode: str | None = None, strict_mode_match: bool = False) -> RunState | None: + def _try_load_existing_state( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, + ) -> RunState | None: return cast( RunState | None, - runtime_try_load_existing_state(self, mode=mode, strict_mode_match=strict_mode_match), + runtime_try_load_existing_state( + self, + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ), ) def _state_matches_scope(self, state: RunState) -> bool: diff --git a/python/envctl_engine/runtime/engine_runtime_startup_support.py b/python/envctl_engine/runtime/engine_runtime_startup_support.py index 9a4abb529..83d471e3e 100644 --- a/python/envctl_engine/runtime/engine_runtime_startup_support.py +++ b/python/envctl_engine/runtime/engine_runtime_startup_support.py @@ -3,6 +3,7 @@ from pathlib import Path from typing import Any, Callable, Mapping +from envctl_engine.requirements.core import dependency_definitions from envctl_engine.runtime.command_router import Route from envctl_engine.startup.run_reuse_support import ( build_startup_identity_metadata as build_startup_identity_metadata_impl, @@ -11,13 +12,21 @@ state_has_resumable_services as state_has_resumable_services_impl, ) from envctl_engine.startup.protocols import ProjectContextLike +from envctl_engine.startup.service_execution_environment import configured_service_types_for_mode +from envctl_engine.startup.service_execution_policy import additional_service_enabled_for_context +from envctl_engine.startup.startup_selection_support import _restart_service_types_for_project from envctl_engine.state.models import PortPlan, RunState from envctl_engine.shared.parsing import parse_bool, parse_int from envctl_engine.planning import discover_tree_projects def effective_start_mode(runtime: Any, route: Route) -> str: - if route.mode == "main" and runtime._setup_worktree_requested(route): + setup_worktree_requested = getattr(runtime, "_setup_worktree_requested", None) + if ( + route.mode == "main" + and callable(setup_worktree_requested) + and setup_worktree_requested(route) + ): return "trees" return route.mode @@ -197,7 +206,21 @@ def discover_projects( def reserve_project_ports(runtime: Any, context: ProjectContextLike, route: Route | None = None) -> None: + selected_app_services = _selected_app_services_for_port_reservation(runtime, context=context, route=route) for service_name, plan in context.ports.items(): + if not _port_reservation_enabled( + runtime, + service_name=service_name, + route=route, + selected_app_services=selected_app_services, + ): + runtime._emit( + "port.reservation.skipped", + project=context.name, + service=service_name, + reason="startup_surface_disabled", + ) + continue owner = f"{context.name}:{service_name}" restart_preferred_port = plan.assigned if getattr(plan, "source", "") == "restart" else None restart_conflict_detail = None @@ -245,6 +268,73 @@ def reserve_project_ports(runtime: Any, context: ProjectContextLike, route: Rout runtime._emit("port.reserved", project=context.name, service=service_name, port=reserved) +def _port_reservation_enabled( + runtime: Any, + *, + service_name: str, + route: Route | None, + selected_app_services: set[str] | None, +) -> bool: + if route is None: + return True + runtime_mode = effective_start_mode(runtime, route) + + dependency_ids = { + definition.id + for definition in dependency_definitions() + if any(resource.legacy_port_key == service_name for resource in definition.resources) + } + if dependency_ids: + if route.flags.get("launch_dependencies") is False: + return False + enabled = getattr(runtime, "_requirement_enabled", None) + if not callable(enabled): + return True + return any( + bool(enabled(dependency_id, mode=runtime_mode, route=route)) + for dependency_id in dependency_ids + ) + + return selected_app_services is None or service_name in selected_app_services + + +def _selected_app_services_for_port_reservation( + runtime: Any, + *, + context: ProjectContextLike, + route: Route | None, +) -> set[str] | None: + if route is None: + return None + runtime_mode = effective_start_mode(runtime, route) + try: + configured_service_types = configured_service_types_for_mode( + runtime, + runtime_mode, + Path(context.root), + ) + except AttributeError: + dependency_port_names = { + resource.legacy_port_key for definition in dependency_definitions() for resource in definition.resources + } + configured_service_types = set(context.ports).difference(dependency_port_names) + configured_additional_services = tuple( + service + for service in getattr(getattr(runtime, "config", None), "additional_services", ()) + if additional_service_enabled_for_context( + service, + mode=runtime_mode, + project_root=Path(context.root), + ) + ) + return _restart_service_types_for_project( + route=route, + project_name=context.name, + default_service_types=configured_service_types, + additional_services=configured_additional_services, + ) + + def _release_restart_preferred_lock(port_planner: Any, port: int, *, owner: str) -> None: release = getattr(port_planner, "release", None) if not callable(release): diff --git a/python/envctl_engine/runtime/engine_runtime_state_lookup.py b/python/envctl_engine/runtime/engine_runtime_state_lookup.py index 6aa0a3713..5f1f82198 100644 --- a/python/envctl_engine/runtime/engine_runtime_state_lookup.py +++ b/python/envctl_engine/runtime/engine_runtime_state_lookup.py @@ -1,15 +1,24 @@ from __future__ import annotations +from collections.abc import Sequence from typing import Any +from envctl_engine.state.lookup import call_state_loader + def try_load_existing_state( runtime: Any, *, mode: str | None = None, strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, ) -> object | None: - state = runtime.state_repository.load_latest(mode=mode, strict_mode_match=strict_mode_match) + state = call_state_loader( + runtime.state_repository.load_latest, + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ) if state is not None: runtime._emit( "state.fingerprint.after_reload", diff --git a/python/envctl_engine/runtime/help_general.py b/python/envctl_engine/runtime/help_general.py index 0e7b00251..7451029a7 100644 --- a/python/envctl_engine/runtime/help_general.py +++ b/python/envctl_engine/runtime/help_general.py @@ -87,6 +87,7 @@ def render_general_help() -> str: " --only-backend launch only backend; skip frontend and dependencies/prep", " --no-deps skip managed dependencies and plan-agent dependency prep", " --no-infra skip backend, frontend, managed dependencies, and plan-agent prep", + " --docker run configured app services as managed Docker containers", "", "Important global flags:", " --headless / --batch do not prompt; use deterministic automation-friendly output", diff --git a/python/envctl_engine/runtime/help_topic_actions.py b/python/envctl_engine/runtime/help_topic_actions.py index 84b358a6c..efd4ac7ec 100644 --- a/python/envctl_engine/runtime/help_topic_actions.py +++ b/python/envctl_engine/runtime/help_topic_actions.py @@ -65,25 +65,27 @@ "pr": CommandHelpTopic( command="pr", summary="create a pull request for the selected branch/worktree", - usage=("envctl pr [--project ] [--pr-base ]",), + usage=("envctl pr [--project |--all --yes] [--pr-base ]",), what_it_does=( "resolves the current branch, checks for an existing PR, and creates one with gh or repo helper scripts", + "defaults the PR base to dev, then main, then master, while --pr-base remains an explicit override", "commits and pushes dirty worktree changes first using envctl commit behavior when needed", "uses MAIN_TASK.md or commit history to build a useful PR title/body unless explicit env is provided", ), flags=( "--project create PR for selected worktree/project", + "--all --yes explicitly approve creating/updating PRs for every discovered target", "--pr-base set the target base branch", "--interactive opt into prompts/selection instead of default headless action mode", ), - examples=("envctl pr --project feature-a-1 --pr-base main", "envctl pr --main --pr-base main"), + examples=("envctl pr --project feature-a-1", "envctl pr --main --pr-base main"), aliases=("prs", "p", "--pr", "--prs"), related=("commit", "review", "test"), ), "commit": CommandHelpTopic( command="commit", summary="commit-only fallback that preserves envctl-local control artifacts", - usage=("envctl commit [--project |--main] [-m |--commit-message-file ]",), + usage=("envctl commit [--project |--main|--all --yes] [-m |--commit-message-file ]",), what_it_does=( "use envctl ship for normal AI handoff; reserve commit for commit-only maintenance or fallback flows", "stages normal changed paths and intentionally skips protected envctl-local artifacts", @@ -96,6 +98,7 @@ "--commit-message-file read commit message from a file", "--project commit inside selected worktree/project", "--main target the main checkout", + "--all --yes explicitly approve committing every discovered target", ), examples=( "envctl commit --main -m 'Ship focused fix'", @@ -107,7 +110,7 @@ "ship": CommandHelpTopic( command="ship", summary="commit, push, create/update PR, and report GitHub checks for the current or selected target", - usage=("envctl ship [--project ] [-m ] [--human]",), + usage=("envctl ship [--project |--all --yes] [-m ] [--human]",), what_it_does=( "when run inside a generated worktree, infers that worktree without requiring --project", "owns the normal AI handoff flow instead of requiring separate commit, push, or PR commands", @@ -123,6 +126,7 @@ ), flags=( "--project ship one worktree/project", + "--all --yes explicitly approve shipping every discovered target", "-m use explicit commit message text for the commit phase", "--json compatibility no-op; JSON is the default", "--human print compact terminal output instead of JSON", @@ -155,7 +159,7 @@ "migrate": CommandHelpTopic( command="migrate", summary="run backend database migrations for selected targets", - usage=("envctl migrate [--project |--main|--all]",), + usage=("envctl migrate [--project |--main|--all --yes]",), what_it_does=( "runs the configured/default backend migration command, usually Alembic upgrade head", "loads backend env files and reuses saved dependency URLs when a current run exists", @@ -164,7 +168,7 @@ flags=( "--project migrate one worktree/project", "--main migrate the main checkout", - "--all migrate all selected/discovered targets where supported", + "--all --yes explicitly approve migrating all discovered targets", ), examples=("envctl migrate --project feature-a-1", "envctl migrate --main"), aliases=("migration", "migrations", "m", "--migrate"), diff --git a/python/envctl_engine/runtime/help_topic_inspection.py b/python/envctl_engine/runtime/help_topic_inspection.py index 696b062bb..a4aa18ad3 100644 --- a/python/envctl_engine/runtime/help_topic_inspection.py +++ b/python/envctl_engine/runtime/help_topic_inspection.py @@ -248,6 +248,8 @@ notes=( "intended for CI on a trusted self-hosted runner with gh, docker, git, and envctl available", "configuration is supplied through ENVCTL_PREVIEW_* environment variables", + "consume this command from CI instead of vendoring a repository-local controller copy", + "feature-branch push events only redeploy matching labeled same-repository PRs", ), aliases=("pr-preview", "github-pr-preview", "--pr-preview-controller", "--pr-preview"), related=("import", "start", "qa-user", "endpoints"), diff --git a/python/envctl_engine/runtime/help_topic_lifecycle.py b/python/envctl_engine/runtime/help_topic_lifecycle.py index 4de800366..5cd01cbaf 100644 --- a/python/envctl_engine/runtime/help_topic_lifecycle.py +++ b/python/envctl_engine/runtime/help_topic_lifecycle.py @@ -26,6 +26,7 @@ "--deps-sequential|--sequential-deps force managed dependencies to start one at a time", "--project limit startup to selected project/worktree when supported", "--no-resume skip automatic resume of a compatible saved run", + "--docker build/use images and run app services in managed containers", ), examples=( "envctl", @@ -33,15 +34,14 @@ "envctl start --trees --entire-system --headless", "envctl start --project feature-a-1 --backend --headless", "envctl --trees --only-backend --headless", + "envctl start --trees --entire-system --docker --headless", ), related=("restart", "resume", "dashboard", "stop", "health"), ), "restart": CommandHelpTopic( command="restart", summary="restart selected envctl-managed services", - usage=( - "envctl restart [--project |--service |--all] [runtime scope] [--headless]", - ), + usage=("envctl restart [--project |--service |--all] [runtime scope] [--headless]",), what_it_does=( "loads saved runtime state, stops the selected services, and starts them again", "keeps the same target model as startup while using existing envctl state as the source of truth", @@ -95,12 +95,17 @@ "stop": CommandHelpTopic( command="stop", summary="stop selected envctl-managed services and update runtime state", - usage=("envctl stop [--project |--service |--all] [runtime scope]",), + usage=("envctl stop [--main|--trees] [--project |--service |--all] [runtime scope]",), what_it_does=( "loads saved runtime state and terminates selected services", - "supports service-scope flags so you can stop backend, frontend, dependencies, or the entire system", + ( + "dependency scope releases saved dependency records and port locks; " + "managed Docker stacks and storage remain reusable" + ), + "use stop-all --remove-volumes or blast-all when managed Docker containers must be removed", ), flags=( + "--main | --trees choose the saved runtime mode; an explicit mode leaves the other mode active", "--project stop services for selected project/worktree", "--service stop one saved service", "--all stop all matching saved services", @@ -117,14 +122,19 @@ "stop-all": CommandHelpTopic( command="stop-all", summary="stop every saved envctl-managed service in the selected scope", - usage=("envctl stop-all [runtime scope]", "envctl kill-all [runtime scope]"), + usage=( + "envctl stop-all [--main|--trees] [runtime scope]", + "envctl kill-all [--main|--trees] [runtime scope]", + ), what_it_does=( - "terminates all saved services for the selected runtime state", - "can optionally remove volumes when supported by the selected cleanup flags", + "terminates all saved app services and releases dependency records and port locks", + "keeps managed Docker dependency stacks and storage reusable unless explicit removal cleanup is requested", + "use --remove-volumes for supported Docker removal, or blast-all for aggressive cleanup", ), flags=( + "--main | --trees choose the saved runtime mode; an explicit mode leaves the other mode active", "--backend|--frontend|--fullstack|--dependencies|--entire-system choose cleanup scope", - "--stop-all-remove-volumes / --remove-volumes remove supported volumes too", + "--stop-all-remove-volumes / --remove-volumes remove supported managed containers and volumes", ), examples=("envctl stop-all", "envctl kill-all", "envctl stop-all --entire-system --remove-volumes"), aliases=("kill-all", "killall", "stopall", "--stop-all", "--kill-all"), diff --git a/python/envctl_engine/runtime/inspection_support.py b/python/envctl_engine/runtime/inspection_support.py index a83f6c337..ec35ac4ad 100644 --- a/python/envctl_engine/runtime/inspection_support.py +++ b/python/envctl_engine/runtime/inspection_support.py @@ -10,6 +10,12 @@ from envctl_engine.config.persistence import managed_values_from_local_state, managed_values_to_payload from envctl_engine.planning import PlanProjectPrediction from envctl_engine.runtime.command_router import Route, list_supported_commands +from envctl_engine.shared.services import resolve_service_project_name +from envctl_engine.runtime.state_inspection_support import ( + emit_cwd_runtime_mismatch_warnings as _emit_cwd_runtime_mismatch_warnings_impl, + print_state as _print_state_impl, + try_load_existing_state as _try_load_existing_state_impl, +) from envctl_engine.startup.startup_selection_support import ( _state_project_names as _state_project_names_impl, _tree_preselected_projects_from_state as _tree_preselected_projects_from_state_impl, @@ -20,15 +26,7 @@ print_attach_command, kill_session, ) -from envctl_engine.state import state_to_dict from envctl_engine.state.models import RunState -from envctl_engine.state.project_runtime import ( - cwd_runtime_warnings, - dependency_mode_summary, - project_resolution_event_payload, - resolve_requested_project_state, -) -from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.runtime.endpoints_command_support import run_endpoints_command from envctl_engine.runtime.startup_inspection_support import ( additional_service_contexts as _additional_service_contexts_impl, @@ -87,14 +85,35 @@ def _print_targets(runtime: Any, route: Route, *, trees_only: bool) -> int: preselected: set[str] = set() state_present: set[str] = set() services_running: set[str] = set() + state_error: dict[str, str] | None = None if mode == "trees": - startup = getattr(runtime, "startup_orchestrator", None) - if startup is not None: - preselected = set(_tree_preselected_projects_from_state_impl(runtime=runtime, project_contexts=projects)) - state = _try_load_existing_state(runtime, mode="trees", strict_mode_match=True) - if state is not None: - state_present = set(_state_project_names_impl(runtime=runtime, state=state)) - services_running = _running_tree_project_names(runtime=runtime, state=state) + states, load_error = _load_all_existing_states(runtime, mode="trees") + if load_error is not None: + state_error = { + "code": "runtime_state_integrity_error", + "message": load_error, + } + else: + startup = getattr(runtime, "startup_orchestrator", None) + if startup is not None: + try: + preselected = set( + _tree_preselected_projects_from_state_impl(runtime=runtime, project_contexts=projects) + ) + except RuntimeError as exc: + # The selection helper performs its own strict state load. If + # the repository changes between the two reads, keep this + # inspection command useful without presenting partial state + # as trustworthy. + states = [] + preselected.clear() + state_error = { + "code": "runtime_state_integrity_error", + "message": str(exc), + } + for state in states: + state_present.update(_state_project_names_impl(runtime=runtime, state=state)) + services_running.update(_running_tree_project_names(runtime=runtime, state=state)) payload = { "mode": mode, @@ -112,6 +131,8 @@ def _print_targets(runtime: Any, route: Route, *, trees_only: bool) -> int: "running": project.name in services_running, }, ), + "state_error": state_error, + "warnings": [state_error] if state_error is not None else [], } print(json.dumps(payload, indent=2, sort_keys=True)) return 0 @@ -134,7 +155,11 @@ def _running_tree_project_names(*, runtime: Any, state: RunState) -> set[str]: if not isinstance(services, dict): return running for service_name, record in services.items(): - project_name = runtime._project_name_from_service(str(service_name)) + project_name = resolve_service_project_name( + str(service_name), + record, + project_name_from_service=runtime._project_name_from_service, + ) normalized = str(project_name).strip() if not normalized: continue @@ -317,87 +342,42 @@ def _apply_runtime_effective_config_values(values: Any, config: Any) -> None: def _print_state(runtime: Any, route: Route, *, json_output: bool) -> int: - strict = runtime._state_lookup_strict_mode_match(route) - state = _try_load_existing_state(runtime, mode=route.mode, strict_mode_match=strict) - if state is None: - if json_output: - print(json.dumps({"found": False, "mode": getattr(route, "mode", None)}, indent=2, sort_keys=True)) - else: - print("No previous state found.") - return 1 - requested_projects = list(getattr(route, "projects", []) or []) - if requested_projects: - resolution = resolve_requested_project_state( - state, - requested_projects, - command="show-state", - runtime=runtime, - allow_multi=False, - ) - if not resolution.ok: - emitter = getattr(runtime, "_emit", None) - if callable(emitter): - emitter( - "state.project_resolution.failed", - **project_resolution_event_payload(resolution, state, runtime=runtime), - ) - if json_output: - print(json.dumps(resolution.payload(), indent=2, sort_keys=True)) - else: - print(f"Requested project is not running: {resolution.requested_project}") - print("Active runtime projects: " + (", ".join(resolution.active_projects) or "none")) - return 1 - emitter = getattr(runtime, "_emit", None) - if callable(emitter): - emitter( - "state.project_resolution.ok", - **project_resolution_event_payload(resolution, state, runtime=runtime), - ) - if resolution.state is not None and not bool(getattr(route, "flags", {}).get("all")): - state = resolution.state - - dependency_summary = dependency_mode_summary(state) - cwd_project, warnings = cwd_runtime_warnings( - state, - runtime=runtime, - requested_projects=list(getattr(route, "projects", []) or []), + return _print_state_impl( + runtime, + route, + json_output=json_output, + state_loader=_try_load_existing_state, + warning_emitter=_emit_cwd_runtime_mismatch_warnings, ) - _emit_cwd_runtime_mismatch_warnings(runtime, command="show-state", state=state, warnings=warnings) - payload: dict[str, Any] = { - "found": True, - "runtime_root": str(runtime.runtime_root), - "run_state_path": str(runtime._run_state_path()), - "state": state_to_dict(state), - "runtime_map": build_runtime_map(state), - "dependency_mode": dependency_summary["dependency_mode"], - "shared_dependencies": dependency_summary["shared_dependencies"], - "cwd_project": cwd_project, - "warnings": warnings, - } - if json_output: - print(json.dumps(payload, indent=2, sort_keys=True)) - return 0 - - state_payload = payload["state"] - print(f"run_id: {state_payload['run_id']}") - print(f"mode: {state_payload['mode']}") - print(f"services: {len(state_payload['services'])}") - print(f"requirements: {len(state_payload['requirements'])}") - for warning in warnings: - print(f"Warning: {warning['message']}") - print("run_state_path:") - print(render_path_for_terminal(payload["run_state_path"], env=getattr(runtime, "env", {}))) - return 0 +def _try_load_existing_state( + runtime: Any, + *, + mode: str | None, + strict_mode_match: bool, + project_names: list[str] | None = None, +) -> RunState | None: + return _try_load_existing_state_impl( + runtime, + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ) -def _try_load_existing_state(runtime: Any, *, mode: str | None, strict_mode_match: bool) -> RunState | None: - loader = getattr(runtime, "_try_load_existing_state", None) - if not callable(loader): - return None - state = loader(mode=mode, strict_mode_match=strict_mode_match) - return state if isinstance(state, RunState) else None +def _load_all_existing_states(runtime: Any, *, mode: str) -> tuple[list[RunState], str | None]: + repository = getattr(runtime, "state_repository", None) + load_all = getattr(repository, "load_all", None) + try: + if callable(load_all): + states = [state for state in load_all(mode=mode) if isinstance(state, RunState)] + if states: + return states, None + state = _try_load_existing_state(runtime, mode=mode, strict_mode_match=True) + except RuntimeError as exc: + return [], str(exc) + return ([state] if state is not None else []), None def _emit_cwd_runtime_mismatch_warnings( @@ -407,20 +387,12 @@ def _emit_cwd_runtime_mismatch_warnings( state: RunState, warnings: list[dict[str, object]], ) -> None: - emitter = getattr(runtime, "_emit", None) - if not callable(emitter): - return - for warning in warnings: - if str(warning.get("code") or "") != "cwd_runtime_mismatch": - continue - emitter( - "state.cwd_runtime_mismatch", - run_id=getattr(state, "run_id", ""), - mode=getattr(state, "mode", ""), - command=command, - cwd_project=warning.get("cwd_project"), - active_projects=warning.get("active_projects"), - ) + _emit_cwd_runtime_mismatch_warnings_impl( + runtime, + command=command, + state=state, + warnings=warnings, + ) def _additional_service_enabled_for_context(runtime: Any, mode: str, service: object, context: object) -> bool: diff --git a/python/envctl_engine/runtime/lifecycle_blast_docker.py b/python/envctl_engine/runtime/lifecycle_blast_docker.py index 73997ac0a..ab62412c1 100644 --- a/python/envctl_engine/runtime/lifecycle_blast_docker.py +++ b/python/envctl_engine/runtime/lifecycle_blast_docker.py @@ -10,7 +10,10 @@ def matches_blast_container(*, image: str, name: str) -> bool: image_l = image.lower() name_l = name.lower() - return any(token in name_l or token in image_l for token in ("supabase", "n8n", "redis", "postgres")) + return any( + token in name_l or token in image_l + for token in ("envctl-app-", "supabase", "n8n", "redis", "postgres") + ) class BlastDockerCleanupSupport: diff --git a/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py b/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py index b2680e985..395f3655b 100644 --- a/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py +++ b/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py @@ -3,18 +3,24 @@ from collections.abc import Iterable from typing import Any, Callable, Mapping, cast -from envctl_engine.runtime.lifecycle_blast_support import LifecycleBlastCleanupSupport +from envctl_engine.requirements.core import dependency_definitions from envctl_engine.runtime.engine_runtime_lifecycle_support import release_requirement_ports +from envctl_engine.runtime.lifecycle_blast_support import LifecycleBlastCleanupSupport from envctl_engine.runtime.lifecycle_dependency_stop import ( release_requirement_component_ports, release_selected_dependency_components, requirements_have_enabled_components, select_dependency_components_for_stop, + stop_requirement_component_containers, ) +from envctl_engine.runtime.lifecycle_requirement_ports import requirement_component_port_owners +from envctl_engine.runtime.lifecycle_full_cleanup import clear_runtime_state as clear_full_runtime_state from envctl_engine.runtime.lifecycle_service_stop_selection import ( + project_selectors_from_route, select_services_for_stop, service_matches_runtime_scope, ) +from envctl_engine.runtime.lifecycle_service_termination import unconfirmed_service_names from envctl_engine.runtime.lifecycle_stopped_metadata import ( has_dashboard_stopped_services, project_name_from_stopped_service, @@ -23,14 +29,16 @@ service_type_from_stopped_service, should_preserve_stopped_dashboard_state, ) +from envctl_engine.runtime.lifecycle_targeted_state import LifecycleTargetedStateSupport from envctl_engine.runtime.runtime_context import ( resolve_port_allocator, resolve_process_runtime, resolve_state_repository, ) from envctl_engine.shared.protocols import PortAllocator, ProcessRuntime, StateRepository -from envctl_engine.state.runtime_map import build_runtime_map +from envctl_engine.shared.services import service_matches_selector from envctl_engine.runtime.command_router import Route +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RequirementsResult, RunState from envctl_engine.ui.selection_support import ( interactive_selection_allowed, @@ -43,7 +51,10 @@ from envctl_engine.ui.spinner_service import emit_spinner_policy, resolve_spinner_policy -class LifecycleCleanupOrchestrator(LifecycleBlastCleanupSupport): +class LifecycleCleanupOrchestrator( + LifecycleTargetedStateSupport, + LifecycleBlastCleanupSupport, +): def __init__(self, runtime: Any) -> None: self.runtime = runtime @@ -54,13 +65,16 @@ def execute(self, route: Route) -> int: def _execute_full_cleanup(self, route: Route) -> int: def operation() -> int: - self.clear_runtime_state( + completed = self.clear_runtime_state( command=route.command, aggressive=(route.command == "blast-all"), route=route, ) - print("Stopped runtime state.") - return 0 + if completed: + print("Stopped runtime state.") + return 0 + print("Runtime cleanup is incomplete; failed resources remain tracked.") + return 1 message = ( "Stopping all services and runtime state..." @@ -75,77 +89,210 @@ def operation() -> int: ) def _execute_stop(self, route: Route) -> int: - state = self.runtime._try_load_existing_state( # type: ignore[attr-defined] + targeted_states = self._targeted_active_states(route) + if targeted_states: + result = 0 + for state in targeted_states: + result = max(result, self._execute_stop_state(route, state)) + return result + + state = call_state_loader( + self.runtime._try_load_existing_state, # type: ignore[attr-defined] mode=route.mode, strict_mode_match=self.runtime._state_lookup_strict_mode_match(route), # type: ignore[attr-defined] + project_names=route.projects or None, ) if state is None: print("No active runtime state found.") return 0 + return self._execute_stop_state(route, state) + + def _targeted_active_states(self, route: Route) -> list[RunState]: + selected_projects = {str(project).strip().casefold() for project in route.projects if str(project).strip()} + if not selected_projects: + return [] + repository = self._state_repository(self.runtime) + load_all = getattr(repository, "load_all", None) + if not callable(load_all): + return [] + matching_states = self._states_matching_projects( + [state for state in load_all(mode=route.mode) if isinstance(state, RunState)], + selected_projects, + ) + if matching_states: + return matching_states + if self.runtime._state_lookup_strict_mode_match(route): # type: ignore[attr-defined] + return [] + + fallback_state = call_state_loader( + self.runtime._try_load_existing_state, # type: ignore[attr-defined] + mode=route.mode, + strict_mode_match=False, + project_names=route.projects, + ) + if not isinstance(fallback_state, RunState) or fallback_state.mode == route.mode: + return [] + return self._states_matching_projects( + [state for state in load_all(mode=fallback_state.mode) if isinstance(state, RunState)], + selected_projects, + ) + + def _states_matching_projects( + self, + states: Iterable[RunState], + selected_projects: set[str], + ) -> list[RunState]: + return [ + state + for state in states + if selected_projects.intersection( + str(getattr(project, "name", project)).strip().casefold() + for project in self._project_names_from_state(state) + ) + ] + + def _execute_stop_state(self, route: Route, state: RunState) -> int: + if route.flags.get("runtime_scope") == "dependencies": return self._execute_stop_dependencies(route, state) + owned_projects = self._owned_project_names(state) + dependency_project_keys, has_dependency_project_scope = self._dependency_scope_project_keys(state, route) + if route.flags.get("runtime_scope") == "entire-system" and has_dependency_project_scope: + targeted_project_keys = frozenset( + project for project in dependency_project_keys if project in owned_projects + ) + else: + targeted_project_keys = frozenset( + str(project).strip().casefold() + for project in route.projects + if str(project).strip().casefold() in owned_projects + ) preserve_stopped_dashboard_state = self._should_preserve_stopped_dashboard_state(route) - selected_dependencies = self._select_dependency_components_for_stop(state, route) + selected_dependencies = self._select_dependency_components_for_stop( + state, + route, + requested_projects=( + dependency_project_keys + if route.flags.get("runtime_scope") == "entire-system" and has_dependency_project_scope + else None + ), + ) selected = self._select_services_for_stop(state, route) if not selected and not selected_dependencies: - if not state.services and not state.requirements: - self.clear_runtime_state(command="stop", aggressive=False) + if targeted_project_keys and self._targeted_projects_have_no_managed_resources( + state, + targeted_project_keys, + ): + self._remove_targeted_requirements(state, targeted_project_keys) + state_remains = self._persist_targeted_project_state( + state, + owned_projects=owned_projects, + targeted_project_keys=targeted_project_keys, + ) + print("Stopped selected project state." if state_remains else "Stopped runtime state.") + return 0 + has_enabled_requirements = any( + self._requirements_have_enabled_components(requirements) for requirements in state.requirements.values() + ) + if not state.services and not has_enabled_requirements: + self._deactivate_runtime_state(state) print("Stopped runtime state.") return 0 print("No matching services selected for stop.") return 1 def operation() -> int: - if selected and preserve_stopped_dashboard_state: - self._remember_dashboard_stopped_services(state, selected) - + self.runtime._emit( # type: ignore[attr-defined] + "cleanup.stop", + aggressive=False, + run_id=state.run_id, + ) + failed_services: set[str] = set() if selected: - self.runtime._terminate_services_from_state( # type: ignore[attr-defined] + termination_result = self.runtime._terminate_services_from_state( # type: ignore[attr-defined] state, selected_services=selected, aggressive=False, verify_ownership=True, ) + failed_services = unconfirmed_service_names(termination_result, selected) + + stopped_services = selected.difference(failed_services) + if stopped_services and preserve_stopped_dashboard_state: + self._remember_dashboard_stopped_services(state, stopped_services) - for service_name in list(selected): + for service_name in stopped_services: state.services.pop(service_name, None) - if selected_dependencies: + if selected_dependencies and not failed_services: self._release_selected_dependency_components(state, selected_dependencies) - if not bool(route.flags.get("stop_preserve_requirements")): + if not failed_services and not bool(route.flags.get("stop_preserve_requirements")): remaining_projects = { - self.runtime._project_name_from_service(name) # type: ignore[attr-defined] - for name in state.services + self._project_name_for_service(name, service) for name, service in state.services.items() } - removed_projects = [project for project in state.requirements if project not in remaining_projects] + removed_projects = [ + project + for project, requirements in state.requirements.items() + if self._requirement_project_key(project, requirements) not in remaining_projects + and ( + not targeted_project_keys + or self._requirement_project_key(project, requirements) in targeted_project_keys + ) + ] for project in removed_projects: self.runtime._release_requirement_ports(state.requirements[project]) # type: ignore[attr-defined] state.requirements.pop(project, None) + if targeted_project_keys: + self._persist_targeted_project_state( + state, + owned_projects=owned_projects, + targeted_project_keys=targeted_project_keys, + ) + if failed_services: + print("Could not safely stop: " + ", ".join(sorted(failed_services))) + return 1 + if stopped_services and selected_dependencies: + print("Stopped selected services and dependencies.") + elif selected_dependencies: + print("Stopped selected dependencies.") + elif stopped_services: + print("Stopped selected services.") + else: + print("Stopped selected project state.") + return 0 + if not state.services and not state.requirements: if preserve_stopped_dashboard_state and self._has_dashboard_stopped_services(state): self._save_selected_stop_state(state) - if selected and selected_dependencies: + if stopped_services and selected_dependencies: print("Stopped selected services and dependencies.") elif selected_dependencies: print("Stopped selected dependencies.") - else: + elif stopped_services: print("Stopped selected services.") + else: + print("No selected service could be stopped.") return 0 - self.clear_runtime_state(command="stop", aggressive=False) + self._deactivate_runtime_state(state) print("Stopped runtime state.") return 0 self._save_selected_stop_state(state) - if selected and selected_dependencies: + if failed_services: + print("Could not safely stop: " + ", ".join(sorted(failed_services))) + return 1 + if stopped_services and selected_dependencies: print("Stopped selected services and dependencies.") elif selected_dependencies: print("Stopped selected dependencies.") - else: + elif stopped_services: print("Stopped selected services.") + else: + print("Stopped selected dependencies.") return 0 selected_count = len(selected) + sum(len(items) for items in selected_dependencies.values()) @@ -188,18 +335,6 @@ def _project_root_from_stopped_service(service: object, *, service_type: str) -> return project_root_from_stopped_service(service, service_type=service_type) def _select_services_for_stop(self, state: RunState, route: Route) -> set[str]: - selectors_from_passthrough = getattr(self.runtime, "_selectors_from_passthrough", None) - - def selectors_from_passthrough_args(args: list[str]) -> Iterable[str]: - if not callable(selectors_from_passthrough): - return set() - result = selectors_from_passthrough(args) - if isinstance(result, str): - return {result} - if isinstance(result, Iterable): - return cast(Iterable[str], result) - return set() - def selected_services_from_ui_selection(selection: object, selected_state: RunState) -> set[str]: return self._services_from_selection(cast(TargetSelection, selection), selected_state) @@ -209,29 +344,88 @@ def selected_services_from_ui_selection(selection: object, selected_state: RunSt project_name_from_service_fn=lambda service_name: self.runtime._project_name_from_service( # type: ignore[attr-defined] service_name ), - selectors_from_passthrough_fn=selectors_from_passthrough_args, + selectors_from_passthrough_fn=self._passthrough_project_selectors, interactive_stop_selection_fn=self._interactive_stop_selection, services_from_selection_fn=selected_services_from_ui_selection, ) + def _passthrough_project_selectors(self, args: list[str]) -> set[str]: + selectors_from_passthrough = getattr(self.runtime, "_selectors_from_passthrough", None) + if callable(selectors_from_passthrough): + result = selectors_from_passthrough(args) + if isinstance(result, str): + return {result} + if isinstance(result, Iterable): + return {str(item).strip().casefold() for item in result if str(item).strip()} + selectors: set[str] = set() + for token in args: + if str(token).startswith("-"): + continue + selectors.update(part.strip().casefold() for part in str(token).split(",") if part.strip()) + return selectors + + def _dependency_scope_project_keys(self, state: RunState, route: Route) -> tuple[frozenset[str], bool]: + project_keys = project_selectors_from_route( + route, + selectors_from_passthrough_fn=self._passthrough_project_selectors, + ) + services_flag = route.flags.get("services") + has_selectors = bool(project_keys) + if isinstance(services_flag, list): + service_selectors = [str(selector).strip() for selector in services_flag if str(selector).strip()] + has_selectors = has_selectors or bool(service_selectors) + for selector in service_selectors: + for name, service in state.services.items(): + if name.casefold() == selector.casefold() or service_matches_selector(service, selector): + project = self._project_name_for_service(name, service) + if project: + project_keys.add(project) + return frozenset(project_keys), has_selectors + def _execute_stop_dependencies(self, route: Route, state: RunState) -> int: def operation() -> int: if not state.requirements: if not state.services: - self._purge_runtime_state(aggressive=False) + self._deactivate_runtime_state(state) print("No dependencies found.") return 0 - for project in list(state.requirements): - self._release_requirement_ports(state.requirements[project]) - state.requirements.pop(project, None) - - if state.services: + targeted_project_keys, has_targeted_scope = self._dependency_scope_project_keys(state, route) + owned_projects = self._owned_project_names(state) + matching_requirement_keys = { + storage_key + for storage_key, requirements in state.requirements.items() + if self._requirement_project_key(storage_key, requirements) in targeted_project_keys + } + if has_targeted_scope and not matching_requirement_keys: + print("No matching dependencies found.") + return 1 + removed_count = 0 + for storage_key in list(state.requirements): + requirements = state.requirements[storage_key] + requirement_project = self._requirement_project_key(storage_key, requirements) + if has_targeted_scope and requirement_project not in targeted_project_keys: + continue + for definition in dependency_definitions(): + component = requirements.component(definition.id) + if bool(component.get("enabled", False)): + self._stop_requirement_component_containers(component) + self._release_requirement_ports(requirements) + state.requirements.pop(storage_key, None) + removed_count += 1 + + if has_targeted_scope: + self._persist_targeted_project_state( + state, + owned_projects=owned_projects, + targeted_project_keys=targeted_project_keys, + ) + elif state.services or state.requirements: self._save_selected_stop_state(state) else: - self._purge_runtime_state(aggressive=False) + self._deactivate_runtime_state(state) - print("Stopped dependencies.") + print("Stopped dependencies." if removed_count else "No matching dependencies found.") return 0 return self._run_spinner_operation( @@ -241,8 +435,18 @@ def operation() -> int: operation=operation, ) - def _select_dependency_components_for_stop(self, state: RunState, route: Route) -> dict[str, set[str]]: - return select_dependency_components_for_stop(state, route) + def _select_dependency_components_for_stop( + self, + state: RunState, + route: Route, + *, + requested_projects: Iterable[str] | None = None, + ) -> dict[str, set[str]]: + return select_dependency_components_for_stop( + state, + route, + requested_projects=requested_projects, + ) def _release_selected_dependency_components( self, @@ -253,11 +457,55 @@ def _release_selected_dependency_components( state, selected_dependencies, release_component_ports_fn=self._release_requirement_component_ports, + stop_component_fn=self._stop_requirement_component_containers, ) - def _release_requirement_component_ports(self, component: Mapping[str, object]) -> None: + def _stop_requirement_component_containers(self, component: Mapping[str, object]) -> None: + stop_requirement_component_containers(self.runtime, component) + + def _stop_requirement_component_containers_best_effort(self, component: Mapping[str, object]) -> bool: + try: + self._stop_requirement_component_containers(component) + return True + except Exception as exc: # noqa: BLE001 + component_id = str(component.get("id") or "dependency").strip() or "dependency" + detail = str(exc).strip() or exc.__class__.__name__ + self.runtime._emit( # type: ignore[attr-defined] + "cleanup.dependency_container.warning", + component=component_id, + detail=detail, + ) + print(f"Warning: could not stop {component_id} container during cleanup: {detail}") + return False + + def _stop_requirement_containers( + self, + requirements: RequirementsResult, + *, + best_effort: bool = False, + ) -> bool: + stopped = True + for definition in dependency_definitions(): + component = requirements.component(definition.id) + if bool(component.get("enabled", False)): + if best_effort: + stopped = self._stop_requirement_component_containers_best_effort(component) and stopped + else: + self._stop_requirement_component_containers(component) + return stopped + + def _release_requirement_component_ports( + self, + requirements: RequirementsResult, + dependency_id: str, + component: Mapping[str, object], + ) -> None: port_allocator = self._port_allocator(self.runtime) - release_requirement_component_ports(component, release_port_fn=port_allocator.release) + release_requirement_component_ports( + component, + port_planner=port_allocator, + owner_candidates=requirement_component_port_owners(requirements, dependency_id), + ) @staticmethod def _requirements_have_enabled_components(requirements: object) -> bool: @@ -270,6 +518,21 @@ def _release_requirement_ports(self, requirements: RequirementsResult) -> None: return release_requirement_ports(self.runtime, requirements) + def _release_requirement_ports_best_effort(self, requirements: RequirementsResult) -> bool: + try: + self._release_requirement_ports(requirements) + return True + except Exception as exc: # noqa: BLE001 + project = str(getattr(requirements, "project", "") or "dependency").strip() or "dependency" + detail = str(exc).strip() or exc.__class__.__name__ + self.runtime._emit( # type: ignore[attr-defined] + "cleanup.requirement_ports.warning", + project=project, + detail=detail, + ) + print(f"Warning: could not release {project} dependency ports during cleanup: {detail}") + return False + @staticmethod def _service_matches_runtime_scope(name: str, service: object, runtime_scope: str) -> bool: return service_matches_runtime_scope(name, service, runtime_scope) @@ -296,61 +559,71 @@ def _project_names_from_state(self, state: RunState) -> list[object]: def _services_from_selection(self, selection: TargetSelection, state: RunState) -> set[str]: return services_from_selection(self.runtime, selection, state) - def clear_runtime_state(self, *, command: str, aggressive: bool = False, route: Route | None = None) -> None: - rt = self.runtime - if command == "stop-all": - rt._emit("cleanup.stop_all", aggressive=aggressive) # type: ignore[attr-defined] - elif aggressive: - rt._emit("cleanup.blast", aggressive=aggressive) # type: ignore[attr-defined] - else: - rt._emit("cleanup.stop", aggressive=aggressive) # type: ignore[attr-defined] - state = rt._try_load_existing_state() # type: ignore[attr-defined] - if state is not None: - rt._terminate_services_from_state( # type: ignore[attr-defined] - state, - selected_services=None, - aggressive=aggressive, - verify_ownership=not aggressive, - ) - - if aggressive and self.blast_all_ecosystem_enabled(): - self.blast_all_ecosystem_cleanup(route=route) - elif command == "stop-all" and route is not None and bool(route.flags.get("stop_all_remove_volumes")): - rt._emit("cleanup.stop_all.remove_volumes", requested=True) # type: ignore[attr-defined] - volume_route = Route( - command=route.command, - mode=route.mode, - raw_args=route.raw_args, - passthrough_args=route.passthrough_args, - projects=route.projects, - flags={ - **route.flags, - "blast_keep_worktree_volumes": False, - "blast_remove_main_volumes": True, - }, - ) - removed = self.blast_all_docker_cleanup(route=volume_route) - rt._emit("cleanup.stop_all.remove_volumes.finish", removed=removed) # type: ignore[attr-defined] - - self._purge_runtime_state(aggressive=aggressive) - if aggressive: - self.blast_all_purge_legacy_state_artifacts() - - def _save_selected_stop_state(self, state: RunState) -> None: - self._state_repository(self.runtime).save_selected_stop_state( - state=state, - emit=self.runtime._emit, # type: ignore[attr-defined] - runtime_map_builder=lambda raw_state: build_runtime_map(cast(RunState, raw_state)), + def clear_runtime_state( + self, + *, + command: str, + aggressive: bool = False, + route: Route | None = None, + ) -> bool: + return clear_full_runtime_state( + self, + command=command, + aggressive=aggressive, + route=route, ) + @staticmethod + def _has_active_runs(repository: object) -> bool: + has_active_runs = getattr(repository, "has_active_runs", None) + if callable(has_active_runs): + return bool(has_active_runs()) + load_all = getattr(repository, "load_all", None) + return bool(load_all()) if callable(load_all) else True + def _purge_runtime_state(self, *, aggressive: bool) -> None: - if aggressive: - self.release_all_runtime_ports() - else: - release_port_session = getattr(self.runtime, "_release_port_session", None) - if callable(release_port_session): - release_port_session() self._state_repository(self.runtime).purge(aggressive=aggressive) + # Purging means no tracked run remains in this repository scope. Port + # locks were created by the original startup process (and therefore a + # different PortPlanner session), so releasing only this command's + # session leaves unused/disabled stack locks behind indefinitely. The + # registry commit happens first so a failed purge cannot erase locks + # that still protect an authoritative active state. + self.release_all_runtime_ports() + + def _deactivate_runtime_state( + self, + state: RunState, + *, + release_session_when_empty: bool = True, + ) -> None: + repository = self._state_repository(self.runtime) + source_run_ids = [ + str(run_id).strip() + for run_id in ( + state.metadata.get("state_source_run_ids") + if isinstance(state.metadata.get("state_source_run_ids"), list) + else [] + ) + if str(run_id).strip() + ] + run_ids = sorted({state.run_id, *source_run_ids}) + deactivate_many = getattr(repository, "deactivate_runs", None) + deactivate = getattr(repository, "deactivate_run", None) + if callable(deactivate_many) or callable(deactivate): + if callable(deactivate_many): + deactivate_many(run_ids) + elif callable(deactivate): + for run_id in run_ids: + deactivate(run_id) + if release_session_when_empty and not self._has_active_runs(repository): + # The final run may have been started by an earlier process and + # therefore a different planner session. At this point the + # repository proves the whole scope inactive, so every scoped + # lock is stale ownership rather than another live run's lock. + self.release_all_runtime_ports() + return + self._purge_runtime_state(aggressive=False) def _run_spinner_operation( self, diff --git a/python/envctl_engine/runtime/lifecycle_dependency_stop.py b/python/envctl_engine/runtime/lifecycle_dependency_stop.py index aedc97432..acbb4455a 100644 --- a/python/envctl_engine/runtime/lifecycle_dependency_stop.py +++ b/python/envctl_engine/runtime/lifecycle_dependency_stop.py @@ -1,35 +1,63 @@ from __future__ import annotations -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterable, Mapping +from typing import Any from envctl_engine.requirements.core import dependency_definitions from envctl_engine.runtime.command_router import Route -from envctl_engine.runtime.lifecycle_requirement_ports import component_port_values -from envctl_engine.state.models import RunState +from envctl_engine.runtime.lifecycle_requirement_ports import release_requirement_component_port_values +from envctl_engine.state.models import RequirementsResult, RunState -def select_dependency_components_for_stop(state: RunState, route: Route) -> dict[str, set[str]]: +def select_dependency_components_for_stop( + state: RunState, + route: Route, + *, + requested_projects: Iterable[str] | None = None, +) -> dict[str, set[str]]: + if route.flags.get("runtime_scope") == "entire-system": + explicit_project_scope = requested_projects is not None + normalized_projects = { + str(project).strip().casefold() + for project in (route.projects if requested_projects is None else requested_projects) + if str(project).strip() + } + return { + project_name: { + definition.id + for definition in dependency_definitions() + if bool(requirements.component(definition.id).get("enabled", False)) + } + for project_name, requirements in state.requirements.items() + if (not explicit_project_scope and not normalized_projects) + or str(requirements.project or project_name).strip().casefold() in normalized_projects + } raw_components = route.flags.get("stop_dependency_components") if not isinstance(raw_components, list): return {} known_definitions = {definition.id for definition in dependency_definitions()} selected: dict[str, set[str]] = {} - project_key_by_lower = {str(project).strip().casefold(): project for project in state.requirements} + project_keys_by_lower: dict[str, list[str]] = {} + for storage_key, requirements in state.requirements.items(): + project = str(requirements.project or storage_key).strip().casefold() + if project: + project_keys_by_lower.setdefault(project, []).append(storage_key) for raw in raw_components: project_name, separator, dependency_id = str(raw).partition(":") if not separator: continue - project_key = project_key_by_lower.get(project_name.strip().casefold()) - if project_key is None: + project_keys = project_keys_by_lower.get(project_name.strip().casefold(), []) + if not project_keys: continue normalized_dependency = dependency_id.strip().lower() if normalized_dependency not in known_definitions: continue - component = state.requirements[project_key].component(normalized_dependency) - if not bool(component.get("enabled", False)): - continue - selected.setdefault(project_key, set()).add(normalized_dependency) + for project_key in project_keys: + component = state.requirements[project_key].component(normalized_dependency) + if not bool(component.get("enabled", False)): + continue + selected.setdefault(project_key, set()).add(normalized_dependency) return selected @@ -37,7 +65,8 @@ def release_selected_dependency_components( state: RunState, selected_dependencies: dict[str, set[str]], *, - release_component_ports_fn: Callable[[Mapping[str, object]], None], + release_component_ports_fn: Callable[[RequirementsResult, str, Mapping[str, object]], None], + stop_component_fn: Callable[[Mapping[str, object]], None] | None = None, ) -> None: for project_name, dependency_ids in selected_dependencies.items(): requirements = state.requirements.get(project_name) @@ -48,7 +77,9 @@ def release_selected_dependency_components( if not bool(component.get("enabled", False)): continue if not bool(component.get("external")): - release_component_ports_fn(component) + if stop_component_fn is not None: + stop_component_fn(component) + release_component_ports_fn(requirements, dependency_id, component) requirements.components[dependency_id] = {} if not requirements_have_enabled_components(requirements): state.requirements.pop(project_name, None) @@ -57,10 +88,14 @@ def release_selected_dependency_components( def release_requirement_component_ports( component: Mapping[str, object], *, - release_port_fn: Callable[[int], None], + port_planner: Any, + owner_candidates: tuple[str, ...] = (), ) -> None: - for port in sorted(component_port_values(component)): - release_port_fn(port) + release_requirement_component_port_values( + port_planner, + component, + owner_candidates=owner_candidates, + ) def requirements_have_enabled_components(requirements: object) -> bool: @@ -68,7 +103,51 @@ def requirements_have_enabled_components(requirements: object) -> bool: if not isinstance(components, Mapping): return False return any( - bool(component.get("enabled", False)) - for component in components.values() - if isinstance(component, Mapping) + bool(component.get("enabled", False)) for component in components.values() if isinstance(component, Mapping) ) + + +def stop_requirement_component_containers(runtime: Any, component: Mapping[str, object]) -> None: + if bool(component.get("external")) or bool(component.get("simulated")): + return + container_name = str(component.get("container_name") or "").strip() + if not container_name: + return + process_runner = getattr(runtime, "process_runner", None) + if process_runner is None: + return + if str(component.get("id") or "").strip().lower() == "supabase": + compose_project = _supabase_compose_project(container_name) + listed = process_runner.run( + [ + "docker", + "ps", + "--all", + "--quiet", + "--filter", + f"label=com.docker.compose.project={compose_project}", + ], + timeout=30.0, + ) + if listed.returncode != 0: + raise RuntimeError(str(listed.stderr or listed.stdout or "failed listing Supabase containers").strip()) + container_ids = [line.strip() for line in str(listed.stdout or "").splitlines() if line.strip()] + if container_ids: + removed = process_runner.run(["docker", "rm", "--force", *container_ids], timeout=60.0) + if removed.returncode != 0: + detail = removed.stderr or removed.stdout or "failed stopping Supabase containers" + raise RuntimeError(str(detail).strip()) + return + # Native Supabase DB startup records the concrete container name and + # does not attach a Compose project label. Fall through to direct + # removal when the saved component is not a Compose stack. + removed = process_runner.run(["docker", "rm", "--force", container_name], timeout=30.0) + if removed.returncode != 0 and "No such container" not in str(removed.stderr or ""): + raise RuntimeError(str(removed.stderr or removed.stdout or f"failed stopping {container_name}").strip()) + + +def _supabase_compose_project(container_name: str) -> str: + db_container_suffix = "-supabase-db-1" + if container_name.endswith(db_container_suffix): + return container_name[: -len(db_container_suffix)] + return container_name diff --git a/python/envctl_engine/runtime/lifecycle_full_cleanup.py b/python/envctl_engine/runtime/lifecycle_full_cleanup.py new file mode 100644 index 000000000..38ac41134 --- /dev/null +++ b/python/envctl_engine/runtime/lifecycle_full_cleanup.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +from typing import Any, cast + +from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.lifecycle_service_termination import unconfirmed_service_names +from envctl_engine.state.lookup import call_state_loader +from envctl_engine.state.models import RequirementsResult, RunState +from envctl_engine.state.runtime_map import build_runtime_map + + +def clear_runtime_state( + owner: Any, + *, + command: str, + aggressive: bool = False, + route: Route | None = None, +) -> bool: + """Reconcile repository-wide stop and blast state.""" + + rt = owner.runtime + if command == "stop-all": + rt._emit("cleanup.stop_all", aggressive=aggressive) + elif aggressive: + rt._emit("cleanup.blast", aggressive=aggressive) + else: + rt._emit("cleanup.stop", aggressive=aggressive) + state_repository = owner._state_repository(rt) + load_all = getattr(state_repository, "load_all", None) + mode_filter = None + if command == "stop-all" and route is not None and rt._state_lookup_strict_mode_match(route): + mode_filter = route.mode + if callable(load_all): + states = [state for state in load_all(mode=mode_filter) if isinstance(state, RunState)] + else: + states = [] + if not states: + state = call_state_loader( + rt._try_load_existing_state, + mode=mode_filter, + strict_mode_match=mode_filter is not None, + ) + states = [state] if isinstance(state, RunState) else [] + seen_run_ids: set[str] = set() + failed_by_run: dict[str, set[str]] = {} + failed_requirements_by_run: dict[str, set[str]] = {} + for state in states: + if state.run_id in seen_run_ids: + continue + seen_run_ids.add(state.run_id) + termination_result = rt._terminate_services_from_state( + state, + selected_services=None, + aggressive=aggressive, + verify_ownership=not aggressive, + ) + failed_services = unconfirmed_service_names(termination_result, set(state.services)) + state.services = {name: service for name, service in state.services.items() if name in failed_services} + if failed_services: + failed_by_run[state.run_id] = failed_services + failed_projects = { + project_name + for name, service in state.services.items() + if (project_name := owner._project_name_for_service(name, service)) + } + requirements_to_release: list[tuple[str, RequirementsResult]] = [] + retained_requirements: dict[str, RequirementsResult] = {} + for storage_key, requirements in state.requirements.items(): + requirement_project = owner._requirement_project_key(storage_key, requirements) + if failed_projects and requirement_project in failed_projects: + retained_requirements[storage_key] = requirements + continue + containers_stopped = owner._stop_requirement_containers( + requirements, + best_effort=command in {"stop-all", "blast-all"}, + ) + if not containers_stopped: + retained_requirements[storage_key] = requirements + failed_requirements_by_run.setdefault(state.run_id, set()).add( + str(getattr(requirements, "project", "") or storage_key).strip() or storage_key + ) + else: + requirements_to_release.append((storage_key, requirements)) + + for storage_key, requirements in requirements_to_release: + if owner._release_requirement_ports_best_effort(requirements): + continue + retained_requirements[storage_key] = requirements + failed_requirements_by_run.setdefault(state.run_id, set()).add( + str(getattr(requirements, "project", "") or storage_key).strip() or storage_key + ) + + state.requirements = retained_requirements + if state.services or state.requirements: + state_repository.save_selected_stop_state( + state=state, + emit=rt._emit, + runtime_map_builder=lambda raw_state: build_runtime_map(cast(RunState, raw_state)), + ) + else: + owner._deactivate_runtime_state(state, release_session_when_empty=False) + + if failed_by_run or failed_requirements_by_run: + rt._emit( + "cleanup.incomplete", + failed_services={run_id: sorted(names) for run_id, names in failed_by_run.items()}, + failed_requirements={run_id: sorted(names) for run_id, names in failed_requirements_by_run.items()}, + ) + return False + + if aggressive and owner.blast_all_ecosystem_enabled(): + owner.blast_all_ecosystem_cleanup(route=route) + elif command == "stop-all" and route is not None and bool(route.flags.get("stop_all_remove_volumes")): + rt._emit("cleanup.stop_all.remove_volumes", requested=True) + volume_route = Route( + command=route.command, + mode=route.mode, + raw_args=route.raw_args, + passthrough_args=route.passthrough_args, + projects=route.projects, + flags={ + **route.flags, + "blast_keep_worktree_volumes": False, + "blast_remove_main_volumes": True, + }, + ) + removed = owner.blast_all_docker_cleanup(route=volume_route) + rt._emit("cleanup.stop_all.remove_volumes.finish", removed=removed) + + if aggressive or command != "stop-all" or not owner._has_active_runs(state_repository): + owner._purge_runtime_state(aggressive=aggressive) + repository = owner._state_repository(rt) + if aggressive and getattr(repository, "compat_mode", None) == getattr( + repository, + "COMPAT_READ_WRITE", + "compat_read_write", + ): + owner.blast_all_purge_legacy_state_artifacts() + return True diff --git a/python/envctl_engine/runtime/lifecycle_operation_lease.py b/python/envctl_engine/runtime/lifecycle_operation_lease.py new file mode 100644 index 000000000..8992961d5 --- /dev/null +++ b/python/envctl_engine/runtime/lifecycle_operation_lease.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import os +import threading +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from envctl_engine.state.persistence import advisory_file_lock + + +_OPERATION_OWNER_ATTR = "_lifecycle_operation_owner" +_OPERATION_DEPTH_ATTR = "_lifecycle_operation_depth" +_OPERATION_LEASE_ATTR = "_lifecycle_operation_lease" + + +class _LifecycleOperationLease: + def __init__(self, manager: Any) -> None: + self._manager = manager + self.released = False + + def release(self) -> None: + if self.released: + return + self.released = True + self._manager.__exit__(None, None, None) + + +def run_exclusive_lifecycle_operation( + runtime: Any, + *, + command: str, + operation: Callable[[], int], +) -> int: + owner = os.getpid(), threading.get_ident() + current_owner = getattr(runtime, _OPERATION_OWNER_ATTR, None) + current_depth = int(getattr(runtime, _OPERATION_DEPTH_ATTR, 0) or 0) + if current_owner == owner and current_depth > 0: + setattr(runtime, _OPERATION_DEPTH_ATTR, current_depth + 1) + try: + return operation() + finally: + if getattr(runtime, _OPERATION_OWNER_ATTR, None) == owner: + setattr(runtime, _OPERATION_DEPTH_ATTR, current_depth) + + lock_path = _operation_lock_path(runtime) + manager = advisory_file_lock(lock_path, exclusive=True, timeout_seconds=0.0) + try: + manager.__enter__() + except TimeoutError: + runtime._emit("lifecycle.operation.busy", command=command, lock_path=str(lock_path)) + print("Another envctl lifecycle operation is already active for this repository.") + return 1 + + lease = _LifecycleOperationLease(manager) + setattr(runtime, _OPERATION_OWNER_ATTR, owner) + setattr(runtime, _OPERATION_DEPTH_ATTR, 1) + setattr(runtime, _OPERATION_LEASE_ATTR, lease) + try: + return operation() + finally: + if getattr(runtime, _OPERATION_LEASE_ATTR, None) is lease: + _clear_runtime_lease(runtime) + lease.release() + + +def release_lifecycle_operation(runtime: Any) -> bool: + """Release a committed lifecycle mutation before entering an interactive wait.""" + + owner = os.getpid(), threading.get_ident() + if getattr(runtime, _OPERATION_OWNER_ATTR, None) != owner: + return False + lease = getattr(runtime, _OPERATION_LEASE_ATTR, None) + if not isinstance(lease, _LifecycleOperationLease): + return False + _clear_runtime_lease(runtime) + lease.release() + return True + + +def lifecycle_operation_active(runtime: Any) -> bool: + """Return whether this thread still owns an unreleased lifecycle lease.""" + + owner = os.getpid(), threading.get_ident() + lease = getattr(runtime, _OPERATION_LEASE_ATTR, None) + return ( + getattr(runtime, _OPERATION_OWNER_ATTR, None) == owner + and int(getattr(runtime, _OPERATION_DEPTH_ATTR, 0) or 0) > 0 + and isinstance(lease, _LifecycleOperationLease) + and not lease.released + ) + + +def _clear_runtime_lease(runtime: Any) -> None: + setattr(runtime, _OPERATION_DEPTH_ATTR, 0) + setattr(runtime, _OPERATION_OWNER_ATTR, None) + setattr(runtime, _OPERATION_LEASE_ATTR, None) + + +def _operation_lock_path(runtime: Any) -> Path: + repository = getattr(runtime, "state_repository", None) + repository_lock_path = getattr(repository, "lifecycle_operation_lock_path", None) + if callable(repository_lock_path): + return Path(repository_lock_path()) + root = getattr(repository, "runtime_root", None) + if root is None: + root = getattr(runtime, "runtime_root") + root_path = Path(root).expanduser().absolute() + if root_path.is_symlink(): + raise RuntimeError(f"runtime_root changed identity or became a symlink: {root_path}") + return root_path.resolve(strict=False) / ".lifecycle_operation.lock" + + +__all__ = [ + "lifecycle_operation_active", + "release_lifecycle_operation", + "run_exclusive_lifecycle_operation", +] diff --git a/python/envctl_engine/runtime/lifecycle_requirement_ports.py b/python/envctl_engine/runtime/lifecycle_requirement_ports.py index 213ef6157..ab5e206d1 100644 --- a/python/envctl_engine/runtime/lifecycle_requirement_ports.py +++ b/python/envctl_engine/runtime/lifecycle_requirement_ports.py @@ -1,15 +1,134 @@ from __future__ import annotations from collections.abc import Mapping +import inspect from typing import Any from envctl_engine.requirements.core import dependency_definitions from envctl_engine.state.models import RequirementsResult, RunState +from envctl_engine.state.project_runtime import ( + requirement_key_for_project as _requirement_key_for_project, + requirement_keys_for_project as _requirement_keys_for_project, +) def release_requirement_ports(runtime: Any, requirements: RequirementsResult) -> None: - for port in sorted(requirement_port_values(requirements)): - runtime.port_planner.release(port) + reservations: list[tuple[int, tuple[str, ...], str | None]] = [] + for definition in dependency_definitions(): + component = requirements.component(definition.id) + if not bool(component.get("enabled", False)) or bool(component.get("external")): + continue + owners = requirement_component_port_owners(requirements, definition.id) + expected_session = _port_lock_session(component.get("port_lock_session")) + reservations.extend((port, owners, expected_session) for port in component_port_values(component)) + for port, owners, expected_session in sorted(reservations, key=lambda item: item[0]): + release_port_reservation( + runtime.port_planner, + port, + owner_candidates=owners, + expected_session=expected_session, + ) + + +def release_requirement_component_port_values( + port_planner: Any, + component: Mapping[str, object], + *, + owner_candidates: tuple[str, ...], +) -> None: + expected_session = _port_lock_session(component.get("port_lock_session")) + for port in sorted(component_port_values(component)): + release_port_reservation( + port_planner, + port, + owner_candidates=owner_candidates, + expected_session=expected_session, + ) + + +def release_port_reservation( + port_planner: Any, + port: int, + *, + owner_candidates: tuple[str, ...], + expected_session: str | None, +) -> bool: + """Release one persisted reservation without crossing planner sessions.""" + + owners = tuple(dict.fromkeys(owner for owner in owner_candidates if owner)) + normalized_session = _port_lock_session(expected_session) + release_owned = getattr(port_planner, "release_owned", None) + if normalized_session and callable(release_owned): + for owner in owners: + if bool(release_owned(port, owner, expected_session=normalized_session)): + return True + + # A legacy record has no session identity. Its only safe eager release is + # through the current planner session, whose release method checks its own + # session before unlinking. For a persisted session, use this fallback only + # when that exact planner session is still active. + current_session = _port_lock_session(getattr(port_planner, "session_id", None)) + if normalized_session is None or current_session == normalized_session: + _release_current_session(port_planner, port, owners) + + # Reaping is the compatibility path for dead foreign sessions. It is + # deliberately attempted only through the planner's liveness proof. + reap_stale = getattr(port_planner, "reap_stale", None) + if callable(reap_stale): + if owners: + for owner in owners: + if bool(reap_stale(port, owner=owner)): + return True + elif bool(reap_stale(port)): + return True + return False + + +def _release_current_session(port_planner: Any, port: int, owners: tuple[str, ...]) -> None: + release = getattr(port_planner, "release", None) + if not callable(release): + return + if not owners: + release(port) + return + try: + parameters = inspect.signature(release).parameters.values() + except (TypeError, ValueError): + return + supports_owner = any( + parameter.name == "owner" or parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters + ) + if not supports_owner: + # Minimal test doubles and legacy adapters may expose release(port). + # Production PortPlanner always takes owner and enforces its session. + release(port) + return + for owner in owners: + release(port, owner=owner) + + +def _port_lock_session(value: object) -> str | None: + normalized = value.strip() if isinstance(value, str) else "" + return normalized or None + + +def requirement_component_port_owners( + requirements: RequirementsResult, + dependency_id: str, +) -> tuple[str, ...]: + """Return every owner form used while reserving one dependency's ports.""" + + project = str(requirements.project).strip() + normalized_dependency = str(dependency_id).strip().lower() + if not project or not normalized_dependency: + return () + owner_suffixes = {"requirements"} + for definition in dependency_definitions(): + if definition.id != normalized_dependency: + continue + owner_suffixes.update(resource.legacy_port_key for resource in definition.resources) + break + return tuple(f"{project}:{suffix}" for suffix in sorted(owner_suffixes)) def requirement_port_values(requirements: RequirementsResult) -> set[int]: @@ -20,9 +139,7 @@ def requirement_port_values(requirements: RequirementsResult) -> set[int]: continue if bool(component.get("external")): continue - final = component.get("final") - if isinstance(final, int) and final > 0: - ports.add(final) + ports.update(component_port_values(component)) return ports @@ -40,10 +157,8 @@ def component_port_values(component: Mapping[str, object]) -> set[int]: def requirement_key_for_project(state: RunState, project_name: str) -> str | None: - target = str(project_name).strip().lower() - if not target: - return None - for key in state.requirements: - if str(key).strip().lower() == target: - return key - return None + return _requirement_key_for_project(state, project_name) + + +def requirement_keys_for_project(state: RunState, project_name: str) -> list[str]: + return _requirement_keys_for_project(state, project_name) diff --git a/python/envctl_engine/runtime/lifecycle_service_stop_selection.py b/python/envctl_engine/runtime/lifecycle_service_stop_selection.py index 917c59505..023e75c4f 100644 --- a/python/envctl_engine/runtime/lifecycle_service_stop_selection.py +++ b/python/envctl_engine/runtime/lifecycle_service_stop_selection.py @@ -4,7 +4,7 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.lifecycle_dependency_stop import select_dependency_components_for_stop -from envctl_engine.shared.services import service_matches_selector +from envctl_engine.shared.services import service_matches_selector, service_project_name from envctl_engine.state.models import RunState @@ -22,24 +22,26 @@ def select_services_for_stop( if bool(route.flags.get("all")): return set(state.services.keys()) + selected, has_selectors = _selected_services_from_route( + state, + route, + project_name_from_service_fn=project_name_from_service_fn, + selectors_from_passthrough_fn=selectors_from_passthrough_fn, + ) + runtime_scope = route.flags.get("runtime_scope") if runtime_scope in {"backend", "frontend"}: - return { + scoped = { name for name, service in state.services.items() if service_matches_runtime_scope(name, service, str(runtime_scope)) } + return scoped.intersection(selected) if has_selectors else scoped if runtime_scope in {"fullstack", "entire-system"}: - return set(state.services.keys()) + return selected if has_selectors else set(state.services.keys()) if runtime_scope == "dependencies": return set() - selected, has_selectors = _selected_services_from_route( - state, - route, - project_name_from_service_fn=project_name_from_service_fn, - selectors_from_passthrough_fn=selectors_from_passthrough_fn, - ) if not selected and has_selectors: return set() if not selected and select_dependency_components_for_stop(state, route): @@ -77,7 +79,10 @@ def _selected_services_from_route( has_selectors = has_selectors or bool(services_flag) selected.update(_services_matching_service_selectors(state, services_flag)) - project_names = _project_selectors(route, selectors_from_passthrough_fn=selectors_from_passthrough_fn) + project_names = project_selectors_from_route( + route, + selectors_from_passthrough_fn=selectors_from_passthrough_fn, + ) has_selectors = has_selectors or bool(project_names) if project_names: selected.update( @@ -103,7 +108,7 @@ def _services_matching_service_selectors(state: RunState, services_flag: list[ob return selected -def _project_selectors( +def project_selectors_from_route( route: Route, *, selectors_from_passthrough_fn: Callable[[list[str]], Iterable[str]], @@ -121,8 +126,8 @@ def _services_matching_project_selectors( project_name_from_service_fn: Callable[[str], str], ) -> set[str]: selected: set[str] = set() - for name in state.services: - project = project_name_from_service_fn(name).lower() + for name, service in state.services.items(): + project = (service_project_name(service) or project_name_from_service_fn(name)).lower() if project and project in project_names: selected.add(name) return selected diff --git a/python/envctl_engine/runtime/lifecycle_service_termination.py b/python/envctl_engine/runtime/lifecycle_service_termination.py index 0c8ff9841..1720390f6 100644 --- a/python/envctl_engine/runtime/lifecycle_service_termination.py +++ b/python/envctl_engine/runtime/lifecycle_service_termination.py @@ -2,13 +2,71 @@ import concurrent.futures import os -import signal +from collections.abc import Collection, Mapping +from dataclasses import dataclass +from pathlib import Path from typing import Any +from envctl_engine.shared.process_termination import ( + pid_is_running, + terminate_pid, + wait_for_pid_exit, +) +from envctl_engine.shared.services import service_project_name +from envctl_engine.runtime.lifecycle_requirement_ports import release_port_reservation -def terminate_started_services(runtime: Any, services: dict[str, object]) -> None: - for service in services.values(): - runtime._terminate_service_record(service, aggressive=False, verify_ownership=False) + +def unconfirmed_service_names(result: object, service_names: Collection[str]) -> set[str]: + """Normalize a termination adapter result without ever assuming success. + + Only a collection containing known service identities is authoritative. An + absent/scalar result or any unknown identity fails closed for the complete + requested set so callers never discard state for a process whose exit was + not positively confirmed. + """ + + requested = {str(name).strip() for name in service_names if str(name).strip()} + if not isinstance(result, Collection) or isinstance(result, (str, bytes, Mapping)): + return requested + raw_reported = list(result) + if any(not str(name).strip() for name in raw_reported): + return requested + reported = {str(name).strip() for name in raw_reported} + return reported if reported.issubset(requested) else requested + + +def failed_listener_pids(result: object) -> set[int] | None: + """Return failed listener PIDs, or ``None`` when cleanup was not confirmed.""" + + if not isinstance(result, Collection) or isinstance(result, (str, bytes, Mapping)): + return None + failed: set[int] = set() + for value in result: + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + return None + failed.add(value) + return failed + + +def terminate_started_services(runtime: Any, services: dict[str, object]) -> set[str]: + """Terminate startup-owned services and report every unconfirmed exit.""" + + failed: set[str] = set() + for name, service in services.items(): + try: + terminated = runtime._terminate_service_record( + service, + aggressive=False, + verify_ownership=False, + ) + except Exception as exc: # noqa: BLE001 + _emit_cleanup_error(runtime, service=name, error=exc) + terminated = False + if not terminated: + failed.add(name) + else: + release_service_ports(runtime, name=name, service=service) + return failed def terminate_services_from_state( @@ -18,19 +76,25 @@ def terminate_services_from_state( selected_services: set[str] | None, aggressive: bool, verify_ownership: bool, -) -> None: +) -> set[str]: work_items: list[tuple[str, object]] = [] for name, service in getattr(state, "services", {}).items(): if selected_services is not None and name not in selected_services: continue work_items.append((name, service)) - def terminate_one(item: tuple[str, object]) -> tuple[str, bool, int | None]: + def terminate_one(item: tuple[str, object]) -> tuple[str, object, bool, int | None]: name, service = item - terminated = runtime._terminate_service_record( - service, aggressive=aggressive, verify_ownership=verify_ownership - ) - return name, terminated, service_port(service) + try: + terminated = runtime._terminate_service_record( + service, + aggressive=aggressive, + verify_ownership=verify_ownership, + ) + except Exception as exc: # noqa: BLE001 + _emit_cleanup_error(runtime, service=name, error=exc) + terminated = False + return name, service, terminated, service_port(service) if len(work_items) <= 1: results = [terminate_one(item) for item in work_items] @@ -39,47 +103,168 @@ def terminate_one(item: tuple[str, object]) -> tuple[str, bool, int | None]: with concurrent.futures.ThreadPoolExecutor(max_workers=worker_count) as pool: results = list(pool.map(terminate_one, work_items)) - for _name, terminated, port in results: - if terminated and port is not None: - runtime.port_planner.release(port) + for name, service, terminated, _port in results: + if terminated: + release_service_ports(runtime, name=name, service=service) + return {name for name, _service, terminated, _port in results if not terminated} + + +def release_service_ports(runtime: Any, *, name: str, service: object) -> None: + port_planner = getattr(runtime, "port_planner", None) + if port_planner is None: + return + project = str(service_project_name(service)).strip() + if not project: + project_resolver = getattr(runtime, "_project_name_from_service", None) + if callable(project_resolver): + project = str(project_resolver(name) or "").strip() + service_type = str(getattr(service, "type", "") or "").strip().lower() + owners: tuple[str, ...] = () + if project: + owner_candidates = [f"{project}:services"] + if service_type: + owner_candidates = [ + f"{project}:{service_type}", + f"{project}:services:{service_type}-launch", + *owner_candidates, + ] + owners = tuple(owner_candidates) + + raw_expected_session = getattr(service, "port_lock_session", None) + expected_session = raw_expected_session.strip() if isinstance(raw_expected_session, str) else None + for port in service_ports(service): + release_port_reservation( + port_planner, + port, + owner_candidates=owners, + expected_session=expected_session, + ) + + +def service_ports(service: object) -> tuple[int, ...]: + """Return every distinct port reservation represented by a service record.""" + + ports: list[int] = [] + for attribute in ("requested_port", "actual_port"): + value = getattr(service, attribute, None) + if isinstance(value, int) and not isinstance(value, bool) and value > 0 and value not in ports: + ports.append(value) + return tuple(ports) def terminate_service_record(runtime: Any, service: object, *, aggressive: bool, verify_ownership: bool) -> bool: - pid = getattr(service, "pid", None) - if not isinstance(pid, int) or pid <= 0: - return True - if pid in {os.getpid(), os.getppid()}: - runtime._emit("cleanup.skip", service=getattr(service, "name", "unknown"), pid=pid, reason="self_or_parent") + if str(getattr(service, "runtime_kind", "process") or "process").lower() == "docker": + from envctl_engine.runtime.docker_service_runtime import DockerServiceRuntime + + container = str( + getattr(service, "container_id", "") or getattr(service, "container_name", "") or "" + ).strip() + launch_token = str(getattr(service, "container_launch_token", "") or "").strip() + pending_since = getattr(service, "container_cleanup_pending_since", None) + stopped = DockerServiceRuntime(runtime, runtime.process_runner).stop( + container, + verify_ownership=verify_ownership, + **({"expected_launch_token": launch_token} if launch_token else {}), + **( + {"pending_cleanup_since": pending_since} + if isinstance(pending_since, (int, float)) + else {} + ), + ) + follower_stopped = True + pid = getattr(service, "pid", None) + if isinstance(pid, int) and pid > 0: + if pid in {os.getpid(), os.getppid()}: + runtime._emit( + "cleanup.skip", + service=getattr(service, "name", "unknown"), + pid=pid, + reason="self_or_parent", + ) + follower_stopped = False + else: + try: + runtime.process_runner.terminate_process_group( + pid, + term_timeout=0.5 if aggressive else 2.0, + kill_timeout=1.0, + ) + except Exception: # noqa: BLE001 + pass + follower_stopped = not _pid_is_running(runtime, pid) + runtime._emit( + "service.container.stop", + service=getattr(service, "name", "unknown"), + container_name=getattr(service, "container_name", None), + stopped=stopped, + follower_stopped=follower_stopped, + ) + return stopped and follower_stopped + + recorded_pids = _recorded_service_pids(service) + if not recorded_pids and str(getattr(service, "status", "") or "") == "termination_failed": + return False + protected_pids = {os.getpid(), os.getppid()} + protected = [pid for pid in recorded_pids if pid in protected_pids] + if protected: + runtime._emit( + "cleanup.skip", + service=getattr(service, "name", "unknown"), + pid=protected[0], + reason="self_or_parent", + ) return False + live_pids = [pid for pid in recorded_pids if _pid_is_running(runtime, pid)] port = service_port(service) + listener_probe = _live_listener_pids_for_port(runtime, port) if port is not None else _ListenerProbe() + if not live_pids: + return not listener_probe.failed and not listener_probe.pids if verify_ownership: if port is None: - runtime._emit( - "cleanup.skip", - service=getattr(service, "name", "unknown"), - pid=pid, - reason="missing_port_for_ownership", - ) - return False - is_owner = _service_pid_or_listener_owns_port(runtime, service, pid=pid, port=port) - if not is_owner: - runtime._emit("cleanup.skip", service=getattr(service, "name", "unknown"), pid=pid, port=port) - return False + if not all(_non_listener_service_matches_recorded_cwd(runtime, service, pid=pid) for pid in live_pids): + runtime._emit( + "cleanup.skip", + service=getattr(service, "name", "unknown"), + pid=live_pids[0], + reason="missing_port_for_ownership", + ) + return False + termination_pids = live_pids + else: + owners = [pid for pid in live_pids if _pid_owns_port(runtime, pid=pid, port=port)] + if not owners: + runtime._emit( + "cleanup.skip", + service=getattr(service, "name", "unknown"), + pid=live_pids[0], + port=port, + reason="ownership_mismatch", + ) + return False + primary_pid = getattr(service, "pid", None) + termination_pids = list(owners) + if isinstance(primary_pid, int) and primary_pid in live_pids and primary_pid not in termination_pids: + termination_pids.insert(0, primary_pid) + else: + termination_pids = live_pids - try: - terminate_group = getattr(runtime.process_runner, "terminate_process_group", None) - if callable(terminate_group): - return bool(terminate_group(pid, term_timeout=0.5 if aggressive else 2.0, kill_timeout=1.0)) - except Exception: # noqa: BLE001 - pass - try: - return bool(runtime.process_runner.terminate(pid, term_timeout=0.5 if aggressive else 2.0, kill_timeout=1.0)) - except Exception: # noqa: BLE001 - pass - try: - os.kill(pid, signal.SIGTERM) - except OSError: - return True + failed = False + for pid in termination_pids: + if not _pid_is_running(runtime, pid): + continue + if not terminate_pid( + pid, + process_runner=runtime.process_runner, + term_timeout=0.5 if aggressive else 2.0, + kill_timeout=1.0, + ): + failed = True + if failed: + return False + if port is not None: + listener_probe = _live_listener_pids_for_port(runtime, port) + if listener_probe.failed or listener_probe.pids: + return False return True @@ -93,21 +278,92 @@ def service_port(service: object) -> int | None: return None -def _service_pid_or_listener_owns_port(runtime: Any, service: object, *, pid: int, port: int) -> bool: +def _emit_cleanup_error(runtime: Any, *, service: str, error: BaseException) -> None: + emit = getattr(runtime, "_emit", None) + if callable(emit): + emit("cleanup.error", service=service, error=str(error)) + + +def _pid_owns_port(runtime: Any, *, pid: int, port: int) -> bool: try: - if bool(runtime.process_runner.pid_owns_port(pid, port)): - return True + return bool(runtime.process_runner.pid_owns_port(pid, port)) except Exception: # noqa: BLE001 - pass - listener_pids = getattr(service, "listener_pids", None) - if not isinstance(listener_pids, list): return False - for listener_pid in listener_pids: - if not isinstance(listener_pid, int) or listener_pid <= 0: - continue - try: - if bool(runtime.process_runner.pid_owns_port(listener_pid, port)): - return True - except Exception: # noqa: BLE001 - continue - return False + + +def _recorded_service_pids(service: object) -> list[int]: + candidates = [getattr(service, "pid", None), *(getattr(service, "listener_pids", None) or [])] + return list( + dict.fromkeys(pid for pid in candidates if isinstance(pid, int) and not isinstance(pid, bool) and pid > 0) + ) + + +@dataclass(frozen=True, slots=True) +class _ListenerProbe: + available: bool = False + failed: bool = False + pids: frozenset[int] = frozenset() + + +def _live_listener_pids_for_port(runtime: Any, port: int) -> _ListenerProbe: + listener_reader = getattr(runtime, "_listener_pids_for_port", None) + if not callable(listener_reader): + return _ListenerProbe() + try: + listeners = listener_reader(port) + except Exception: # noqa: BLE001 + return _ListenerProbe(available=True, failed=True) + if listeners is None or isinstance(listeners, (str, bytes, Mapping)): + return _ListenerProbe(available=True, failed=True) + try: + listener_items = iter(listeners) + except TypeError: + return _ListenerProbe(available=True, failed=True) + return _ListenerProbe( + available=True, + pids=frozenset( + pid + for pid in listener_items + if isinstance(pid, int) and not isinstance(pid, bool) and pid > 0 and _pid_is_running(runtime, pid) + ), + ) + + +def _non_listener_service_matches_recorded_cwd(runtime: Any, service: object, *, pid: int) -> bool: + if bool(getattr(service, "listener_expected", True)): + return False + recorded_cwd = str(getattr(service, "cwd", "") or "").strip() + if not recorded_cwd: + return False + cwd_reader = getattr(runtime.process_runner, "process_cwd", None) + if not callable(cwd_reader): + return False + try: + actual_cwd = str(cwd_reader(pid) or "").strip() + except Exception: # noqa: BLE001 + return False + if not actual_cwd: + return False + try: + return Path(actual_cwd).resolve(strict=False) == Path(recorded_cwd).resolve(strict=False) + except OSError: + return actual_cwd == recorded_cwd + + +def _wait_for_pid_exit( + runtime: Any, + pid: int, + *, + timeout: float, + initial_identity: str | None, +) -> bool: + return wait_for_pid_exit( + runtime.process_runner, + pid, + timeout=timeout, + initial_identity=initial_identity, + ) + + +def _pid_is_running(runtime: Any, pid: int) -> bool: + return pid_is_running(runtime.process_runner, pid) diff --git a/python/envctl_engine/runtime/lifecycle_targeted_state.py b/python/envctl_engine/runtime/lifecycle_targeted_state.py new file mode 100644 index 000000000..35960e4ff --- /dev/null +++ b/python/envctl_engine/runtime/lifecycle_targeted_state.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from typing import Any, cast + +from envctl_engine.shared.services import resolve_service_project_name +from envctl_engine.state.models import RequirementsResult, RunState +from envctl_engine.state.runtime_map import build_runtime_map +from envctl_engine.state.state_aggregation import filter_state_to_owned_projects + + +class LifecycleTargetedStateSupport: + """Own project-scoped state reduction after targeted lifecycle stops.""" + + runtime: Any = cast(Any, None) + + def _project_name_for_service(self, name: str, service: object) -> str: + return resolve_service_project_name( + name, + service, + project_name_from_service=self.runtime._project_name_from_service, + ).casefold() + + def _save_selected_stop_state( + self, + state: RunState, + *, + authoritative_project_names: list[str] | None = None, + ) -> None: + owner = cast(Any, self) + repository = owner._state_repository(self.runtime) + + def runtime_map_builder(raw_state: object) -> dict[str, object]: + return build_runtime_map(cast(RunState, raw_state)) + + if authoritative_project_names is None: + repository.save_selected_stop_state( + state=state, + emit=self.runtime._emit, + runtime_map_builder=runtime_map_builder, + ) + return + repository.save_selected_stop_state( + state=state, + emit=self.runtime._emit, + runtime_map_builder=runtime_map_builder, + authoritative_project_names=authoritative_project_names, + ) + + def _persist_targeted_project_state( + self, + state: RunState, + *, + owned_projects: dict[str, str], + targeted_project_keys: frozenset[str], + ) -> bool: + owner = cast(Any, self) + active_project_keys = self._active_project_keys(state) + remaining_keys = (set(owned_projects).difference(targeted_project_keys)).union( + targeted_project_keys.intersection(active_project_keys) + ) + if not remaining_keys: + for requirements in state.requirements.values(): + owner._release_requirement_ports(requirements) + owner._deactivate_runtime_state(state) + return False + + remaining_names = [owned_projects[key] for key in sorted(remaining_keys)] + filtered = filter_state_to_owned_projects( + state, + frozenset(remaining_keys), + service_project_name=self._project_name_for_service, + ) + filtered.metadata["project_names"] = remaining_names + self._save_selected_stop_state( + filtered, + authoritative_project_names=remaining_names, + ) + return True + + def _owned_project_names(self, state: RunState) -> dict[str, str]: + projects: dict[str, str] = {} + + def add(raw_name: object) -> None: + name = str(raw_name or "").strip() + if name: + projects.setdefault(name.casefold(), name) + + metadata_names = state.metadata.get("project_names") + if isinstance(metadata_names, Iterable) and not isinstance(metadata_names, (str, bytes, Mapping)): + for project in metadata_names: + add(project) + metadata_roots = state.metadata.get("project_roots") + if isinstance(metadata_roots, Mapping): + for project in metadata_roots: + add(project) + for project, requirements in state.requirements.items(): + add(getattr(requirements, "project", "") or project) + for name, service in state.services.items(): + add(self._project_name_for_service(name, service)) + if state.mode == "trees" and any(project != "main" for project in projects): + projects.pop("main", None) + return projects + + def _active_project_keys(self, state: RunState) -> set[str]: + active = { + self._project_name_for_service(name, service) + for name, service in state.services.items() + if self._project_name_for_service(name, service) + } + active.update( + self._requirement_project_key(project, requirements) for project, requirements in state.requirements.items() + ) + raw_stopped = state.metadata.get("dashboard_stopped_services") + if isinstance(raw_stopped, list): + active.update( + str(item.get("project", "")).strip().casefold() + for item in raw_stopped + if isinstance(item, Mapping) and str(item.get("project", "")).strip() + ) + return {project for project in active if project} + + def _targeted_projects_have_no_managed_resources( + self, + state: RunState, + targeted_project_keys: frozenset[str], + ) -> bool: + if any( + self._project_name_for_service(name, service) in targeted_project_keys + for name, service in state.services.items() + ): + return False + return not any( + self._requirement_project_key(project, requirements) in targeted_project_keys + and cast(Any, self)._requirements_have_enabled_components(requirements) + for project, requirements in state.requirements.items() + ) + + def _remove_targeted_requirements( + self, + state: RunState, + targeted_project_keys: frozenset[str], + ) -> None: + owner = cast(Any, self) + for project, requirements in list(state.requirements.items()): + if self._requirement_project_key(project, requirements) not in targeted_project_keys: + continue + owner._release_requirement_ports(requirements) + state.requirements.pop(project, None) + + @staticmethod + def _requirement_project_key(project: str, requirements: RequirementsResult) -> str: + return str(getattr(requirements, "project", "") or project).strip().casefold() diff --git a/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py b/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py index 771073dbb..27de6d04b 100644 --- a/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py +++ b/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py @@ -6,12 +6,15 @@ from pathlib import Path from typing import Any, Protocol +from envctl_engine.requirements.core import dependency_definitions +from envctl_engine.runtime.lifecycle_dependency_stop import stop_requirement_component_containers from envctl_engine.runtime.lifecycle_requirement_ports import ( release_requirement_ports, - requirement_key_for_project, + requirement_keys_for_project, requirement_port_values, ) from envctl_engine.runtime.lifecycle_service_termination import service_port +from envctl_engine.runtime.lifecycle_service_termination import unconfirmed_service_names from envctl_engine.runtime.lifecycle_worktree_containers import ( legacy_container_name as _legacy_container_name, ) @@ -30,6 +33,8 @@ from envctl_engine.runtime.lifecycle_worktree_processes import ( blast_tree_listener_ports as _blast_tree_listener_ports, ) +from envctl_engine.shared.services import resolve_service_project_name +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RequirementsResult, RunState from envctl_engine.state.runtime_map import build_runtime_map @@ -136,14 +141,23 @@ def blast_worktree_before_delete( ) for mode in ("trees", "main"): - state = runtime._try_load_existing_state(mode=mode, strict_mode_match=True) + state = call_state_loader( + runtime._try_load_existing_state, + mode=mode, + strict_mode_match=True, + ) if state is None: continue selected_services = { name - for name in list(state.services.keys()) - if runtime._project_name_from_service(name).strip().lower() == target_lower + for name, service in list(state.services.items()) + if resolve_service_project_name( + name, + service, + project_name_from_service=runtime._project_name_from_service, + ).lower() + == target_lower } for service_name in selected_services: service = state.services.get(service_name) @@ -155,37 +169,38 @@ def blast_worktree_before_delete( log_path_raw = str(getattr(service, "log_path", "") or "").strip() if blast_mode and log_path_raw: artifact_paths.add(Path(log_path_raw)) - requirement_key = requirement_key_for_project(state, normalized_project) - if not selected_services and requirement_key is None: + requirement_keys = requirement_keys_for_project(state, normalized_project) + if not selected_services and not requirement_keys: continue - runtime._terminate_services_from_state( + termination_result = runtime._terminate_services_from_state( state, selected_services=selected_services, aggressive=True, verify_ownership=False, ) - for service_name in selected_services: + failed_services = unconfirmed_service_names(termination_result, selected_services) + stopped_services = selected_services.difference(failed_services) + for service_name in stopped_services: state.services.pop(service_name, None) - if requirement_key is not None: - requirement_entry = state.requirements.pop(requirement_key, None) - if requirement_entry is not None: + requirements_removed = 0 + if requirement_keys and not failed_services: + for requirement_key in requirement_keys: + requirement_entry = state.requirements.get(requirement_key) + if requirement_entry is None: + continue + for definition in dependency_definitions(): + component = requirement_entry.component(definition.id) + if bool(component.get("enabled", False)): + stop_requirement_component_containers(runtime, component) + state.requirements.pop(requirement_key, None) + requirements_removed += 1 target_ports.update(cleanup.collect_requirement_ports(requirement_entry)) release_requirement_ports(runtime, requirement_entry) - remaining_projects: set[str] = set() - for service_name in state.services: - project = runtime._project_name_from_service(service_name) - if project: - remaining_projects.add(project) - for project in list(state.requirements.keys()): - if project in remaining_projects: - continue - requirement_entry = state.requirements.pop(project) - release_requirement_ports(runtime, requirement_entry) - - artifact_paths.update(cleanup.prune_project_metadata(state, project_name=normalized_project)) + if not failed_services: + artifact_paths.update(cleanup.prune_project_metadata(state, project_name=normalized_project)) try: runtime.state_repository.save_selected_stop_state( @@ -202,15 +217,27 @@ def blast_worktree_before_delete( mode=mode, warning=warning, ) - continue + raise RuntimeError(warning) from exc runtime._emit( "cleanup.worktree.state.updated", project=normalized_project, mode=mode, - services_removed=len(selected_services), - requirements_removed=(requirement_key is not None), + services_removed=len(stopped_services), + requirements_removed=requirements_removed, ) + if failed_services: + warning = ( + f"worktree cleanup could not confirm service exit for {normalized_project} ({mode}): " + + ", ".join(sorted(failed_services)) + ) + runtime._emit( + "cleanup.worktree.warning", + project=normalized_project, + mode=mode, + warning=warning, + ) + raise RuntimeError(warning) if blast_mode: cleanup.blast_tree_listener_ports( diff --git a/python/envctl_engine/runtime/lifecycle_worktree_containers.py b/python/envctl_engine/runtime/lifecycle_worktree_containers.py index 55717aab7..6f796743b 100644 --- a/python/envctl_engine/runtime/lifecycle_worktree_containers.py +++ b/python/envctl_engine/runtime/lifecycle_worktree_containers.py @@ -7,6 +7,7 @@ from envctl_engine.requirements.common_contracts import build_container_name from envctl_engine.requirements.docker_runtime import run_docker, run_result_error from envctl_engine.requirements.supabase import build_supabase_project_name +from envctl_engine.runtime.docker_service_runtime import docker_service_container_name def legacy_container_name(*, prefix: str, project_name: str) -> str: @@ -36,6 +37,20 @@ def remove_tree_containers( legacy_container_name(prefix="envctl-redis", project_name=project_name): "redis", legacy_container_name(prefix="envctl-n8n", project_name=project_name): "n8n", } + app_service_names = ["backend", "frontend"] + app_service_names.extend( + str(getattr(service, "name", "") or "").strip() + for service in getattr(getattr(runtime, "config", None), "additional_services", ()) + if str(getattr(service, "name", "") or "").strip() + ) + for service_name in app_service_names: + exact_names[ + docker_service_container_name( + project_name=project_name, + project_root=resolved_root, + service_name=service_name, + ) + ] = service_name supabase_prefixes: set[str] = set() if include_supabase: supabase_prefixes = { diff --git a/python/envctl_engine/runtime/lifecycle_worktree_metadata.py b/python/envctl_engine/runtime/lifecycle_worktree_metadata.py index 297a5143f..dcc87fa09 100644 --- a/python/envctl_engine/runtime/lifecycle_worktree_metadata.py +++ b/python/envctl_engine/runtime/lifecycle_worktree_metadata.py @@ -8,6 +8,18 @@ def prune_project_metadata(state: RunState, *, project_name: str) -> list[Path]: removed_paths: list[Path] = [] + normalized_project = str(project_name).strip().casefold() + raw_project_names = state.metadata.get("project_names") + if isinstance(raw_project_names, list): + remaining_project_names = [ + name + for name in raw_project_names + if str(name).strip().casefold() != normalized_project + ] + if remaining_project_names: + state.metadata["project_names"] = remaining_project_names + else: + state.metadata.pop("project_names", None) for metadata_key in ("project_pr_links", "project_roots"): raw = state.metadata.get(metadata_key) if not isinstance(raw, dict): diff --git a/python/envctl_engine/runtime/requirement_reconcile_truth.py b/python/envctl_engine/runtime/requirement_reconcile_truth.py index da910b260..c4be27c17 100644 --- a/python/envctl_engine/runtime/requirement_reconcile_truth.py +++ b/python/envctl_engine/runtime/requirement_reconcile_truth.py @@ -118,9 +118,9 @@ def requirement_truth_identity( state_key: str, requirements: RequirementsResult, ) -> tuple[str, Path | None]: - truth_project = state_key - shared_scope = str(state.metadata.get("dashboard_dependency_scope", "")).strip().lower() == "shared" requirement_project = str(getattr(requirements, "project", "") or "").strip() + truth_project = requirement_project or state_key + shared_scope = str(state.metadata.get("dashboard_dependency_scope", "")).strip().lower() == "shared" shared_project = str(state.metadata.get("dashboard_shared_dependency_project", "") or "").strip() if shared_scope and requirement_project: truth_project = requirement_project @@ -134,7 +134,8 @@ def requirement_truth_identity( def requirement_truth_issues(runtime: Any, state: RunState) -> list[dict[str, object]]: issues: list[dict[str, object]] = [] - for project, requirements in state.requirements.items(): + for storage_key, requirements in state.requirements.items(): + project = str(getattr(requirements, "project", "") or storage_key).strip() for definition in dependency_definitions(): component_name = definition.id component_data = requirements.component(component_name) diff --git a/python/envctl_engine/runtime/service_listener_truth.py b/python/envctl_engine/runtime/service_listener_truth.py index f6f6c1377..8cdc4390d 100644 --- a/python/envctl_engine/runtime/service_listener_truth.py +++ b/python/envctl_engine/runtime/service_listener_truth.py @@ -35,6 +35,12 @@ def wait_for_service_listener( return True except Exception: # noqa: BLE001 pass + # A generic port probe is only a safe fallback while the process envctl + # launched is still alive. Once that PID has exited, waiting on the port can + # hide the real startup error for minutes and can adopt an unrelated process + # that happens to bind the same port later. + if _launch_pid_definitively_exited(runtime, pid): + return False if ( debug_listener_group in {"", "port_fallback"} and service_truth_fallback_enabled(runtime) @@ -51,6 +57,16 @@ def wait_for_service_listener( return False +def _launch_pid_definitively_exited(runtime: Any, pid: int) -> bool: + checker = getattr(runtime.process_runner, "is_pid_running", None) + if not callable(checker): + return False + try: + return not bool(checker(pid)) + except Exception: # noqa: BLE001 + return False + + def process_tree_probe_supported(runtime: Any) -> bool: try: return bool(runtime.process_runner.supports_process_tree_probe()) diff --git a/python/envctl_engine/runtime/service_manager.py b/python/envctl_engine/runtime/service_manager.py index a26833372..a5240bfe8 100644 --- a/python/envctl_engine/runtime/service_manager.py +++ b/python/envctl_engine/runtime/service_manager.py @@ -1,13 +1,13 @@ from __future__ import annotations import concurrent.futures -import os -import signal import time +from collections.abc import Iterable from dataclasses import dataclass from typing import Callable from envctl_engine.requirements.common_contracts import is_bind_conflict +from envctl_engine.shared.process_termination import terminate_pid from envctl_engine.shared.services import service_display_name from envctl_engine.state.models import ServiceRecord @@ -16,6 +16,14 @@ class ServiceStartError(RuntimeError): """Raised when a service fails to start after retry policy.""" +class ServiceCleanupError(ServiceStartError): + """Raised when a failed startup leaves processes whose exit is unconfirmed.""" + + def __init__(self, message: str, unterminated_services: dict[str, ServiceRecord]) -> None: + super().__init__(message) + self.unterminated_services = dict(unterminated_services) + + @dataclass(slots=True) class ServiceStartOutcome: record: ServiceRecord @@ -35,9 +43,13 @@ class ServiceStartDescriptor: log_path: str | None = None public_url: str | None = None health_url: str | None = None + pid_required: bool = True class ServiceManager: + def __init__(self, *, process_runner: object | None = None) -> None: + self.process_runner = process_runner + def start_service_with_retry( self, *, @@ -49,6 +61,7 @@ def start_service_with_retry( reserve_next: Callable[[int], int], detect_actual: Callable[[int | None, int], int | None] | None = None, listener_expected: bool = True, + pid_required: bool = True, max_retries: int = 3, on_retry: Callable[[str, int, int, int, str | None], None] | None = None, ) -> ServiceRecord: @@ -58,6 +71,21 @@ def start_service_with_retry( while True: success, error, pid = start(current_port) if success: + valid_pid = isinstance(pid, int) and not isinstance(pid, bool) and pid > 0 + if pid_required and not valid_pid: + record = _failed_start_record( + project=project, + service_type=service_type, + cwd=cwd, + pid=None, + requested_port=current_port if listener_expected else None, + listener_expected=listener_expected, + failure_detail="service launcher reported success without a valid PID", + ) + raise ServiceCleanupError( + f"Service launcher for {record.name} reported success without a valid PID", + {record.name: record}, + ) try: actual_port: int | None = current_port if listener_expected else None if detect_actual is not None: @@ -66,16 +94,54 @@ def start_service_with_retry( name=f"{project} {service_display_name(service_type)}", type=service_type, cwd=cwd, - pid=pid, + pid=pid if valid_pid else None, requested_port=requested_port if listener_expected else None, actual_port=actual_port, status="running", started_at=time.time(), listener_expected=listener_expected, ) - except RuntimeError as exc: + except BaseException as exc: # noqa: BLE001 - cancellation must not leak the launched PID + cleanup_confirmed = _terminate_pid_safely( + pid, + process_runner=self.process_runner, + ) + if not cleanup_confirmed: + record = _failed_start_record( + project=project, + service_type=service_type, + cwd=cwd, + pid=pid, + requested_port=current_port if listener_expected else None, + listener_expected=listener_expected, + failure_detail=str(exc), + ) + unterminated = {record.name: record} + if not isinstance(exc, Exception): + _retain_unterminated_services(exc, unterminated) + raise + raise ServiceCleanupError( + f"{exc}; cleanup could not confirm exit of PID {pid}", + unterminated, + ) from exc + if not isinstance(exc, Exception): + raise error = str(exc) - _terminate_pid(pid, process_runner=self) + elif isinstance(pid, int) and pid > 0: + if not _terminate_pid(pid, process_runner=self.process_runner): + record = _failed_start_record( + project=project, + service_type=service_type, + cwd=cwd, + pid=pid, + requested_port=current_port if listener_expected else None, + listener_expected=listener_expected, + failure_detail=error or "service launcher reported failure", + ) + raise ServiceCleanupError( + f"{error or 'service launcher reported failure'}; cleanup could not confirm exit of PID {pid}", + {record.name: record}, + ) if retries >= max_retries or not _is_retryable_error(error): raise ServiceStartError( @@ -156,10 +222,13 @@ def start_record(descriptor: ServiceStartDescriptor) -> ServiceRecord: reserve_next=reserve_next, detect_actual=descriptor.detect_actual, listener_expected=descriptor.listener_expected, + pid_required=descriptor.pid_required, max_retries=descriptor.max_retries, on_retry=on_retry, ) except Exception as exc: + if isinstance(exc, ServiceCleanupError): + raise if descriptor.critical: raise record = ServiceRecord( @@ -186,32 +255,44 @@ def start_record(descriptor: ServiceStartDescriptor) -> ServiceRecord: partial_records: list[ServiceRecord] = [] try: if parallel_start and len(descriptors) > 1: - worker_count = max_workers or len(descriptors) - with concurrent.futures.ThreadPoolExecutor(max_workers=min(len(descriptors), worker_count)) as executor: - future_map = { - executor.submit(start_record, descriptor): index - for index, descriptor in enumerate(descriptors) - } - records_by_index: dict[int, ServiceRecord] = {} - failures_by_index: dict[int, Exception] = {} - for future in concurrent.futures.as_completed(future_map): - index = future_map[future] - try: - records_by_index[index] = future.result() - except Exception as exc: # noqa: BLE001 - failures_by_index[index] = exc - - partial_records.extend( - records_by_index[index] for index in sorted(records_by_index) - ) - if failures_by_index: - raise failures_by_index[min(failures_by_index)] + records_by_index, failures_by_index, interruption = _start_parallel_records( + descriptors=descriptors, + start_record=start_record, + max_workers=max_workers, + ) + partial_records.extend(records_by_index[index] for index in sorted(records_by_index)) + primary_failure = _primary_parallel_failure( + failures_by_index, + interruption=interruption, + ) + if primary_failure is not None: + unterminated = _unterminated_services_from_failures(failures_by_index.values()) + if unterminated: + if not isinstance(primary_failure, Exception): + _retain_unterminated_services(primary_failure, unterminated) + raise primary_failure + raise ServiceCleanupError(str(primary_failure), unterminated) from primary_failure + raise primary_failure else: for descriptor in descriptors: partial_records.append(start_record(descriptor)) - except Exception: - for record in partial_records: - _terminate_pid(record.pid, process_runner=self) + except BaseException as exc: # noqa: BLE001 - cancellation still requires transactional rollback + unterminated = _unterminated_services_from_failures((exc,)) + unterminated.update( + _cleanup_service_records( + partial_records, + process_runner=self.process_runner, + ) + ) + if unterminated: + if not isinstance(exc, Exception): + _retain_unterminated_services(exc, unterminated) + raise + names = ", ".join(sorted(unterminated)) + raise ServiceCleanupError( + f"{exc}; cleanup could not confirm exit for: {names}", + unterminated, + ) from exc raise return {record.name: record for record in partial_records} @@ -225,17 +306,156 @@ def _is_retryable_error(error: str | None) -> bool: return False -def _terminate_pid(pid: int | None, *, process_runner: object | None = None) -> None: - if not isinstance(pid, int) or pid <= 0: - return - terminator = getattr(process_runner, "terminate_process_group", None) if process_runner is not None else None - if callable(terminator): +def _terminate_pid(pid: int | None, *, process_runner: object | None = None) -> bool: + return terminate_pid( + pid, + process_runner=process_runner, + term_timeout=2.0, + kill_timeout=1.0, + ) + + +def _terminate_pid_safely(pid: int | None, *, process_runner: object | None = None) -> bool: + try: + return _terminate_pid(pid, process_runner=process_runner) + except BaseException: # noqa: BLE001 - cleanup must not mask the triggering cancellation + return False + + +def _cleanup_service_records( + records: Iterable[ServiceRecord], + *, + process_runner: object | None, +) -> dict[str, ServiceRecord]: + unterminated: dict[str, ServiceRecord] = {} + for record in records: + if not _terminate_pid_safely(record.pid, process_runner=process_runner): + unterminated[record.name] = record + return unterminated + + +def _retain_unterminated_services( + failure: BaseException, + services: dict[str, ServiceRecord], +) -> None: + retained = _unterminated_services_from_failures((failure,)) + retained.update(services) + try: + failure.unterminated_services = retained # type: ignore[attr-defined] + except Exception: # noqa: BLE001 - built-in cancellation types normally allow metadata + failure.add_note("cleanup could not confirm exit for: " + ", ".join(sorted(retained))) + + +def _start_parallel_records( + *, + descriptors: tuple[ServiceStartDescriptor, ...], + start_record: Callable[[ServiceStartDescriptor], ServiceRecord], + max_workers: int | None, +) -> tuple[dict[int, ServiceRecord], dict[int, BaseException], BaseException | None]: + records_by_index: dict[int, ServiceRecord] = {} + failures_by_index: dict[int, BaseException] = {} + future_map: dict[concurrent.futures.Future[ServiceRecord], int] = {} + collected: set[concurrent.futures.Future[ServiceRecord]] = set() + interruption: BaseException | None = None + worker_count = max_workers or len(descriptors) + executor = concurrent.futures.ThreadPoolExecutor(max_workers=min(len(descriptors), worker_count)) + + def collect(future: concurrent.futures.Future[ServiceRecord]) -> None: + collected.add(future) + index = future_map[future] try: - terminator(pid, term_timeout=2.0, kill_timeout=1.0) - return - except Exception: # noqa: BLE001 - pass + records_by_index[index] = future.result() + except BaseException as exc: # noqa: BLE001 - a worker cancellation is transaction data + failures_by_index[index] = exc + try: - os.kill(pid, signal.SIGTERM) - except OSError: - return + try: + for index, descriptor in enumerate(descriptors): + future = executor.submit(start_record, descriptor) + future_map[future] = index + for future in concurrent.futures.as_completed(future_map): + collect(future) + except BaseException as exc: # noqa: BLE001 - main-thread cancellation must drain submitted launches + interruption = exc + for future in future_map: + if future not in collected: + future.cancel() + finally: + try: + executor.shutdown(wait=True, cancel_futures=interruption is not None) + except BaseException as exc: # noqa: BLE001 - preserve the first cancellation and retry teardown + if interruption is None: + interruption = exc + for future in future_map: + future.cancel() + try: + executor.shutdown(wait=True, cancel_futures=True) + except BaseException as shutdown_exc: # noqa: BLE001 - retain diagnostics on the original failure + assert interruption is not None + interruption.add_note(f"parallel executor shutdown failed: {shutdown_exc}") + + for future in future_map: + if future in collected or future.cancelled(): + continue + if future.done(): + collect(future) + return records_by_index, failures_by_index, interruption + + +def _primary_parallel_failure( + failures_by_index: dict[int, BaseException], + *, + interruption: BaseException | None, +) -> BaseException | None: + if interruption is not None: + return interruption + cancellation_indexes = [ + index for index, failure in failures_by_index.items() if not isinstance(failure, Exception) + ] + if cancellation_indexes: + return failures_by_index[min(cancellation_indexes)] + if failures_by_index: + return failures_by_index[min(failures_by_index)] + return None + + +def _failed_start_record( + *, + project: str, + service_type: str, + cwd: str, + pid: int | None, + requested_port: int | None, + listener_expected: bool, + failure_detail: str, +) -> ServiceRecord: + record = ServiceRecord( + name=f"{project} {service_display_name(service_type)}", + type=service_type, + cwd=cwd, + pid=pid, + requested_port=requested_port, + actual_port=None, + status="termination_failed", + started_at=time.time(), + listener_expected=listener_expected, + failure_detail=failure_detail, + degraded=True, + ) + record.project = project + record.service_slug = service_type + return record + + +def _unterminated_services_from_failures( + failures: Iterable[object], +) -> dict[str, ServiceRecord]: + unterminated: dict[str, ServiceRecord] = {} + for failure in failures: + services = getattr(failure, "unterminated_services", None) + if not isinstance(services, dict): + continue + for name, service in services.items(): + if isinstance(name, str) and isinstance(service, ServiceRecord): + unterminated[name] = service + return unterminated diff --git a/python/envctl_engine/runtime/service_status_truth.py b/python/envctl_engine/runtime/service_status_truth.py index c5001a5a3..e2335c245 100644 --- a/python/envctl_engine/runtime/service_status_truth.py +++ b/python/envctl_engine/runtime/service_status_truth.py @@ -6,6 +6,33 @@ def service_truth_status(runtime: Any, service: object) -> str: + if str(getattr(service, "runtime_kind", "process") or "process").lower() == "docker": + from envctl_engine.runtime.docker_service_runtime import DockerServiceRuntime + + container = str( + getattr(service, "container_id", "") or getattr(service, "container_name", "") or "" + ).strip() + docker_runtime = DockerServiceRuntime(runtime, runtime.process_runner) + if not docker_runtime.container_running(container): + status = "stale" + elif not bool(getattr(service, "listener_expected", True)): + status = "running" + else: + port = getattr(service, "actual_port", None) or getattr(service, "requested_port", None) + try: + reachable = isinstance(port, int) and port > 0 and bool( + runtime.process_runner.wait_for_port(port, timeout=runtime._service_truth_timeout()) + ) + except Exception: # noqa: BLE001 + reachable = False + status = "running" if reachable else "unreachable" + runtime._emit( + "service.truth.check", + service=service_display_name(service), + status=status, + reason_code=f"container_truth_status_{status}", + ) + return status backend = getattr(runtime.process_probe, "backend", None) if isinstance(backend, ShellProbeBackend): backend.process_runner = runtime.process_runner diff --git a/python/envctl_engine/runtime/service_truth_diagnostics.py b/python/envctl_engine/runtime/service_truth_diagnostics.py index 094fb23ff..7fcf7b4b5 100644 --- a/python/envctl_engine/runtime/service_truth_diagnostics.py +++ b/python/envctl_engine/runtime/service_truth_diagnostics.py @@ -1,8 +1,11 @@ from __future__ import annotations from pathlib import Path +import re from typing import Any +from envctl_engine.shared.reason_codes import ServiceFailureReason + STARTUP_PROGRESS_TOKENS = ( "Waiting for application startup", @@ -28,6 +31,9 @@ "No module named", "Address already in use", "address already in use", + "command not found", + "exec:", + "No such file or directory", ) @@ -64,6 +70,25 @@ def service_listener_failure_detail(runtime: Any, *, log_path: str | None, pid: return "; ".join(parts) +def service_listener_failure_class(detail: str | None) -> str: + """Classify listener failures using process and log evidence.""" + + text = str(detail or "") + normalized = text.casefold() + missing_executable = bool( + "command not found" in normalized + or "modulenotfounderror" in normalized + or "no module named" in normalized + or re.search(r"\bexec:\s+[^:;\s]+:\s+not found\b", normalized) + or ("/usr/bin/env:" in normalized and "no such file or directory" in normalized) + ) + if missing_executable: + return ServiceFailureReason.DEPENDENCY_MISSING.value + if re.search(r"\bprocess(?:\s+\d+)?\s+exited\b", normalized): + return ServiceFailureReason.PROCESS_EXITED.value + return "listener_not_detected" + + def tail_log_error_line(log_path: str | None, *, max_chars: int = 800) -> str | None: lines = _tail_candidate_lines(log_path) if not lines: @@ -77,6 +102,9 @@ def tail_log_error_line(log_path: str | None, *, max_chars: int = 800) -> str | "Error", "Exception", "No module named", + "command not found", + "exec:", + "No such file or directory", ) selected = lines[-1] selected_index = len(lines) - 1 @@ -115,9 +143,7 @@ def _tail_candidate_lines(log_path: str | None) -> list[str]: return [] try: return [ - line.strip() - for line in path.read_text(encoding="utf-8", errors="replace").splitlines() - if line.strip() + line.strip() for line in path.read_text(encoding="utf-8", errors="replace").splitlines() if line.strip() ] except OSError: return [] diff --git a/python/envctl_engine/runtime/startup_inspection_support.py b/python/envctl_engine/runtime/startup_inspection_support.py index 2d07a1170..656a4298d 100644 --- a/python/envctl_engine/runtime/startup_inspection_support.py +++ b/python/envctl_engine/runtime/startup_inspection_support.py @@ -84,6 +84,9 @@ def build(self) -> dict[str, Any]: payload = { "command": getattr(self.startup_route, "command", "start"), "mode": self.runtime_mode, + "application_runtime": ( + "docker" if bool(getattr(self.startup_route, "flags", {}).get("docker")) else "process" + ), "headless": self.batch or not self.can_tty, "interactive_tty": self.can_tty, "selection": selection_info, diff --git a/python/envctl_engine/runtime/state_fingerprint_support.py b/python/envctl_engine/runtime/state_fingerprint_support.py index 3c16058bd..cb06bea1b 100644 --- a/python/envctl_engine/runtime/state_fingerprint_support.py +++ b/python/envctl_engine/runtime/state_fingerprint_support.py @@ -1,12 +1,8 @@ from __future__ import annotations -import hashlib -import json - -from envctl_engine.state import state_to_dict +from envctl_engine.state.fingerprints import state_fingerprint as _state_fingerprint from envctl_engine.state.models import RunState def state_fingerprint(state: RunState) -> str: - payload = json.dumps(state_to_dict(state), sort_keys=True) - return hashlib.sha256(payload.encode("utf-8")).hexdigest() + return _state_fingerprint(state) diff --git a/python/envctl_engine/runtime/state_inspection_support.py b/python/envctl_engine/runtime/state_inspection_support.py new file mode 100644 index 000000000..260989fcf --- /dev/null +++ b/python/envctl_engine/runtime/state_inspection_support.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +from envctl_engine.runtime.command_router import Route +from envctl_engine.state import state_to_dict +from envctl_engine.state.lookup import call_state_loader +from envctl_engine.state.models import RunState +from envctl_engine.state.project_runtime import ( + cwd_runtime_warnings, + dependency_mode_summary, + project_resolution_event_payload, + resolve_requested_project_state, +) +from envctl_engine.state.runtime_map import build_runtime_map +from envctl_engine.ui.path_links import render_path_for_terminal + +StateLoader = Callable[..., RunState | None] +WarningEmitter = Callable[..., None] + + +def print_state( + runtime: Any, + route: Route, + *, + json_output: bool, + state_loader: StateLoader, + warning_emitter: WarningEmitter, +) -> int: + strict = runtime._state_lookup_strict_mode_match(route) + requested_projects = list(getattr(route, "projects", []) or []) + state = state_loader( + runtime, + mode=route.mode, + strict_mode_match=strict, + project_names=requested_projects or None, + ) + if state is None and requested_projects: + # A targeted repository lookup correctly returns no owner for an + # unknown project. Load the mode-wide aggregate once so the command can + # distinguish that case from an entirely missing runtime and return the + # structured requested_project_not_running contract. + state = state_loader( + runtime, + mode=route.mode, + strict_mode_match=strict, + project_names=None, + ) + if state is None: + if json_output: + print(json.dumps({"found": False, "mode": getattr(route, "mode", None)}, indent=2, sort_keys=True)) + else: + print("No previous state found.") + return 1 + if requested_projects: + resolution = resolve_requested_project_state( + state, + requested_projects, + command="show-state", + runtime=runtime, + allow_multi=False, + ) + if not resolution.ok: + _emit_project_resolution(runtime, "state.project_resolution.failed", resolution, state) + if json_output: + print(json.dumps(resolution.payload(), indent=2, sort_keys=True)) + else: + print(f"Requested project is not running: {resolution.requested_project}") + print("Active runtime projects: " + (", ".join(resolution.active_projects) or "none")) + return 1 + _emit_project_resolution(runtime, "state.project_resolution.ok", resolution, state) + if resolution.state is not None and not bool(getattr(route, "flags", {}).get("all")): + state = resolution.state + + dependency_summary = dependency_mode_summary(state) + cwd_project, warnings = cwd_runtime_warnings( + state, + runtime=runtime, + requested_projects=list(getattr(route, "projects", []) or []), + ) + warning_emitter(runtime, command="show-state", state=state, warnings=warnings) + payload: dict[str, Any] = { + "found": True, + "runtime_root": str(runtime.runtime_root), + "run_state_path": str(runtime._run_state_path()), + "state": state_to_dict(state), + "runtime_map": build_runtime_map(state), + "dependency_mode": dependency_summary["dependency_mode"], + "shared_dependencies": dependency_summary["shared_dependencies"], + "cwd_project": cwd_project, + "warnings": warnings, + } + if json_output: + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + + state_payload = payload["state"] + print(f"run_id: {state_payload['run_id']}") + print(f"mode: {state_payload['mode']}") + print(f"services: {len(state_payload['services'])}") + print(f"requirements: {len(state_payload['requirements'])}") + for warning in warnings: + print(f"Warning: {warning['message']}") + print("run_state_path:") + print(render_path_for_terminal(payload["run_state_path"], env=getattr(runtime, "env", {}))) + return 0 + + +def try_load_existing_state( + runtime: Any, + *, + mode: str | None, + strict_mode_match: bool, + project_names: list[str] | None = None, +) -> RunState | None: + loader = getattr(runtime, "_try_load_existing_state", None) + if not callable(loader): + return None + state = call_state_loader( + loader, + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ) + return state if isinstance(state, RunState) else None + + +def emit_cwd_runtime_mismatch_warnings( + runtime: Any, + *, + command: str, + state: RunState, + warnings: list[dict[str, object]], +) -> None: + emitter = getattr(runtime, "_emit", None) + if not callable(emitter): + return + for warning in warnings: + if str(warning.get("code") or "") != "cwd_runtime_mismatch": + continue + emitter( + "state.cwd_runtime_mismatch", + run_id=getattr(state, "run_id", ""), + mode=getattr(state, "mode", ""), + command=command, + cwd_project=warning.get("cwd_project"), + active_projects=warning.get("active_projects"), + ) + + +def _emit_project_resolution(runtime: Any, event: str, resolution: Any, state: RunState) -> None: + emitter = getattr(runtime, "_emit", None) + if callable(emitter): + emitter( + event, + **project_resolution_event_payload(resolution, state, runtime=runtime), + ) diff --git a/python/envctl_engine/runtime_feature_lifecycle_command_definitions.py b/python/envctl_engine/runtime_feature_lifecycle_command_definitions.py index 76fc32727..e803bbd16 100644 --- a/python/envctl_engine/runtime_feature_lifecycle_command_definitions.py +++ b/python/envctl_engine/runtime_feature_lifecycle_command_definitions.py @@ -82,6 +82,7 @@ python_source_of_truth=( "python/envctl_engine/runtime/engine_runtime.py", "python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py", + "python/envctl_engine/runtime/lifecycle_full_cleanup.py", ), evidence_tests=("tests/python/runtime/test_lifecycle_parity_stop_health.py",), parity_status="verified_python", @@ -95,6 +96,7 @@ python_source_of_truth=( "python/envctl_engine/runtime/engine_runtime.py", "python/envctl_engine/runtime/engine_runtime_lifecycle_support.py", + "python/envctl_engine/runtime/lifecycle_full_cleanup.py", "python/envctl_engine/runtime/lifecycle_blast_support.py", "python/envctl_engine/runtime/lifecycle_blast_ports.py", "python/envctl_engine/runtime/lifecycle_blast_processes.py", diff --git a/python/envctl_engine/shared/git_branch_support.py b/python/envctl_engine/shared/git_branch_support.py new file mode 100644 index 000000000..0b4efe602 --- /dev/null +++ b/python/envctl_engine/shared/git_branch_support.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import dataclass + + +PREFERRED_PR_BASE_BRANCHES = ("dev", "main", "master") + + +@dataclass(frozen=True, slots=True) +class _RemoteBranchInventory: + preferred_heads: frozenset[str] + default_branch: str | None + reachable: bool + + +def detect_preferred_default_branch( + git_output: Callable[[Sequence[str]], str], + *, + probe_remote_heads: bool = False, +) -> str: + """Resolve the PR base contract before consulting a repository's remote HEAD.""" + + tracked_remote_branches: set[str] = set() + for branch in PREFERRED_PR_BASE_BRANCHES: + if git_output(("rev-parse", "--verify", f"refs/remotes/origin/{branch}")).strip(): + tracked_remote_branches.add(branch) + + remote_inventory = _remote_branch_inventory(git_output) if probe_remote_heads else None + # A successful live inventory is authoritative over local tracking refs, + # including when none of the preferred branches exist. When the probe is + # unavailable or times out, retain the offline tracking-ref fallback. + available_branches = ( + remote_inventory.preferred_heads + if remote_inventory is not None and remote_inventory.reachable + else tracked_remote_branches + ) + for branch in PREFERRED_PR_BASE_BRANCHES: + if branch in available_branches: + return branch + if remote_inventory is not None and remote_inventory.reachable and remote_inventory.default_branch: + return remote_inventory.default_branch + + remote_head = git_output(("symbolic-ref", "--short", "refs/remotes/origin/HEAD")).strip() + if remote_head.startswith("origin/"): + return remote_head.removeprefix("origin/") + for branch in PREFERRED_PR_BASE_BRANCHES: + if git_output(("rev-parse", "--verify", f"refs/heads/{branch}")).strip(): + return branch + return "main" + + +def _remote_branch_inventory( + git_output: Callable[[Sequence[str]], str], +) -> _RemoteBranchInventory: + output = git_output( + ( + "ls-remote", + "--symref", + "origin", + "HEAD", + *(f"refs/heads/{branch}" for branch in PREFERRED_PR_BASE_BRANCHES), + ) + ) + heads: set[str] = set() + default_branch: str | None = None + for raw_line in output.splitlines(): + fields = raw_line.split() + if len(fields) == 3 and fields[0] == "ref:" and fields[2] == "HEAD": + ref = fields[1] + if ref.startswith("refs/heads/"): + default_branch = ref.removeprefix("refs/heads/") or None + continue + if len(fields) != 2 or not fields[1].startswith("refs/heads/"): + continue + branch = fields[1].removeprefix("refs/heads/") + if branch in PREFERRED_PR_BASE_BRANCHES: + heads.add(branch) + return _RemoteBranchInventory( + preferred_heads=frozenset(heads), + default_branch=default_branch, + reachable=bool(output.strip()), + ) + + +__all__ = ["PREFERRED_PR_BASE_BRANCHES", "detect_preferred_default_branch"] diff --git a/python/envctl_engine/shared/ports.py b/python/envctl_engine/shared/ports.py index 76f5f4fa0..783705ddc 100644 --- a/python/envctl_engine/shared/ports.py +++ b/python/envctl_engine/shared/ports.py @@ -1,18 +1,24 @@ from __future__ import annotations +import errno import hashlib import json +import math import os import shutil import socket import subprocess +import threading import time import uuid +from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path from typing import Final from typing import Callable from envctl_engine.state.models import PortPlan +from envctl_engine.state.persistence import advisory_file_lock, fsync_directory _CORE_SERVICE_NAMES: Final[frozenset[str]] = frozenset( @@ -26,6 +32,12 @@ } ) +_PORT_GUARD_SHARDS: Final[int] = 64 +# ``flock`` behavior for separate descriptors in one process differs across +# platforms. A bounded in-process stripe set makes the contract explicit while +# the on-disk guard shards serialize independent processes. +_PORT_THREAD_GUARDS: Final[tuple[threading.RLock, ...]] = tuple(threading.RLock() for _ in range(_PORT_GUARD_SHARDS)) + class PortPlanner: def __init__( @@ -41,6 +53,7 @@ def __init__( lock_dir: str | None = None, session_id: str | None = None, stale_lock_seconds: int = 3600, + corrupt_lock_grace_seconds: float = 1.0, availability_checker: Callable[[int], bool] | None = None, pid_checker: Callable[[int], bool] | None = None, time_provider: Callable[[], float] | None = None, @@ -59,10 +72,11 @@ def __init__( self.supabase_api_base = supabase_api_base self.additional_service_bases = self._normalized_additional_service_bases(additional_service_bases or {}) self.lock_dir = Path(lock_dir or "/tmp/envctl-python-locks") - self.lock_dir.mkdir(parents=True, exist_ok=True) + self._ensure_real_directory(self.lock_dir) self.session_id = session_id or f"session-{uuid.uuid4().hex[:8]}" self.session_pid = os.getpid() self.stale_lock_seconds = max(stale_lock_seconds, 0) + self.corrupt_lock_grace_seconds = max(float(corrupt_lock_grace_seconds), 0.0) self.availability_checker = availability_checker self.pid_checker = pid_checker self.time_provider = time_provider or time.time @@ -75,6 +89,8 @@ def __init__( self.scope_key = str(scope_key or "global").strip() or "global" self.dynamic_main_dependency_ports = bool(dynamic_main_dependency_ports) self.max_port: Final[int] = 65000 + self._guard_dir = self.lock_dir / ".port-guards" + self._thread_guard_namespace = str(self.lock_dir.resolve(strict=False)) def plan_project( self, @@ -196,57 +212,83 @@ def attach_existing_port(self, plan: PortPlan, existing_port: int) -> PortPlan: def release(self, port: int, owner: str | None = None) -> None: lock_path = self._lock_path(port) - if not lock_path.exists(): - return - if owner is None: - lock_path.unlink(missing_ok=True) - self._emit("port.lock.release", port=port, owner=None, session=self.session_id) - return - try: - payload = json.loads(lock_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): + with self._port_guard(port): + payload = self._read_lock_payload(lock_path) + if payload is None or payload.get("session") != self.session_id: + return + if owner is not None and payload.get("owner") != owner: + return lock_path.unlink(missing_ok=True) self._emit("port.lock.release", port=port, owner=owner, session=self.session_id) - return - if payload.get("owner") == owner: + + def release_owned(self, port: int, owner: str, *, expected_session: str) -> bool: + """Release an exact logical owner from one persisted planner session.""" + + normalized_session = expected_session.strip() if isinstance(expected_session, str) else "" + if not normalized_session: + return False + + lock_path = self._lock_path(port) + with self._port_guard(port): + payload = self._read_lock_payload(lock_path) + if payload is None or payload.get("owner") != owner or payload.get("session") != normalized_session: + return False lock_path.unlink(missing_ok=True) - self._emit("port.lock.release", port=port, owner=owner, session=self.session_id) + self._emit("port.lock.release", port=port, owner=owner, session=normalized_session) + return True + + def reap_stale(self, port: int, owner: str | None = None) -> bool: + """Remove one proven-stale reservation without touching a live foreign session.""" + + lock_path = self._lock_path(port) + with self._port_guard(port): + if not lock_path.exists() or not self._lock_is_stale(lock_path): + return False + payload = self._read_lock_payload(lock_path) + if owner is not None and payload is not None and payload.get("owner") != owner: + return False + lock_path.unlink(missing_ok=True) + self._emit_reclaim(port=port, owner=owner, payload=payload) + return True + + def reap_stale_locks(self) -> int: + """Reap every proven-stale numeric reservation in this repository scope.""" + + reaped = 0 + for lock_path in tuple(self.lock_dir.glob("*.lock")): + port = self._port_from_lock_path(lock_path) + if port is not None and self.reap_stale(port): + reaped += 1 + return reaped def release_session(self) -> None: for lock_path in self.lock_dir.glob("*.lock"): - try: - payload = json.loads(lock_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): + port = self._port_from_lock_path(lock_path) + if port is None: + continue + with self._port_guard(port): + payload = self._read_lock_payload(lock_path) + if payload is None or payload.get("session") != self.session_id: + continue + lock_path.unlink(missing_ok=True) + self._emit("port.lock.release", port=port, owner=payload.get("owner"), session=self.session_id) + + def release_all(self) -> None: + for lock_path in self.lock_dir.glob("*.lock"): + port = self._port_from_lock_path(lock_path) + if port is None: lock_path.unlink(missing_ok=True) - self._emit( - "port.lock.release", port=self._port_from_lock_path(lock_path), owner=None, session=self.session_id - ) continue - if payload.get("session") == self.session_id: + with self._port_guard(port): + payload = self._read_lock_payload(lock_path) lock_path.unlink(missing_ok=True) self._emit( "port.lock.release", - port=self._port_from_lock_path(lock_path), - owner=payload.get("owner"), + port=port, + owner=payload.get("owner") if payload is not None else None, session=self.session_id, ) - def release_all(self) -> None: - for lock_path in self.lock_dir.glob("*.lock"): - owner = None - try: - payload = json.loads(lock_path.read_text(encoding="utf-8")) - owner = payload.get("owner") - except (OSError, json.JSONDecodeError): - owner = None - lock_path.unlink(missing_ok=True) - self._emit( - "port.lock.release", - port=self._port_from_lock_path(lock_path), - owner=owner, - session=self.session_id, - ) - def _lock_path(self, port: int) -> Path: return self.lock_dir / f"{port}.lock" @@ -370,87 +412,161 @@ def _normalize_project_identity(project: str) -> str: def _reserve_port(self, port: int, owner: str) -> bool: lock_path = self._lock_path(port) - if lock_path.exists() and self._lock_is_stale(lock_path): - stale_owner = None - stale_session = None - stale_pid = None - try: - stale_payload = json.loads(lock_path.read_text(encoding="utf-8")) - if isinstance(stale_payload, dict): - stale_owner = stale_payload.get("owner") - stale_session = stale_payload.get("session") - stale_pid = stale_payload.get("pid") - except (OSError, json.JSONDecodeError): - stale_owner = None - stale_session = None - stale_pid = None - lock_path.unlink(missing_ok=True) - self._emit( - "port.lock.reclaim", - port=port, - owner=owner, - session=self.session_id, - reclaimed_owner=stale_owner, - reclaimed_session=stale_session, - reclaimed_pid=stale_pid, - ) - if lock_path.exists(): - return False + with self._port_guard(port): + availability_confirmed = False + if lock_path.exists(): + if not self._lock_is_stale(lock_path): + return False + stale_payload = self._read_lock_payload(lock_path) + if stale_payload is None or stale_payload.get("owner") != owner: + # A targeted startup may reclaim a dead allocator session + # only for the same logical project/service. A dead or + # corrupt foreign lock is still durable ownership evidence + # and requires an explicit all-scope cleanup operation. + return False + # The allocator process recorded in an otherwise stale lock + # can exit while its launched service is still listening. Do + # not erase that last ownership record merely because the + # allocator PID is gone. + if not self._is_port_available(port): + return False + availability_confirmed = True + lock_path.unlink(missing_ok=True) + self._emit_reclaim(port=port, owner=owner, payload=stale_payload) - if not self._is_port_available(port): - return False + if not availability_confirmed and not self._is_port_available(port): + return False - payload = { - "owner": owner, - "session": self.session_id, - "pid": self.session_pid, - "created_at": self.time_provider(), - } - try: - fd = lock_path.open("x", encoding="utf-8") - except FileExistsError: - return False - with fd: - fd.write(json.dumps(payload, sort_keys=True)) - self._emit("port.lock.acquire", port=port, owner=owner, session=self.session_id) - return True + payload = { + "owner": owner, + "session": self.session_id, + "pid": self.session_pid, + "created_at": self.time_provider(), + } + if not self._publish_lock(lock_path, payload, port=port): + return False + self._emit("port.lock.acquire", port=port, owner=owner, session=self.session_id) + return True def _lock_is_stale(self, lock_path: Path) -> bool: - try: - payload = json.loads(lock_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return True + payload = self._read_lock_payload(lock_path) + if payload is None: + return self._corrupt_lock_is_stale(lock_path) pid = payload.get("pid") - if isinstance(pid, int) and pid > 0: + if isinstance(pid, int) and not isinstance(pid, bool) and pid > 0: if self._pid_is_running(pid): return False return True created_at_raw = payload.get("created_at") if created_at_raw is None: - return False + return self._corrupt_lock_is_stale(lock_path) created_at = self._parse_created_at(created_at_raw) if created_at is None: - return True + return self._corrupt_lock_is_stale(lock_path) return (self.time_provider() - created_at) > self.stale_lock_seconds def _pid_is_running(self, pid: int) -> bool: if self.pid_checker is not None: - return self.pid_checker(pid) + try: + return bool(self.pid_checker(pid)) + except Exception: # noqa: BLE001 - an inconclusive probe must preserve a foreign lock + return True try: os.kill(pid, 0) - except OSError: + except ProcessLookupError: return False + except PermissionError: + return True + except OSError as exc: + return exc.errno != errno.ESRCH return True + def _corrupt_lock_is_stale(self, lock_path: Path) -> bool: + try: + modified_at = lock_path.stat().st_mtime + except FileNotFoundError: + return True + except OSError: + return False + return (self.time_provider() - modified_at) >= self.corrupt_lock_grace_seconds + + @staticmethod + def _read_lock_payload(lock_path: Path) -> dict[str, object] | None: + try: + payload = json.loads(lock_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + return payload if isinstance(payload, dict) else None + + def _publish_lock(self, lock_path: Path, payload: dict[str, object], *, port: int) -> bool: + """Publish a complete payload without ever exposing an empty lock path.""" + + self._ensure_real_directory(self._guard_dir) + pending_path = self._guard_dir / f"{self._guard_shard(port):02x}.pending" + pending_path.unlink(missing_ok=True) + flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(pending_path, flags, 0o600) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as pending_file: + pending_file.write(json.dumps(payload, sort_keys=True)) + pending_file.flush() + os.fsync(pending_file.fileno()) + try: + os.link(pending_path, lock_path) + except FileExistsError: + return False + fsync_directory(self.lock_dir) + return True + finally: + pending_path.unlink(missing_ok=True) + + def _emit_reclaim(self, *, port: int, owner: str | None, payload: dict[str, object] | None) -> None: + payload = payload or {} + self._emit( + "port.lock.reclaim", + port=port, + owner=owner, + session=self.session_id, + reclaimed_owner=payload.get("owner"), + reclaimed_session=payload.get("session"), + reclaimed_pid=payload.get("pid"), + ) + + @contextmanager + def _port_guard(self, port: int) -> Iterator[None]: + shard = self._guard_shard(port) + self._ensure_real_directory(self._guard_dir) + guard_path = self._guard_dir / f"{shard:02x}.guard" + thread_key = f"{self._thread_guard_namespace}:{shard}".encode("utf-8") + thread_index = int(hashlib.sha1(thread_key).hexdigest()[:8], 16) % len(_PORT_THREAD_GUARDS) + with _PORT_THREAD_GUARDS[thread_index]: + with advisory_file_lock(guard_path, exclusive=True): + yield + + @staticmethod + def _guard_shard(port: int) -> int: + return int(port) % _PORT_GUARD_SHARDS + + @staticmethod + def _ensure_real_directory(path: Path) -> None: + try: + path.mkdir(parents=True, exist_ok=False) + except FileExistsError: + pass + if path.is_symlink() or not path.is_dir(): + raise OSError(f"Port reservation directory is not a real directory: {path}") + def _parse_created_at(self, value: object) -> float | None: - if isinstance(value, (int, float)): - return float(value) + if isinstance(value, (int, float)) and not isinstance(value, bool): + parsed = float(value) + return parsed if math.isfinite(parsed) else None if not isinstance(value, str): return None try: - return float(value) + parsed = float(value) + return parsed if math.isfinite(parsed) else None except ValueError: pass iso = value @@ -459,8 +575,9 @@ def _parse_created_at(self, value: object) -> float | None: try: from datetime import datetime - return datetime.fromisoformat(iso).timestamp() - except ValueError: + parsed = datetime.fromisoformat(iso).timestamp() + return parsed if math.isfinite(parsed) else None + except (OverflowError, ValueError): return None def _is_port_available(self, port: int) -> bool: @@ -545,7 +662,10 @@ def _is_port_available_via_docker_publish_filter(self, port: int) -> bool: def _emit(self, event_name: str, **payload: object) -> None: if self.event_handler is None: return - self.event_handler(event_name, payload) + try: + self.event_handler(event_name, payload) + except Exception: # noqa: BLE001 - telemetry must never change lock ownership + return @staticmethod def _port_from_lock_path(lock_path: Path) -> int | None: diff --git a/python/envctl_engine/shared/process_cwd.py b/python/envctl_engine/shared/process_cwd.py new file mode 100644 index 000000000..5b2668a47 --- /dev/null +++ b/python/envctl_engine/shared/process_cwd.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import os +import subprocess +from collections.abc import Callable, Sequence +from typing import Any + + +def process_cwd( + pid: int, + *, + run_probe: Callable[[Sequence[str]], Any] | None = None, +) -> str | None: + """Read a process CWD portably, returning only explicit probe evidence.""" + + if pid <= 0: + return None + try: + return os.readlink(f"/proc/{pid}/cwd") + except OSError: + pass + + command = ["lsof", "-a", "-p", str(pid), "-d", "cwd", "-Fn"] + try: + completed = ( + run_probe(command) + if run_probe is not None + else subprocess.run( # noqa: S603 + command, + text=True, + encoding="utf-8", + errors="replace", + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=1.0, + check=False, + ) + ) + except (OSError, subprocess.TimeoutExpired): + return None + try: + returncode = int(getattr(completed, "returncode", 1)) + except (TypeError, ValueError): + return None + if returncode != 0: + return None + return parse_lsof_cwd(str(getattr(completed, "stdout", "") or "")) + + +def parse_lsof_cwd(output: str) -> str | None: + for line in output.splitlines(): + if line.startswith("n") and len(line) > 1: + return line[1:] + return None diff --git a/python/envctl_engine/shared/process_launch_support.py b/python/envctl_engine/shared/process_launch_support.py index 10a684383..f57ae247e 100644 --- a/python/envctl_engine/shared/process_launch_support.py +++ b/python/envctl_engine/shared/process_launch_support.py @@ -1,6 +1,8 @@ from __future__ import annotations import hashlib +import os +import signal import subprocess import time from dataclasses import dataclass @@ -23,6 +25,20 @@ class LaunchRecord: launched_at_monotonic: float +class ProcessHandoffCleanupError(RuntimeError): + """A spawned process whose exit could not be confirmed after handoff failed.""" + + def __init__(self, *, process: object, launch_intent: str) -> None: + raw_pid = getattr(process, "pid", None) + self.pid = raw_pid if isinstance(raw_pid, int) and not isinstance(raw_pid, bool) and raw_pid > 0 else None + self.process = process + self.launch_intent = launch_intent + identity = f"PID {self.pid}" if self.pid is not None else "an unknown PID" + super().__init__( + f"{launch_intent} failed and cleanup could not confirm exit of {identity}" + ) + + def _hash_command(command: Sequence[str]) -> tuple[str, int]: rendered = "\0".join(str(part) for part in command) return hashlib.sha256(rendered.encode("utf-8", errors="ignore")).hexdigest(), len(rendered) @@ -40,6 +56,126 @@ def _stdio_policy_name(target: object, *, inherited_label: str) -> str: return "file" +def _process_exited(process: object, *, timeout: float) -> bool: + wait = getattr(process, "wait", None) + if callable(wait): + try: + wait(timeout=max(timeout, 0.0)) + return True + except subprocess.TimeoutExpired: + return False + except (ChildProcessError, ProcessLookupError): + return True + except BaseException: # noqa: BLE001 - cleanup must preserve the original failure + pass + poll = getattr(process, "poll", None) + if not callable(poll): + return False + try: + return poll() is not None + except BaseException: # noqa: BLE001 - cleanup must preserve the original failure + return False + + +def _process_pid(process: object) -> int | None: + pid = getattr(process, "pid", None) + return ( + pid + if ( + isinstance(pid, int) + and not isinstance(pid, bool) + and pid > 0 + and pid not in {os.getpid(), os.getppid()} + ) + else None + ) + + +def _process_group_exists(pid: int) -> bool | None: + try: + os.killpg(pid, 0) + return True + except ProcessLookupError: + return False + except PermissionError: + return True + except BaseException: # noqa: BLE001 - cleanup must preserve the original failure + return None + + +def _wait_for_process_group_exit(pid: int, *, timeout: float) -> bool: + deadline = time.monotonic() + max(timeout, 0.0) + while True: + exists = _process_group_exists(pid) + if exists is False: + return True + if exists is None or time.monotonic() >= deadline: + return False + try: + time.sleep(min(0.01, max(deadline - time.monotonic(), 0.0))) + except BaseException: # noqa: BLE001 - cancellation cannot abandon authority + return False + + +def _signal_spawned_process( + process: object, + sig: signal.Signals, + *, + process_group: bool, +) -> None: + pid = _process_pid(process) + if process_group and pid is not None: + try: + os.killpg(pid, sig) + return + except BaseException: # noqa: BLE001 - fall back to the Popen handle + pass + method_name = "terminate" if sig == signal.SIGTERM else "kill" + method = getattr(process, method_name, None) + if callable(method): + try: + method() + except BaseException: # noqa: BLE001 - cleanup must preserve the original failure + pass + + +def _terminate_unhanded_process( + process: object, + *, + term_timeout: float = 0.5, + kill_timeout: float = 1.0, + process_group: bool = True, +) -> bool: + """Best-effort terminate and reap a spawned process whose handoff failed.""" + + pid = _process_pid(process) + process_exited = _process_exited(process, timeout=0.0) + group_exited = ( + _process_group_exists(pid) is False + if process_group and pid is not None + else True + ) + if process_exited and group_exited: + return True + _signal_spawned_process(process, signal.SIGTERM, process_group=process_group) + process_exited = _process_exited(process, timeout=term_timeout) + group_exited = ( + _process_group_exists(pid) is False + if process_group and pid is not None + else True + ) + if process_exited and group_exited: + return True + _signal_spawned_process(process, signal.SIGKILL, process_group=process_group) + process_exited = _process_exited(process, timeout=kill_timeout) + group_exited = ( + _wait_for_process_group_exit(pid, timeout=kill_timeout) + if process_group and pid is not None + else True + ) + return process_exited and group_exited + + class ProcessLaunchMixin: _emit: Callable[..., Any] | None _launch_records: list[LaunchRecord] @@ -56,6 +192,7 @@ def _record_launch( stderr_policy: str, controller_input_owner_allowed: bool, active: bool, + emit_event: bool = True, ) -> None: command_hash, command_length = _hash_command(command) record = LaunchRecord( @@ -72,7 +209,7 @@ def _record_launch( launched_at_monotonic=time.monotonic(), ) self._launch_records.append(record) - if callable(self._emit): + if emit_event and callable(self._emit): self._emit( "process.launch", component="shared.process_runner", @@ -88,6 +225,15 @@ def _record_launch( active=bool(active), ) + def _mark_launch_inactive(self, process: object) -> None: + pid = getattr(process, "pid", None) + if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0: + return + for record in reversed(self._launch_records): + if record.pid == pid and record.active: + record.active = False + return + @staticmethod def _prepare_log_targets( *, @@ -135,6 +281,7 @@ def start_background( stdout_path=stdout_path, stderr_path=stderr_path, ) + process: subprocess.Popen[str] | None = None try: process = subprocess.Popen( list(cmd), @@ -160,11 +307,21 @@ def start_background( active=True, ) return process + except BaseException as exc: # noqa: BLE001 - a failed handoff must not orphan the child + if process is not None: + if _terminate_unhanded_process(process): + self._mark_launch_inactive(process) + else: + raise ProcessHandoffCleanupError( + process=process, + launch_intent="background_service", + ) from exc + raise finally: for handle in opened_handles: try: handle.close() - except OSError: + except BaseException: # noqa: BLE001 - diagnostics cleanup cannot orphan the child continue def start( @@ -194,29 +351,41 @@ def start_interactive_child( stdout: int | TextIO | None = None, stderr: int | TextIO | None = None, ) -> subprocess.Popen[str]: - process = subprocess.Popen( - list(cmd), - cwd=str(cwd) if cwd is not None else None, - env=dict(env) if env is not None else None, - text=True, - encoding="utf-8", - errors="replace", - stdin=stdin, - stdout=stdout, - stderr=stderr, - ) - self._record_launch( - launch_intent="interactive_child", - command=cmd, - pid=getattr(process, "pid", None), - cwd=cwd, - stdin_policy=_stdio_policy_name(stdin, inherited_label="inherit"), - stdout_policy=_stdio_policy_name(stdout, inherited_label="inherit"), - stderr_policy=_stdio_policy_name(stderr, inherited_label="inherit"), - controller_input_owner_allowed=True, - active=True, - ) - return process + process: subprocess.Popen[str] | None = None + try: + process = subprocess.Popen( + list(cmd), + cwd=str(cwd) if cwd is not None else None, + env=dict(env) if env is not None else None, + text=True, + encoding="utf-8", + errors="replace", + stdin=stdin, + stdout=stdout, + stderr=stderr, + ) + self._record_launch( + launch_intent="interactive_child", + command=cmd, + pid=getattr(process, "pid", None), + cwd=cwd, + stdin_policy=_stdio_policy_name(stdin, inherited_label="inherit"), + stdout_policy=_stdio_policy_name(stdout, inherited_label="inherit"), + stderr_policy=_stdio_policy_name(stderr, inherited_label="inherit"), + controller_input_owner_allowed=True, + active=True, + ) + return process + except BaseException as exc: # noqa: BLE001 - a failed handoff must not orphan the child + if process is not None: + if _terminate_unhanded_process(process, process_group=False): + self._mark_launch_inactive(process) + else: + raise ProcessHandoffCleanupError( + process=process, + launch_intent="interactive_child", + ) from exc + raise def tracked_launches(self, *, active_only: bool = False) -> list[dict[str, object]]: launches: list[dict[str, object]] = [] diff --git a/python/envctl_engine/shared/process_lifecycle_probe.py b/python/envctl_engine/shared/process_lifecycle_probe.py index f72a7c4ed..018d5a238 100644 --- a/python/envctl_engine/shared/process_lifecycle_probe.py +++ b/python/envctl_engine/shared/process_lifecycle_probe.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable +import errno import os import re import shutil @@ -8,6 +9,8 @@ import socket import time +from envctl_engine.shared.process_cwd import process_cwd as read_process_cwd + class ProcessLifecycleProbeMixin: def is_pid_running(self, pid: int) -> bool: @@ -15,8 +18,12 @@ def is_pid_running(self, pid: int) -> bool: return False try: os.kill(pid, 0) - except OSError: + except ProcessLookupError: return False + except PermissionError: + return True + except OSError as exc: + return exc.errno != errno.ESRCH identity = self._pid_identity(pid) if identity is not None: try: @@ -29,6 +36,9 @@ def is_pid_running(self, pid: int) -> bool: return False return True + def process_cwd(self, pid: int) -> str | None: + return read_process_cwd(pid, run_probe=self.run_probe) + def wait_for_port(self, port: int, *, host: str = "127.0.0.1", timeout: float = 30.0) -> bool: if port <= 0: return False @@ -311,7 +321,7 @@ def _terminate_with_signal_sender( try: signal_sender(pid, signal.SIGTERM) except OSError: - return True + return not self.is_pid_running(pid) deadline = time.monotonic() + max(term_timeout, 0.0) while time.monotonic() < deadline: @@ -330,7 +340,7 @@ def _terminate_with_signal_sender( return True signal_sender(pid, signal.SIGKILL) except OSError: - return True + return not self.is_pid_running(pid) kill_deadline = time.monotonic() + max(kill_timeout, 0.0) while time.monotonic() < kill_deadline: @@ -356,12 +366,100 @@ def terminate(self, pid: int, *, term_timeout: float = 2.0, kill_timeout: float ) def terminate_process_group(self, pid: int, *, term_timeout: float = 2.0, kill_timeout: float = 1.0) -> bool: - return self._terminate_with_signal_sender( - pid, - term_timeout=term_timeout, - kill_timeout=kill_timeout, - signal_sender=os.killpg, - ) + if pid <= 0: + return True + try: + process_group_id = os.getpgid(pid) + except ProcessLookupError: + return not self.is_pid_running(pid) + except (PermissionError, OSError): + return False + if process_group_id <= 0: + return False + if process_group_id == os.getpgrp(): + return False + + initial_members = self._process_group_member_pids(process_group_id) + initial_identities = {member_pid: self._pid_identity(member_pid) for member_pid in initial_members or ()} + try: + os.killpg(process_group_id, signal.SIGTERM) + except OSError: + return not self._process_group_is_running(process_group_id) and not self.is_pid_running(pid) + + if self._wait_for_process_group_exit(process_group_id, timeout=term_timeout): + return True + if not self._process_group_escalation_is_safe( + process_group_id, + initial_members=initial_members, + initial_identities=initial_identities, + ): + return False + try: + os.killpg(process_group_id, signal.SIGKILL) + except OSError: + return not self._process_group_is_running(process_group_id) and not self.is_pid_running(pid) + return self._wait_for_process_group_exit(process_group_id, timeout=kill_timeout) + + def _process_group_member_pids(self, process_group_id: int) -> set[int] | None: + try: + completed = self.run_probe(["ps", "-axo", "pid=,pgid=,stat="]) + except OSError: + return None + if completed.returncode != 0: + return None + + members: set[int] = set() + for line in completed.stdout.splitlines(): + parts = line.strip().split(maxsplit=2) + if len(parts) != 3 or not parts[0].isdigit() or not parts[1].isdigit(): + continue + if int(parts[1]) != process_group_id or parts[2].upper().startswith("Z"): + continue + members.add(int(parts[0])) + return members + + def _process_group_is_running(self, process_group_id: int) -> bool: + members = self._process_group_member_pids(process_group_id) + if members is not None: + return bool(members) + try: + os.killpg(process_group_id, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError as exc: + return exc.errno != errno.ESRCH + return True + + def _wait_for_process_group_exit(self, process_group_id: int, *, timeout: float) -> bool: + deadline = time.monotonic() + max(timeout, 0.0) + while True: + if not self._process_group_is_running(process_group_id): + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + time.sleep(min(0.05, remaining)) + + def _process_group_escalation_is_safe( + self, + process_group_id: int, + *, + initial_members: set[int] | None, + initial_identities: dict[int, str | None], + ) -> bool: + if process_group_id == os.getpgrp(): + return False + current_members = self._process_group_member_pids(process_group_id) + if current_members is None or initial_members is None: + return self._process_group_is_running(process_group_id) + for member_pid in initial_members.intersection(current_members): + initial_identity = initial_identities.get(member_pid) + current_identity = self._pid_identity(member_pid) + if initial_identity is None or current_identity is None or current_identity == initial_identity: + return True + return False def _pid_identity(self, pid: int) -> str | None: if pid <= 0: diff --git a/python/envctl_engine/shared/process_runner.py b/python/envctl_engine/shared/process_runner.py index 06b15a16f..b525d4f98 100644 --- a/python/envctl_engine/shared/process_runner.py +++ b/python/envctl_engine/shared/process_runner.py @@ -7,12 +7,14 @@ import subprocess import time as time from pathlib import Path -from typing import Any, Callable, Mapping, Sequence, TextIO +from typing import Any, Callable, Mapping, NoReturn, Sequence, TextIO from envctl_engine.shared.process_launch_support import ( LaunchRecord as LaunchRecord, + ProcessHandoffCleanupError as ProcessHandoffCleanupError, ProcessLaunchMixin, _hash_command as _hash_command, _stdio_policy_name as _stdio_policy_name, + _terminate_unhanded_process, ) from envctl_engine.shared.process_lifecycle_probe import ProcessLifecycleProbeMixin from envctl_engine.shared.process_streaming_support import ProcessStreamingMixin @@ -25,6 +27,35 @@ def __init__(self, *, emit: Callable[..., Any] | None = None) -> None: self._emit = emit self._launch_records: list[LaunchRecord] = [] + def _raise_unconfirmed_process_cleanup( + self, + *, + process: object, + command: Sequence[str], + cwd: str | Path | None, + stdin: object, + cause: BaseException, + ) -> NoReturn: + try: + self._record_launch( + launch_intent="foreground_cleanup_unconfirmed", + command=command, + pid=getattr(process, "pid", None), + cwd=cwd, + stdin_policy=_stdio_policy_name(stdin, inherited_label="inherit"), + stdout_policy="pipe", + stderr_policy="pipe", + controller_input_owner_allowed=False, + active=True, + emit_event=False, + ) + except BaseException: # noqa: BLE001 - the durable error still carries the process + pass + raise ProcessHandoffCleanupError( + process=process, + launch_intent="foreground_completion", + ) from cause + def run( self, cmd: Sequence[str], @@ -55,30 +86,14 @@ def run( process_started_callback(int(process.pid)) except Exception: pass - stdout, stderr = process.communicate(timeout=timeout) - return subprocess.CompletedProcess( - args=command, - returncode=process.returncode, - stdout=stdout, - stderr=stderr, - ) - except subprocess.TimeoutExpired as exc: - stdout = exc.stdout if isinstance(exc.stdout, str) else "" - stderr_prefix = exc.stderr if isinstance(exc.stderr, str) else "" - if process is not None and process.pid > 0: - try: - os.killpg(process.pid, signal.SIGTERM) - except OSError: - pass - try: - extra_stdout, extra_stderr = process.communicate(timeout=2.0) - if isinstance(extra_stdout, str): - stdout = f"{stdout}{extra_stdout}" - if isinstance(extra_stderr, str): - stderr_prefix = f"{stderr_prefix}{extra_stderr}" - except subprocess.TimeoutExpired: + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout if isinstance(exc.stdout, str) else "" + stderr_prefix = exc.stderr if isinstance(exc.stderr, str) else "" + if process.pid > 0: try: - os.killpg(process.pid, signal.SIGKILL) + os.killpg(process.pid, signal.SIGTERM) except OSError: pass try: @@ -88,14 +103,52 @@ def run( if isinstance(extra_stderr, str): stderr_prefix = f"{stderr_prefix}{extra_stderr}" except subprocess.TimeoutExpired: - pass - return self._timeout_result( - command, - timeout=timeout, + try: + os.killpg(process.pid, signal.SIGKILL) + except OSError: + pass + try: + extra_stdout, extra_stderr = process.communicate(timeout=2.0) + if isinstance(extra_stdout, str): + stdout = f"{stdout}{extra_stdout}" + if isinstance(extra_stderr, str): + stderr_prefix = f"{stderr_prefix}{extra_stderr}" + except subprocess.TimeoutExpired: + pass + cleanup_confirmed = _terminate_unhanded_process(process) + if not cleanup_confirmed: + self._raise_unconfirmed_process_cleanup( + process=process, + command=command, + cwd=cwd, + stdin=stdin, + cause=exc, + ) + return self._timeout_result( + command, + timeout=timeout, + stdout=stdout, + stderr_prefix=stderr_prefix, + ) + return subprocess.CompletedProcess( + args=command, + returncode=process.returncode, stdout=stdout, - stderr_prefix=stderr_prefix, + stderr=stderr, ) - except OSError: + except BaseException as exc: # noqa: BLE001 - a failed completion must not orphan the child + if isinstance(exc, ProcessHandoffCleanupError): + raise + if process is not None: + if _terminate_unhanded_process(process): + raise + self._raise_unconfirmed_process_cleanup( + process=process, + command=command, + cwd=cwd, + stdin=stdin, + cause=exc, + ) raise def run_probe( diff --git a/python/envctl_engine/shared/process_termination.py b/python/envctl_engine/shared/process_termination.py new file mode 100644 index 000000000..3ff0d9bbd --- /dev/null +++ b/python/envctl_engine/shared/process_termination.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import errno +import os +import signal +import time +from collections.abc import Callable +from typing import Any + +_IDENTITY_UNSET = object() + + +def terminate_pid( + pid: int | None, + *, + process_runner: object | None, + term_timeout: float, + kill_timeout: float, +) -> bool: + """Terminate a PID without treating an attempted signal as proof of exit. + + Process-runner implementations get the first opportunity to terminate the + process group and then the individual process. A false result (or an + exception) is not success: the raw-signal fallback confirms exit and + escalates to SIGKILL when necessary. + """ + + if not isinstance(pid, int) or pid <= 0: + return True + if pid in {os.getpid(), os.getppid()}: + return False + if not pid_is_running(process_runner, pid): + return True + initial_identity = pid_identity(process_runner, pid) + + group_terminator = getattr(process_runner, "terminate_process_group", None) + if callable(group_terminator): + try: + terminated = bool(group_terminator(pid, term_timeout=term_timeout, kill_timeout=kill_timeout)) + except Exception: # noqa: BLE001 + if callable(getattr(process_runner, "_process_group_is_running", None)): + return False + if _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity): + return True + else: + if not terminated: + # Killing only the recorded leader after group cleanup failed + # can orphan descendants and must never be reported as success. + return False + return _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity) + + terminator = getattr(process_runner, "terminate", None) + if callable(terminator): + try: + terminated = bool(terminator(pid, term_timeout=term_timeout, kill_timeout=kill_timeout)) + except Exception: # noqa: BLE001 + if _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity): + return True + else: + if _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity): + return True + if terminated: + # A capable runner that claims success while the same PID remains + # alive is not proof. Do not signal again: the PID may have raced + # with reuse between the runner and this verification. + return False + + return terminate_with_raw_signals( + process_runner, + pid, + term_timeout=term_timeout, + kill_timeout=kill_timeout, + initial_identity=initial_identity, + ) + + +def terminate_with_raw_signals( + process_runner: object | None, + pid: int, + *, + term_timeout: float, + kill_timeout: float, + initial_identity: str | None | object = _IDENTITY_UNSET, +) -> bool: + """Last-resort termination that proves the recorded process exited.""" + + if initial_identity is _IDENTITY_UNSET: + initial_identity = pid_identity(process_runner, pid) + assert isinstance(initial_identity, str) or initial_identity is None + if _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity): + return True + term_sent = _send_signal(pid, signal.SIGTERM) + if term_sent is None: + return True + if not term_sent: + return not pid_is_running(process_runner, pid) + if wait_for_pid_exit( + process_runner, + pid, + timeout=term_timeout, + initial_identity=initial_identity, + ): + return True + kill_sent = _send_signal(pid, signal.SIGKILL) + if kill_sent is None: + return True + if not kill_sent: + return not pid_is_running(process_runner, pid) + return wait_for_pid_exit( + process_runner, + pid, + timeout=kill_timeout, + initial_identity=initial_identity, + ) + + +def _pid_gone_or_replaced( + process_runner: object | None, + pid: int, + *, + initial_identity: str | None, +) -> bool: + if not pid_is_running(process_runner, pid): + return True + if initial_identity is None: + return False + current_identity = pid_identity(process_runner, pid) + return current_identity is None or current_identity != initial_identity + + +def wait_for_pid_exit( + process_runner: object | None, + pid: int, + *, + timeout: float, + initial_identity: str | None, +) -> bool: + deadline = time.monotonic() + max(timeout, 0.0) + while True: + if not pid_is_running(process_runner, pid): + return True + if initial_identity is not None: + current_identity = pid_identity(process_runner, pid) + if current_identity is None or current_identity != initial_identity: + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + time.sleep(min(0.05, remaining)) + + +def pid_is_running(process_runner: object | None, pid: int) -> bool: + checker = getattr(process_runner, "is_pid_running", None) + if callable(checker): + try: + return bool(checker(pid)) + except Exception: # noqa: BLE001 + pass + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError as exc: + return exc.errno != errno.ESRCH + return True + + +def pid_identity(process_runner: object | None, pid: int) -> str | None: + identity_reader = getattr(process_runner, "_pid_identity", None) + if not callable(identity_reader): + return None + try: + identity = identity_reader(pid) + except Exception: # noqa: BLE001 + return None + if not isinstance(identity, str): + return None + normalized = identity.strip() + return normalized or None + + +def _send_signal(pid: int, requested_signal: signal.Signals) -> bool | None: + sender: Callable[[int, signal.Signals], Any] = os.kill + try: + sender(pid, requested_signal) + except ProcessLookupError: + return None + except OSError: + return False + return True diff --git a/python/envctl_engine/shared/protocols.py b/python/envctl_engine/shared/protocols.py index 07facf1b7..e050af301 100644 --- a/python/envctl_engine/shared/protocols.py +++ b/python/envctl_engine/shared/protocols.py @@ -72,6 +72,8 @@ def update_final_port(self, plan: object, final_port: int, source: str = "retry" def release(self, port: int, owner: str | None = None) -> None: ... + def release_owned(self, port: int, owner: str, *, expected_session: str) -> bool: ... + def release_session(self) -> None: ... def release_all(self) -> None: ... @@ -92,6 +94,7 @@ def save_run( emit: Callable[..., None], runtime_map_builder: Callable[[object], dict[str, object]], write_runtime_readiness_report: Callable[[Path], None] | None = None, + on_commit: Callable[[], None] | None = None, ) -> object: ... def save_resume_state( @@ -108,11 +111,32 @@ def save_selected_stop_state( state: object, emit: Callable[..., None], runtime_map_builder: Callable[[object], dict[str, object]], + authoritative_project_names: Sequence[str] | None = None, ) -> dict[str, object]: ... - def load_latest(self, *, mode: str | None = None, strict_mode_match: bool = False) -> object | None: ... + def load_latest( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, + ) -> object | None: ... + + def load_all(self, *, mode: str | None = None) -> Sequence[object]: ... + + def has_active_runs(self) -> bool: ... + + def write_runtime_artifact( + self, + *, + run_id: str | None, + artifact_name: str, + text: str, + ) -> bool: ... + + def deactivate_run(self, run_id: str) -> bool: ... - def load_by_pointer(self, pointer_path: str) -> object: ... + def deactivate_runs(self, run_ids: Sequence[str]) -> bool: ... def purge(self, *, aggressive: bool = False) -> None: ... diff --git a/python/envctl_engine/shared/services.py b/python/envctl_engine/shared/services.py index ab470a7d2..ef11d5da3 100644 --- a/python/envctl_engine/shared/services.py +++ b/python/envctl_engine/shared/services.py @@ -1,16 +1,15 @@ from __future__ import annotations -from envctl_engine.state.models import RunState +from collections.abc import Callable + +from envctl_engine.state.models import RunState, ServiceRecord def frontend_env_for_project(state: RunState, project: str, *, host: str = "localhost") -> dict[str, str]: + backend = service_for_project_type(state, project=project, service_type="backend") backend_port = None - for service in state.services.values(): - if project_name_from_service_name(service.name) != project: - continue - if service.type != "backend": - continue - backend_port = service.actual_port if service.actual_port is not None else service.requested_port + if backend is not None: + backend_port = backend.actual_port if backend.actual_port is not None else backend.requested_port if backend_port is None: return {} backend_url = f"http://{host}:{backend_port}" @@ -47,9 +46,63 @@ def service_project_name(service: object) -> str: explicit = str(getattr(service, "project", "") or "").strip() if explicit: return explicit + typed_project = _typed_service_project_name(service) + if typed_project: + return typed_project return project_name_from_service_name(str(getattr(service, "name", "") or "")) +def _typed_service_project_name(service: object) -> str: + name = str(getattr(service, "name", "") or "").strip() + display = service_display_name(service_slug_from_record(service)) + suffix = f" {display}" if display else "" + if suffix and name.casefold().endswith(suffix.casefold()): + return name[: -len(suffix)].strip() + collision_marker = f" {display} " if display else "" + if collision_marker: + marker_index = name.casefold().rfind(collision_marker.casefold()) + if marker_index > 0: + collision_suffix = name[marker_index + len(collision_marker) :].strip().casefold() + if collision_suffix.startswith(("restart collision", "resume collision", "state collision")): + return name[:marker_index].strip() + return "" + + +def service_for_project_type(state: RunState, *, project: str, service_type: str) -> ServiceRecord | None: + project_key = str(project).strip().casefold() + type_key = str(service_type).strip().lower() + for service in state.services.values(): + if service_project_name(service).casefold() != project_key: + continue + if service_slug_from_record(service) == type_key: + return service + return None + + +def resolve_service_project_name( + service_name: str, + service: object | None, + *, + project_name_from_service: Callable[[str], object] | None = None, +) -> str: + """Resolve project ownership without discarding authoritative record metadata.""" + explicit = str(getattr(service, "project", "") or "").strip() + if explicit: + return explicit + typed_project = _typed_service_project_name(service) if service is not None else "" + if typed_project: + return typed_project + if project_name_from_service is not None: + legacy = str(project_name_from_service(service_name) or "").strip() + if legacy: + return legacy + if service is not None: + record_project = service_project_name(service) + if record_project: + return record_project + return project_name_from_service_name(str(service_name)) + + def service_matches_selector(service: object, selector: str) -> bool: target = str(selector or "").strip().lower() if not target: diff --git a/python/envctl_engine/startup/disabled_startup_resolution.py b/python/envctl_engine/startup/disabled_startup_resolution.py index 6f4801f52..ccd03fdfd 100644 --- a/python/envctl_engine/startup/disabled_startup_resolution.py +++ b/python/envctl_engine/startup/disabled_startup_resolution.py @@ -6,6 +6,7 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_env import route_is_implicit_start +from envctl_engine.runtime.lifecycle_operation_lease import release_lifecycle_operation from envctl_engine.startup.finalization import ( build_planning_dashboard_state, headless_plan_output_only, @@ -68,6 +69,7 @@ def resolve_disabled_startup_mode( artifacts_started = time.monotonic() runtime._write_artifacts(run_state, session.selected_contexts, errors=[]) emit_phase(session, "artifacts_write", artifacts_started, status="ok") + release_lifecycle_operation(runtime) if route.command == "plan": validate_plan_agent_handoff(session, phase="disabled_startup_finalization") print_plan_dry_run_preview(session) @@ -139,13 +141,11 @@ def validate_plan_agent_handoff(session: StartupSession, *, phase: str) -> None: session=session, validate_plan_agent_handoff=validate_plan_agent_handoff, attach_plan_agent_terminal=attach_plan_agent_terminal, - print_headless_plan_session_summary=lambda session, *, attach_target: ( - print_headless_plan_session_summary( - session, - validate_plan_agent_handoff=validate_plan_agent_handoff, - print_fn=print_fn, - attach_target=attach_target, - ) + print_headless_plan_session_summary=lambda session, *, attach_target: print_headless_plan_session_summary( + session, + validate_plan_agent_handoff=validate_plan_agent_handoff, + print_fn=print_fn, + attach_target=attach_target, ), ), print_fn=print_fn, diff --git a/python/envctl_engine/startup/finalization.py b/python/envctl_engine/startup/finalization.py index 77ce8611d..fdafc8af2 100644 --- a/python/envctl_engine/startup/finalization.py +++ b/python/envctl_engine/startup/finalization.py @@ -3,9 +3,11 @@ import time import sys from collections.abc import Callable +import inspect from envctl_engine.planning.plan_agent.tmux_transport import attach_plan_agent_terminal from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.lifecycle_operation_lease import release_lifecycle_operation from envctl_engine.startup.finalization_plan_output import ( format_degraded_handoff_text_for_terminal, headless_plan_output_only, @@ -104,8 +106,9 @@ def finalize_successful_startup( run_state = build_success_run_state(runtime, session) emit_preserved_service_merge(session) artifacts_started = time.monotonic() - runtime._write_artifacts(run_state, session.selected_contexts, errors=session.errors) + _write_artifacts_at_commit_boundary(runtime, session, run_state) emit_phase(session, "artifacts_write", artifacts_started, status="ok") + release_lifecycle_operation(runtime) if requirements_timing_enabled(session.effective_route) and not suppress_timing_output(session.effective_route): runtime._emit( "startup.debug_tty_group", @@ -213,13 +216,11 @@ def finalize_successful_startup_with_runtime( session=session, validate_plan_agent_handoff=validate_plan_agent_handoff, attach_plan_agent_terminal=attach_plan_agent_terminal, - print_headless_plan_session_summary=lambda session, *, attach_target: ( - print_headless_plan_session_summary( - session, - validate_plan_agent_handoff=validate_plan_agent_handoff, - print_fn=print_fn, - attach_target=attach_target, - ) + print_headless_plan_session_summary=lambda session, *, attach_target: print_headless_plan_session_summary( + session, + validate_plan_agent_handoff=validate_plan_agent_handoff, + print_fn=print_fn, + attach_target=attach_target, ), ), finalize_plan_agent_degraded_handoff=lambda session: finalize_plan_agent_degraded_handoff_with_runtime( @@ -257,13 +258,11 @@ def finalize_plan_agent_degraded_handoff_with_runtime( session=session, validate_plan_agent_handoff=validate_plan_agent_handoff, attach_plan_agent_terminal=attach_plan_agent_terminal, - print_headless_plan_session_summary=lambda session, *, attach_target: ( - print_headless_plan_session_summary( - session, - validate_plan_agent_handoff=validate_plan_agent_handoff, - print_fn=print_fn, - attach_target=attach_target, - ) + print_headless_plan_session_summary=lambda session, *, attach_target: print_headless_plan_session_summary( + session, + validate_plan_agent_handoff=validate_plan_agent_handoff, + print_fn=print_fn, + attach_target=attach_target, ), ), ) @@ -289,6 +288,7 @@ def finalize_plan_agent_degraded_handoff( build_success_run_state=build_success_run_state, emit_phase=emit_phase, ) + release_lifecycle_operation(runtime) render_plan_agent_degraded_handoff(session) if headless_plan_output_only(session): return 0 @@ -311,11 +311,44 @@ def finalize_plan_agent_degraded_handoff_artifacts( validate_plan_agent_handoff(session, phase="degraded_finalization") run_state = build_success_run_state(runtime, session) artifacts_started = time.monotonic() - runtime._write_artifacts(run_state, session.selected_contexts, errors=session.errors) + _write_artifacts_at_commit_boundary(runtime, session, run_state) emit_phase(session, "artifacts_write", artifacts_started, status="degraded") return run_state +def _write_artifacts_at_commit_boundary( + runtime: StartupRuntime, + session: StartupSession, + run_state: RunState, +) -> None: + writer = runtime._write_artifacts + + def mark_committed() -> None: + session.authority_committed = True + + try: + parameters = inspect.signature(writer).parameters.values() + except (TypeError, ValueError): + parameters = () + # Treat the callback as an explicit protocol capability. A generic + # ``**kwargs`` proxy can forward into a legacy writer or raise after doing + # durable work; probing it and retrying would risk executing a commit twice. + supports_callback = any(parameter.name == "on_commit" for parameter in parameters) + if supports_callback: + writer( + run_state, + session.selected_contexts, + errors=session.errors, + on_commit=mark_committed, + ) + else: + writer(run_state, session.selected_contexts, errors=session.errors) + # Legacy adapters cannot expose their internal commit point. Preserve the + # historical contract by marking successful returns as committed too. + if not session.authority_committed: + mark_committed() + + def render_project_startup_warnings( runtime: StartupRuntime, *, diff --git a/python/envctl_engine/startup/finalization_failure.py b/python/envctl_engine/startup/finalization_failure.py index d5d9928fc..81fd3cf4f 100644 --- a/python/envctl_engine/startup/finalization_failure.py +++ b/python/envctl_engine/startup/finalization_failure.py @@ -1,16 +1,18 @@ from __future__ import annotations from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path import sys import time from typing import Any from envctl_engine.startup.protocols import ProjectContextLike, StartupRuntime -from envctl_engine.startup.session import StartupSession +from envctl_engine.startup.session import StartupSession, unconfirmed_service_names from envctl_engine.startup.finalization_run_state import _build_run_state -from envctl_engine.state.models import RunState +from envctl_engine.requirements.core import dependency_definitions +from envctl_engine.runtime.lifecycle_requirement_ports import component_port_values +from envctl_engine.state.models import RunState, ServiceRecord from envctl_engine.ui.color_policy import colors_enabled from envctl_engine.ui.path_links import local_paths_in_text, render_paths_in_terminal_text from envctl_engine.ui.status_symbols import STATUS_FAILURE @@ -40,8 +42,10 @@ def finalize(self) -> int: final_error = self._final_error() self.session.failure_message = final_error self.session.errors.append(final_error) + self._normalize_colliding_service_state() self._emit_failure(final_error) self._terminate_started_services() + self._prepare_collision_failure_state() self._release_ports() self._write_failure_artifacts(final_error) self._print_final_status(final_error) @@ -54,6 +58,83 @@ def _final_error(self) -> str: return self.error return f"Startup failed: {self.error}" + def _normalize_colliding_service_state(self) -> None: + """Give every newly started process a unique durable storage key. + + This runs before cleanup so duplicate hook-provided names cannot be + collapsed in the termination request. The ServiceRecord keeps its + authoritative project/service metadata; only its storage identity is + disambiguated. + """ + + replacement_maps = [*self.session.services_by_project.values(), self.session.unterminated_services] + reserved_names = set(self.session.preserved_services) + for services in replacement_maps: + reserved_names.update(services) + assigned_names = set(self.session.preserved_services) + identity_records: dict[int, tuple[str, ServiceRecord]] = {} + collision_rows = list(self.session.service_state_collision_rows) + recorded_replacements = { + str(row.get("replacement_name") or "").strip() + for row in collision_rows + if isinstance(row, dict) + } + + def normalized_services( + services: dict[str, ServiceRecord], + *, + fallback_project: str = "", + ) -> dict[str, ServiceRecord]: + normalized: dict[str, ServiceRecord] = {} + for original_name, service in services.items(): + prior_record = identity_records.get(id(service)) + if prior_record is not None: + prior_name, prior_service = prior_record + normalized[prior_name] = prior_service + continue + stored_name = original_name + stored_service = service + if stored_name in assigned_names: + stored_name = _collision_service_name( + original_name, + service, + reserved_names.union(assigned_names), + ) + if isinstance(stored_service, ServiceRecord): + stored_service = replace( + stored_service, + name=stored_name, + project=stored_service.project or fallback_project or None, + ) + self.session.service_state_collisions.add(original_name) + if stored_name not in recorded_replacements: + collision_rows.append( + { + "original_name": original_name, + "replacement_name": stored_name, + "replacement_pid": getattr(service, "pid", None), + "replacement_project": getattr(service, "project", None), + } + ) + recorded_replacements.add(stored_name) + assigned_names.add(stored_name) + reserved_names.add(stored_name) + identity_records[id(service)] = (stored_name, stored_service) + normalized[stored_name] = stored_service + return normalized + + normalized_by_project: dict[str, dict[str, ServiceRecord]] = {} + for project, services in self.session.services_by_project.items(): + normalized_by_project[project] = normalized_services( + dict(services), + fallback_project=project, + ) + self.session.services_by_project = normalized_by_project + self.session.unterminated_services = normalized_services( + dict(self.session.unterminated_services) + ) + self.session.service_state_collision_rows = collision_rows + def _emit_failure(self, final_error: str) -> None: failure_payload: dict[str, object] = { "mode": self.session.runtime_mode, @@ -62,43 +143,185 @@ def _emit_failure(self, final_error: str) -> None: } if self.session.strict_truth_failed: failure_payload["services"] = sorted(self.session.merged_services) - self.runtime._emit("startup.failed", **failure_payload) + try: + self.runtime._emit("startup.failed", **failure_payload) + except Exception: # noqa: BLE001 + pass def _terminate_started_services(self) -> None: - started_services = { - service_name: service - for project_name in self.session.started_context_names - for service_name, service in self.session.services_by_project.get(project_name, {}).items() - } + started_services = dict(self.session.unterminated_services) + started_services.update( + { + service_name: service + for project_services in self.session.services_by_project.values() + for service_name, service in project_services.items() + } + ) if started_services: - self.runtime._terminate_started_services(started_services) + try: + failed_names = unconfirmed_service_names( + self.runtime._terminate_started_services(started_services), + started_services, + ) + except Exception: # noqa: BLE001 + failed_names = set(started_services) + for project_name in list(self.session.services_by_project): + retained = { + name: service + for name, service in self.session.services_by_project[project_name].items() + if name in failed_names + } + if retained: + self.session.services_by_project[project_name] = retained + else: + self.session.services_by_project.pop(project_name, None) + self.session.unterminated_services = { + name: service for name, service in started_services.items() if name in failed_names + } + for service in self.session.unterminated_services.values(): + service.status = "termination_failed" + service.degraded = True + + def _prepare_collision_failure_state(self) -> None: + collision_names = set(self.session.service_state_collisions) + retained_service_names = set(self.session.unterminated_services) + for services in self.session.services_by_project.values(): + retained_service_names.update(services) + collision_rows: list[dict[str, object]] = [] + for row in self.session.service_state_collision_rows: + normalized_row = dict(row) + replacement_name = str(normalized_row.get("replacement_name") or "").strip() + normalized_row["replacement_exit_unconfirmed"] = replacement_name in retained_service_names + collision_rows.append(normalized_row) + recorded_originals = { + str(row.get("original_name") or "").strip() + for row in collision_rows + if isinstance(row, dict) + } + collision_rows.extend( + { + "original_name": name, + "replacement_exit_unconfirmed": False, + } + for name in sorted(collision_names - recorded_originals) + ) + + occupied_projects = set(self.session.preserved_requirements).union( + self.session.requirements_by_project + ) + remapped_requirements = {} + requirement_rows = list(self.session.requirement_state_collision_rows) + for project, requirements in self.session.requirements_by_project.items(): + preserved = self.session.preserved_requirements.get(project) + if preserved is None: + remapped_requirements[project] = requirements + continue + if requirements == preserved: + continue + synthetic_project = _collision_project_name(project, occupied_projects) + occupied_projects.add(synthetic_project) + # The dictionary key is only a durable storage identity. Keep the + # authoritative project on the value so later port release and + # container cleanup use the owner that acquired the resources. + remapped_requirements[synthetic_project] = requirements + requirement_rows.append( + { + "original_project": project, + "replacement_project": synthetic_project, + "replacement_requirements_retained": True, + } + ) + self.session.requirements_by_project = remapped_requirements + collision_rows.extend(requirement_rows) + if collision_rows: + self.session.base_metadata["startup_state_collisions"] = collision_rows + # A collision-aware failure state can now retain both the previous + # authority and any replacement resource whose cleanup was unconfirmed. + if collision_rows or collision_names: + self.session.preserve_existing_state_on_failure = False def _release_ports(self) -> None: - self.port_allocator(self.runtime).release_session() + if self.session.unterminated_services or _has_retained_managed_requirements(self.session): + try: + self.runtime._emit( + "port.session_release.skipped", + reason=( + "unterminated_services" + if self.session.unterminated_services + else "started_requirements_remain_tracked" + ), + services=sorted(self.session.unterminated_services), + ) + except Exception: # noqa: BLE001 + pass + return + try: + self.port_allocator(self.runtime).release_session() + except Exception: # noqa: BLE001 + pass def _write_failure_artifacts(self, final_error: str) -> None: - run_state = build_failure_run_state(self.runtime, self.session, final_error) artifacts_started = time.monotonic() + if self.session.preserve_existing_state_on_failure: + try: + self.runtime._emit( + "state.failure_write.skipped", + reason="existing_runtime_state_remains_authoritative", + run_id=self.session.run_id, + ) + self.emit_phase(self.session, "artifacts_write", artifacts_started, status="skipped") + except Exception: # noqa: BLE001 + pass + return + run_state = build_failure_run_state(self.runtime, self.session, final_error) self.runtime._write_artifacts(run_state, self.session.selected_contexts, errors=self.session.errors) - self.emit_phase(self.session, "artifacts_write", artifacts_started, status="error") + try: + self.emit_phase(self.session, "artifacts_write", artifacts_started, status="error") + except Exception: # noqa: BLE001 + pass def _print_final_status(self, final_error: str) -> None: - interactive_tty = failure_hyperlink_interactive_tty(self.runtime) - rendered_error = self.render_final_failure_status( - self.runtime, - self.session, - final_error, - interactive_tty=interactive_tty, - ) - self.print_fn( - render_paths_in_terminal_text( - rendered_error, - paths=local_paths_in_text(rendered_error), - env=self.runtime.env, - stream=self.output_stream, + try: + interactive_tty = failure_hyperlink_interactive_tty(self.runtime) + rendered_error = self.render_final_failure_status( + self.runtime, + self.session, + final_error, interactive_tty=interactive_tty, ) - ) + self.print_fn( + render_paths_in_terminal_text( + rendered_error, + paths=local_paths_in_text(rendered_error), + env=self.runtime.env, + stream=self.output_stream, + interactive_tty=interactive_tty, + ) + ) + except Exception: # noqa: BLE001 + pass + + +def _collision_service_name(name: str, service: object, occupied: set[str]) -> str: + pid = getattr(service, "pid", None) + suffix = f"Restart Collision {pid}" if isinstance(pid, int) and pid > 0 else "Restart Collision" + base = f"{name} {suffix}" + candidate = base + index = 2 + while candidate in occupied: + candidate = f"{base} {index}" + index += 1 + return candidate + + +def _collision_project_name(project: str, occupied: set[str]) -> str: + base = f"{project} Restart Collision" + candidate = base + index = 2 + while candidate in occupied: + candidate = f"{base} {index}" + index += 1 + return candidate def finalize_failed_startup( @@ -122,6 +345,21 @@ def finalize_failed_startup( ).finalize() +def _has_retained_managed_requirements(session: StartupSession) -> bool: + for requirements in session.requirements_by_project.values(): + for definition in dependency_definitions(): + component = requirements.component(definition.id) + if not bool(component.get("enabled", False)): + continue + if bool(component.get("external", False)) or bool(component.get("simulated", False)): + continue + if not bool(component.get("success", False)) and not str(component.get("container_name") or "").strip(): + continue + if component_port_values(component) or str(component.get("container_name") or "").strip(): + return True + return False + + def failure_hyperlink_interactive_tty(runtime: StartupRuntime) -> bool | None: link_mode = str(runtime.env.get("ENVCTL_UI_HYPERLINK_MODE", "")).strip().lower() return True if link_mode == "on" else None diff --git a/python/envctl_engine/startup/finalization_run_state.py b/python/envctl_engine/startup/finalization_run_state.py index 37a34cc90..4e18017dc 100644 --- a/python/envctl_engine/startup/finalization_run_state.py +++ b/python/envctl_engine/startup/finalization_run_state.py @@ -1,17 +1,22 @@ from __future__ import annotations import shlex +from pathlib import Path from typing import cast from envctl_engine.dashboard_metadata import ( APP_SERVICE_TYPES, DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY, + dashboard_project_configured_services_from_metadata, serialize_dashboard_project_configured_services, ) from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_env import effective_dependency_scope +from envctl_engine.shared.services import service_slug_from_record from envctl_engine.startup.protocols import ProjectContextLike, StartupRuntime +from envctl_engine.startup.run_reuse_dashboard_restore import metadata_without_dashboard_stopped_services from envctl_engine.startup.run_reuse_support import build_startup_identity_metadata +from envctl_engine.startup.service_execution_policy import additional_service_enabled_for_context from envctl_engine.startup.session import StartupSession from envctl_engine.state.models import RunState @@ -54,6 +59,9 @@ def build_planning_dashboard_state( ), } ) + metadata["application_runtime"] = ( + "docker" if bool(route.flags.get("docker")) else "process" + ) run_state = RunState( run_id=run_id, mode=runtime_mode, @@ -80,6 +88,9 @@ def _build_run_state(runtime: StartupRuntime, session: StartupSession, *, failed { "command": session.effective_route.command, "repo_scope_id": runtime.config.runtime_scope_id, + "application_runtime": ( + "docker" if bool(session.effective_route.flags.get("docker")) else "process" + ), } ) dependency_mode = effective_dependency_scope(session.effective_route, session.runtime_mode) @@ -93,14 +104,33 @@ def _build_run_state(runtime: StartupRuntime, session: StartupSession, *, failed if requested_dependency_scope is not None: metadata["dependency_scope_requested"] = str(requested_dependency_scope) project_configured_services = _project_configured_services_metadata(runtime, session) - if project_configured_services: - metadata[DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY] = project_configured_services + existing_configured_services = dashboard_project_configured_services_from_metadata(metadata) + selected_project_keys = { + str(context.name).strip().casefold() + for context in session.selected_contexts + if str(getattr(context, "name", "")).strip() + } + merged_configured_services = { + project: service_types + for project, service_types in existing_configured_services.items() + if project.casefold() not in selected_project_keys + } + merged_configured_services.update(project_configured_services) + serialized_configured_services = serialize_dashboard_project_configured_services( + merged_configured_services + ) + if serialized_configured_services: + metadata[DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY] = serialized_configured_services + else: + metadata.pop(DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY, None) shared_dependency_project = _shared_dependency_dashboard_project(session) if shared_dependency_project: metadata["dashboard_dependency_scope"] = "shared" metadata["dashboard_shared_dependency_project"] = shared_dependency_project if session.warnings: metadata["warnings"] = list(session.warnings) + if session.unterminated_services: + metadata["termination_failed_services"] = sorted(session.unterminated_services) if session.plan_agent_launch_result is not None: launch_result = session.plan_agent_launch_result metadata["plan_agent_launch_status"] = str(getattr(launch_result, "status", "")).strip() @@ -183,6 +213,18 @@ def _build_run_state(runtime: StartupRuntime, session: StartupSession, *, failed metadata["plan_agent_session_name"] = session_name if attach_command: metadata["plan_agent_attach_command"] = attach_command + if not failed: + restored_types: dict[str, set[str]] = {} + for project_name, services in session.services_by_project.items(): + for service in services.values(): + project = str(getattr(service, "project", "") or project_name).strip() + service_type = service_slug_from_record(service) + if project and service_type: + restored_types.setdefault(project, set()).add(service_type) + metadata = metadata_without_dashboard_stopped_services( + metadata, + restored_service_types_by_project=restored_types, + ) run_state = RunState( run_id=session.run_id, mode=session.runtime_mode, @@ -197,9 +239,7 @@ def _build_run_state(runtime: StartupRuntime, session: StartupSession, *, failed return run_state -def _project_configured_services_metadata( - runtime: StartupRuntime, session: StartupSession -) -> dict[str, list[str]]: +def _project_configured_services_metadata(runtime: StartupRuntime, session: StartupSession) -> dict[str, list[str]]: configured: dict[str, list[str]] = {} for context in session.selected_contexts: service_types = [ @@ -207,6 +247,16 @@ def _project_configured_services_metadata( for service_type in APP_SERVICE_TYPES if runtime._service_enabled_for_mode(session.runtime_mode, service_type) ] + service_types.extend( + str(getattr(service, "name", "") or "").strip().lower() + for service in tuple(getattr(runtime.config, "additional_services", ()) or ()) + if str(getattr(service, "name", "") or "").strip() + and additional_service_enabled_for_context( + service, + mode=session.runtime_mode, + project_root=Path(context.root), + ) + ) if service_types: configured[str(context.name)] = service_types return serialize_dashboard_project_configured_services(configured) diff --git a/python/envctl_engine/startup/lifecycle.py b/python/envctl_engine/startup/lifecycle.py index a82529b67..5a49ea71e 100644 --- a/python/envctl_engine/startup/lifecycle.py +++ b/python/envctl_engine/startup/lifecycle.py @@ -6,6 +6,7 @@ from envctl_engine.planning.plan_agent.launch import launch_plan_agent_terminals from envctl_engine.planning.plan_agent.omx_transport import validate_plan_agent_attach_target from envctl_engine.planning.plan_agent.tmux_transport import attach_plan_agent_terminal +from envctl_engine.runtime.lifecycle_operation_lease import release_lifecycle_operation from envctl_engine.startup.context_selection import select_startup_contexts from envctl_engine.startup.dependency_bootstrap import prepare_project_dependencies from envctl_engine.startup.disabled_startup_resolution import resolve_disabled_startup_mode_with_runtime @@ -95,9 +96,59 @@ def execute_startup_lifecycle(orchestrator: Any, route: Any) -> int: ), ) except RuntimeError as exc: - return finalize_failure(error=str(exc)) + return _finalize_startup_exception( + runtime=runtime, + session=session, + error=exc, + finalize_failure=finalize_failure, + ) except Exception as exc: - return finalize_failure(error=str(exc)) + return _finalize_startup_exception( + runtime=runtime, + session=session, + error=exc, + finalize_failure=finalize_failure, + ) + except BaseException as exc: # noqa: BLE001 - cancellation still requires failure finalization + try: + _finalize_startup_exception( + runtime=runtime, + session=session, + error=exc, + finalize_failure=finalize_failure, + ) + except BaseException as finalization_error: # noqa: BLE001 + if finalization_error is not exc: + try: + exc.add_note(f"startup cancellation finalization also failed: {finalization_error}") + except Exception: # noqa: BLE001 + pass + raise + + +def _finalize_startup_exception( + *, + runtime: Any, + session: StartupSession, + error: BaseException, + finalize_failure: Any, +) -> int: + if not session.authority_committed: + return int(finalize_failure(error=str(error))) + # The durable active revision is already committed. Presentation, terminal + # attachment, or dashboard failures after this boundary must not terminate + # its services or rewrite it as a failed startup. + release_lifecycle_operation(runtime) + try: + runtime._emit( + "startup.post_commit_error", + run_id=session.run_id, + error=str(error), + ) + except Exception: # noqa: BLE001 + pass + print(f"Startup state is active, but post-start handling failed: {error}") + return 1 def _pre_start_phases( diff --git a/python/envctl_engine/startup/protocols.py b/python/envctl_engine/startup/protocols.py index 29ff99624..f04872afc 100644 --- a/python/envctl_engine/startup/protocols.py +++ b/python/envctl_engine/startup/protocols.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from pathlib import Path from typing import Protocol @@ -40,8 +40,14 @@ def _requirements_ready(self, result: RequirementsResult) -> bool: ... def _assert_project_services_post_start_truth( self, *, context: ProjectContextLike, services: dict[str, ServiceRecord] ) -> None: ... - def _terminate_started_services(self, services: dict[str, ServiceRecord]) -> None: ... - def _try_load_existing_state(self, *, mode: str, strict_mode_match: bool = False) -> RunState | None: ... + def _terminate_started_services(self, services: dict[str, ServiceRecord]) -> set[str]: ... + def _try_load_existing_state( + self, + *, + mode: str, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, + ) -> RunState | None: ... def _requirement_enabled(self, requirement_name: str, *, mode: str, route: Route | None = None) -> bool: ... def _service_enabled_for_mode(self, mode: str, service_name: str) -> bool: ... def _set_plan_port_from_component(self, plan: PortPlan, component: dict[str, object]) -> None: ... @@ -86,7 +92,7 @@ def _terminate_services_from_state( selected_services: set[str] | None = None, aggressive: bool = False, verify_ownership: bool = False, - ) -> None: ... + ) -> set[str]: ... def _enforce_runtime_readiness_contract(self, *, scope: str) -> bool: ... def _new_run_id(self) -> str: ... def _current_session_id(self) -> str | None: ... @@ -157,7 +163,14 @@ def _wait_for_service_listener( ) -> None: ... def _refresh_service_listener_pids(self, service: ServiceRecord) -> None: ... def _run_interactive_dashboard_loop(self, state: RunState) -> int: ... - def _write_artifacts(self, state: RunState, contexts: list[ProjectContextLike], *, errors: list[str]) -> None: ... + def _write_artifacts( + self, + state: RunState, + contexts: list[ProjectContextLike], + *, + errors: list[str], + on_commit: Callable[[], None] | None = None, + ) -> None: ... def _print_summary(self, state: RunState, contexts: list[ProjectContextLike]) -> None: ... def _should_enter_post_start_interactive(self, route: Route) -> bool: ... def _run_pr_action(self, route: Route, targets: list[ProjectContextLike]) -> int: ... diff --git a/python/envctl_engine/startup/requirements_component_startup.py b/python/envctl_engine/startup/requirements_component_startup.py index a251b2479..72e142912 100644 --- a/python/envctl_engine/startup/requirements_component_startup.py +++ b/python/envctl_engine/startup/requirements_component_startup.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Protocol +from envctl_engine.requirements.core import dependency_definitions from envctl_engine.requirements.orchestrator import FailureClass, RequirementOutcome from envctl_engine.requirements.supabase import ( evaluate_managed_supabase_reliability_contract, @@ -273,6 +274,8 @@ def _on_retry( failure_class: FailureClass, error: str | None, ) -> None: + if retry_port != failed_port: + self._release_failed_requirement_port(failed_port) reason_code = self.runtime.requirements.reason_code_for_failure(service_name, failure_class, error=error) self.runtime._emit( "requirements.retry", @@ -287,6 +290,23 @@ def _on_retry( error=(error or "").strip() or None, ) + def _release_failed_requirement_port(self, port: int) -> None: + release = getattr(self.runtime.port_planner, "release", None) + if not callable(release): + return + resource_keys = { + resource.legacy_port_key + for definition in dependency_definitions() + if definition.id == self.name + for resource in definition.resources + } + for owner_suffix in (*sorted(resource_keys), "requirements"): + try: + release(port, owner=f"{self.context.name}:{owner_suffix}") + except TypeError: + release(port) + return + def _finalize_outcome(self, outcome: RequirementOutcome) -> RequirementOutcome: if outcome.success and isinstance(self.native_effective_port, int) and self.native_effective_port > 0: outcome.final_port = self.native_effective_port diff --git a/python/envctl_engine/startup/requirements_execution.py b/python/envctl_engine/startup/requirements_execution.py index cf02b0f41..b2c8054dc 100644 --- a/python/envctl_engine/startup/requirements_execution.py +++ b/python/envctl_engine/startup/requirements_execution.py @@ -17,7 +17,9 @@ ) from envctl_engine.startup.startup_progress import suppress_timing_output from envctl_engine.startup.startup_selection_support import restart_include_requirements +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RequirementsResult +from envctl_engine.state.project_runtime import requirements_for_project _DOCKER_SOCKET_PATTERNS = ( re.compile(r"unix://(?P[^\s;]+docker\.sock)"), @@ -61,9 +63,14 @@ def requirements_for_restart_context( if restart_include_requirements(route): return orchestrator.start_requirements_for_project(context, mode=mode, route=route) - previous = rt._try_load_existing_state(mode=mode, strict_mode_match=True) + previous = call_state_loader( + rt._try_load_existing_state, + mode=mode, + strict_mode_match=True, + project_names=[context.name], + ) if previous is not None: - existing = previous.requirements.get(context.name) + existing = requirements_for_project(previous, context.name) if isinstance(existing, RequirementsResult): rt._emit( "requirements.restart.reuse", diff --git a/python/envctl_engine/startup/requirements_project_startup.py b/python/envctl_engine/startup/requirements_project_startup.py index 9747253e8..914224d02 100644 --- a/python/envctl_engine/startup/requirements_project_startup.py +++ b/python/envctl_engine/startup/requirements_project_startup.py @@ -15,7 +15,7 @@ external_dependency_resources, external_dependency_url, ) -from envctl_engine.requirements.orchestrator import RequirementOutcome +from envctl_engine.requirements.orchestrator import FailureClass, RequirementOutcome from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.runtime_context import resolve_port_allocator from envctl_engine.shared.parsing import parse_bool, parse_int @@ -27,6 +27,12 @@ REQUIREMENTS_PROGRESS_PROJECT_FLAG = "_requirements_progress_project" +def _has_reserved_ports(final_port: object, resources: dict[str, int]) -> bool: + return (isinstance(final_port, int) and not isinstance(final_port, bool) and final_port > 0) or any( + isinstance(port, int) and not isinstance(port, bool) and port > 0 for port in resources.values() + ) + + def requirements_parallel_platform_default() -> bool: return sys.platform != "darwin" @@ -248,31 +254,62 @@ def _run_components_parallel(self, enabled_definitions: list[Any], *, worker_cou future_map[future] = definition.id for future in concurrent.futures.as_completed(future_map): definition_id = future_map[future] - self.outcomes[definition_id] = future.result() + try: + self.outcomes[definition_id] = future.result() + except Exception as exc: # noqa: BLE001 + self.outcomes[definition_id] = self._failed_outcome( + definition_id, + self._plan_for_dependency(definition_id), + exc, + ) def _run_component(self, component: str, plan: object, *, strict: bool = False) -> RequirementOutcome: started = time.monotonic() with self.progress_state_lock: self.pending_requirements.discard(component) self.active_requirements.add(component) - self._emit_progress() + self._emit_progress_safely() try: - outcome = self.orchestrator.runtime._start_requirement_component( - self.context, - component, - plan, - self._reserve_next, - strict=strict, - route=self.route, - ) + try: + outcome = self.orchestrator.runtime._start_requirement_component( + self.context, + component, + plan, + self._reserve_next, + strict=strict, + route=self.route, + ) + except Exception as exc: # noqa: BLE001 + outcome = self._failed_outcome(component, plan, exc) duration_ms = round((time.monotonic() - started) * 1000.0, 2) self.component_timings_ms[component] = duration_ms - self._emit_component_timing(component, outcome=outcome, enabled=True, duration_ms=duration_ms) + try: + self._emit_component_timing(component, outcome=outcome, enabled=True, duration_ms=duration_ms) + except Exception: # noqa: BLE001 + pass return outcome finally: with self.progress_state_lock: self.active_requirements.discard(component) - self._emit_progress() + self._emit_progress_safely() + + @staticmethod + def _failed_outcome(component: str, plan: object, error: BaseException) -> RequirementOutcome: + return RequirementOutcome( + service_name=component, + success=False, + requested_port=int(getattr(plan, "requested", 0) or 0), + final_port=int(getattr(plan, "final", 0) or 0), + retries=0, + failure_class=FailureClass.HARD_START_FAILURE, + error=str(error), + ) + + def _emit_progress_safely(self) -> None: + try: + self._emit_progress() + except Exception: # noqa: BLE001 + return def _build_components( self, @@ -301,6 +338,18 @@ def _build_components( "error": outcome.error, "container_name": outcome.container_name, } + if ( + enabled_lookup[definition.id] + and not external_lookup[definition.id] + and _has_reserved_ports( + outcome.final_port, + resources, + ) + ): + raw_session_id = getattr(resolve_port_allocator(rt), "session_id", None) + session_id = raw_session_id.strip() if isinstance(raw_session_id, str) else "" + if session_id: + components[definition.id]["port_lock_session"] = session_id if external_lookup[definition.id]: components[definition.id].update( { @@ -318,9 +367,7 @@ def _emit_progress(self) -> None: return progress_project_flag = self.route.flags.get(REQUIREMENTS_PROGRESS_PROJECT_FLAG) progress_project = ( - str(progress_project_flag).strip() - if progress_project_flag is not None - else self.context.name + str(progress_project_flag).strip() if progress_project_flag is not None else self.context.name ) progress_message = format_requirements_progress_message( active=self.active_requirements, diff --git a/python/envctl_engine/startup/restart_prestop_support.py b/python/envctl_engine/startup/restart_prestop_support.py index aa0d1e981..1fafb72c5 100644 --- a/python/envctl_engine/startup/restart_prestop_support.py +++ b/python/envctl_engine/startup/restart_prestop_support.py @@ -1,18 +1,30 @@ from __future__ import annotations -from dataclasses import dataclass -from pathlib import Path +from dataclasses import dataclass, field from typing import Any, Callable, Iterable from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.docker_service_runtime import state_uses_docker_services +from envctl_engine.runtime.lifecycle_requirement_ports import release_port_reservation +from envctl_engine.runtime.lifecycle_service_termination import failed_listener_pids, unconfirmed_service_names +from envctl_engine.state.lookup import call_state_loader from envctl_engine.runtime.runtime_context import resolve_port_allocator, resolve_process_runtime -from envctl_engine.shared.services import service_project_name, service_slug_from_record +from envctl_engine.shared.process_cwd import process_cwd +from envctl_engine.shared.services import ( + service_project_name, + service_slug_from_record, +) from envctl_engine.startup.startup_selection_support import ( - _restart_selected_services, + _configured_service_projects_for_selector, + _observed_service_types_for_selector, restart_include_requirements, + restart_selected_services_for_type_map, + restart_service_types_by_project, + state_project_names, restart_target_projects, restart_target_projects_for_selected_services, ) +from envctl_engine.startup.session import metadata_with_state_sources @dataclass(frozen=True, slots=True) @@ -34,25 +46,23 @@ class RestartPrestopSelection: selected_services: set[str] target_projects: set[str] include_requirements: bool + service_types_by_project: dict[str, set[str]] = field(default_factory=dict) @dataclass(frozen=True, slots=True) class RestartOrphanListenerScan: ports_by_type: dict[str, set[int]] selected_by_cwd: dict[str, set[str]] + owners_by_cwd_type: dict[tuple[str, str], tuple[str, ...]] = field(default_factory=dict) + port_lock_sessions_by_cwd_type: dict[tuple[str, str], tuple[str, ...]] = field(default_factory=dict) @dataclass(frozen=True, slots=True) class RestartOrphanListenerMatch: pid: int port: int - - -def process_cwd(pid: int) -> str | None: - try: - return str(Path(f"/proc/{pid}/cwd").resolve()) - except OSError: - return None + owner_candidates: tuple[str, ...] = () + port_lock_sessions: tuple[str, ...] = () def restart_fallback_start_route(route: Route, *, restart_lookup_mode: str) -> Route: @@ -68,7 +78,11 @@ def restart_fallback_start_route(route: Route, *, restart_lookup_mode: str) -> R def restart_prestop_state(*, route: Route, runtime: Any) -> RestartPrestopState: restart_lookup_mode = runtime._effective_start_mode(route) - resumed = runtime._try_load_existing_state(mode=restart_lookup_mode) + resumed = call_state_loader( + runtime._try_load_existing_state, + mode=restart_lookup_mode, + project_names=route.projects or None, + ) if resumed is not None and resumed.mode != restart_lookup_mode: runtime._emit( "restart.state_mode_mismatch", @@ -77,6 +91,9 @@ def restart_prestop_state(*, route: Route, runtime: Any) -> RestartPrestopState: run_id=resumed.run_id, ) resumed = None + if resumed is not None and state_uses_docker_services(resumed): + route.flags["docker"] = True + runtime.env["DOCKER_MODE"] = "true" fallback_route = None if resumed is None: fallback_route = restart_fallback_start_route(route, restart_lookup_mode=restart_lookup_mode) @@ -94,37 +111,58 @@ def restart_start_route( selected_services: set[str], target_projects: set[str], include_requirements: bool, + service_types_by_project: dict[str, set[str]] | None = None, ) -> Route: + type_map = ( + { + project: sorted(service_types) + for project, service_types in sorted(service_types_by_project.items(), key=lambda item: item[0].casefold()) + } + if service_types_by_project is not None + else None + ) return Route( command="start", mode=restart_lookup_mode, raw_args=route.raw_args, passthrough_args=route.passthrough_args, - projects=route.projects, + projects=sorted(target_projects, key=str.casefold) if target_projects else route.projects, flags={ **route.flags, "_restart_request": True, "_restart_selected_services": sorted(selected_services), "_restart_target_projects": sorted(target_projects), "_restart_include_requirements": include_requirements, + **({"_restart_service_types_by_project": type_map} if type_map is not None else {}), }, ) def restart_prestop_selection(*, state: Any, route: Route, runtime: Any) -> RestartPrestopSelection: - selected_services = _restart_selected_services(state=state, route=route) target_projects = restart_target_projects(state=state, route=route, runtime=runtime) include_requirements = restart_include_requirements(route) - if include_requirements and not target_projects: + if include_requirements and not target_projects and not isinstance(route.flags.get("services"), list): target_projects = restart_target_projects_for_selected_services( - selected_services=selected_services, + selected_services=set(), state=state, runtime=runtime, ) + service_types_by_project = restart_service_types_by_project( + state=state, + route=route, + runtime=runtime, + target_projects=target_projects, + ) + selected_services = restart_selected_services_for_type_map( + state=state, + runtime=runtime, + service_types_by_project=service_types_by_project, + ) return RestartPrestopSelection( selected_services=selected_services, target_projects=target_projects, include_requirements=include_requirements, + service_types_by_project=service_types_by_project, ) @@ -133,7 +171,7 @@ def handle_restart_prestop( runtime: Any, session: Any, suppress_progress_output: Callable[[Route], bool], - terminate_restart_orphan_listeners: Callable[..., None], + terminate_restart_orphan_listeners: Callable[..., set[int]], spinner_factory: Callable[..., Any], use_spinner_policy_fn: Callable[[Any], Any], resolve_spinner_policy_fn: Callable[[dict[str, str]], Any], @@ -150,11 +188,35 @@ def handle_restart_prestop( session.runtime_mode = restart_lookup_mode return None session.restart_state = resumed + previous_preserve_flag = bool(getattr(session, "preserve_existing_state_on_failure", False)) + source_services = { + **dict(getattr(resumed, "services", {})), + **dict(session.preserved_services), + } + source_requirements = { + **dict(getattr(resumed, "requirements", {})), + **dict(session.preserved_requirements), + } + source_metadata = metadata_with_state_sources( + {**dict(getattr(resumed, "metadata", {})), **dict(session.base_metadata)}, + resumed, + ) + session.preserved_services = source_services + session.preserved_requirements = source_requirements + session.base_metadata = source_metadata + session.preserve_existing_state_on_failure = True selection = restart_prestop_selection(state=resumed, route=route, runtime=runtime) selected_services = selection.selected_services target_projects = selection.target_projects include_requirements = selection.include_requirements + _validate_restart_selection( + state=resumed, + route=route, + runtime=runtime, + target_projects=target_projects, + service_types_by_project=selection.service_types_by_project, + ) runtime._emit( "restart.selection", include_requirements=include_requirements, @@ -168,6 +230,12 @@ def handle_restart_prestop( prestop_policy, context={"component": "startup_orchestrator", "op_id": "restart.prestop"}, ) + preservation = restart_prestop_preservation( + resumed, + selected_services=selected_services, + include_requirements=include_requirements, + target_projects=target_projects, + ) with ( use_spinner_policy_fn(prestop_policy), spinner_factory("Restarting services...", enabled=use_prestop_spinner) as prestop_spinner, @@ -181,27 +249,42 @@ def handle_restart_prestop( message="Restarting services...", ) try: - runtime._terminate_services_from_state( + termination_result = runtime._terminate_services_from_state( resumed, selected_services=selected_services, aggressive=False, verify_ownership=True, ) - terminate_restart_orphan_listeners( + failed_services = unconfirmed_service_names(termination_result, selected_services) + if failed_services: + session.preserve_existing_state_on_failure = True + raise RuntimeError( + "Restart aborted because existing services could not be stopped: " + + ", ".join(sorted(failed_services)) + ) + orphan_result = terminate_restart_orphan_listeners( state=resumed, selected_services=selected_services, aggressive=True, ) - preservation = restart_prestop_preservation( - resumed, - selected_services=selected_services, - include_requirements=include_requirements, - target_projects=target_projects, - ) + failed_orphan_pids = failed_listener_pids(orphan_result) + if failed_orphan_pids is None: + session.preserve_existing_state_on_failure = True + raise RuntimeError("Restart aborted because orphan listener cleanup was not confirmed") + if failed_orphan_pids: + session.preserve_existing_state_on_failure = True + raise RuntimeError( + "Restart aborted because orphan listeners could not be stopped: " + + ", ".join(str(pid) for pid in sorted(failed_orphan_pids)) + ) for requirements in preservation.requirements_to_release.values(): runtime._release_requirement_ports(requirements) session.preserved_requirements = dict(preservation.preserved_requirements) session.preserved_services = dict(preservation.preserved_services) + session.base_metadata = metadata_with_state_sources( + {**dict(resumed.metadata), **dict(session.base_metadata)}, + resumed, + ) if use_prestop_spinner: prestop_spinner.succeed("Restart pre-stop complete") runtime._emit( @@ -212,6 +295,10 @@ def handle_restart_prestop( message="Restart pre-stop complete", ) except Exception: + session.preserved_services = source_services + session.preserved_requirements = source_requirements + session.base_metadata = source_metadata + session.preserve_existing_state_on_failure = True if use_prestop_spinner: prestop_spinner.fail("Restart pre-stop failed") runtime._emit( @@ -236,11 +323,106 @@ def handle_restart_prestop( selected_services=selected_services, target_projects=target_projects, include_requirements=include_requirements, + service_types_by_project=selection.service_types_by_project, ) session.runtime_mode = restart_lookup_mode + session.preserve_existing_state_on_failure = previous_preserve_flag return None +def _validate_restart_selection( + *, + state: Any, + route: Route, + runtime: Any, + target_projects: set[str], + service_types_by_project: dict[str, set[str]] | None = None, +) -> None: + requested_projects = {str(project).strip() for project in route.projects if str(project).strip()} + if requested_projects: + known_projects = {name.casefold() for name in state_project_names(runtime=runtime, state=state)} + unknown_projects = sorted( + project for project in requested_projects if project.casefold() not in known_projects + ) + if unknown_projects: + raise RuntimeError("No active restart target found for project(s): " + ", ".join(unknown_projects)) + + service_selectors = route.flags.get("services") + requested_services = [ + str(selector).strip() + for selector in (service_selectors if isinstance(service_selectors, list) else []) + if str(selector).strip() + ] + requested_project_keys = {project.casefold() for project in requested_projects} + selector_matches: dict[str, dict[str, set[str]]] = {} + unmatched_services: list[str] = [] + for selector in requested_services: + matches: dict[str, set[str]] = {} + for source in ( + _observed_service_types_for_selector( + state=state, + runtime=runtime, + selector=selector, + ), + _configured_service_projects_for_selector( + state=state, + runtime=runtime, + selector=selector, + ), + ): + for project, service_types in source.items(): + matches.setdefault(project, set()).update(service_types) + scoped_matches = { + project: service_types + for project, service_types in matches.items() + if not requested_project_keys or project.casefold() in requested_project_keys + } + selector_matches[selector] = scoped_matches + if not scoped_matches: + unmatched_services.append(selector) + if unmatched_services: + raise RuntimeError("No matching services found for restart: " + ", ".join(unmatched_services)) + if requested_services and not target_projects: + possible_projects = { + project.casefold() + for matches in selector_matches.values() + for project in matches + } + if len(possible_projects) > 1: + raise RuntimeError( + "Restart service selector is ambiguous across active projects; add --project: " + + ", ".join(requested_services) + ) + raise RuntimeError("No active restart target found for service(s): " + ", ".join(requested_services)) + + if service_types_by_project is None: + service_types_by_project = restart_service_types_by_project( + state=state, + route=route, + runtime=runtime, + target_projects=target_projects, + ) + target_keys = {project.casefold() for project in target_projects} + final_by_key = { + project.casefold(): set(service_types) + for project, service_types in service_types_by_project.items() + } + filtered_services = [ + selector + for selector, matches in selector_matches.items() + if not any( + project.casefold() in target_keys + and bool(service_types.intersection(final_by_key.get(project.casefold(), set()))) + for project, service_types in matches.items() + ) + ] + if filtered_services: + raise RuntimeError( + "Restart service selector conflicts with the requested launch scope: " + + ", ".join(filtered_services) + ) + + def restart_prestop_preservation( state: Any, *, @@ -250,11 +432,15 @@ def restart_prestop_preservation( ) -> RestartPrestopPreservation: requirements_to_release: dict[str, object] = {} preserved_requirements: dict[str, object] = {} - for project_name, requirements in getattr(state, "requirements", {}).items(): - if include_requirements and (not target_projects or project_name in target_projects): - requirements_to_release[project_name] = requirements + target_project_keys = {str(project).strip().casefold() for project in target_projects if str(project).strip()} + for storage_name, requirements in getattr(state, "requirements", {}).items(): + project_name = str(getattr(requirements, "project", "") or storage_name).strip() + if include_requirements and ( + not target_project_keys or project_name.casefold() in target_project_keys + ): + requirements_to_release[storage_name] = requirements else: - preserved_requirements[project_name] = requirements + preserved_requirements[storage_name] = requirements preserved_services = { name: service for name, service in getattr(state, "services", {}).items() if name not in selected_services } @@ -342,6 +528,8 @@ def restart_orphan_listener_scan( "frontend": set(range(int(frontend_port_base), int(frontend_port_base) + span)), } selected_by_cwd: dict[str, set[str]] = {} + owners_by_cwd_type: dict[tuple[str, str], tuple[str, ...]] = {} + port_lock_sessions_by_cwd_type: dict[tuple[str, str], tuple[str, ...]] = {} for service_name, service in getattr(state, "services", {}).items(): if service_name not in selected_services: continue @@ -350,6 +538,20 @@ def restart_orphan_listener_scan( if service_type not in ports_by_type or not cwd: continue selected_by_cwd.setdefault(cwd, set()).add(service_type) + project = str(service_project_name(service) or service_name.rsplit(" ", 1)[0]).strip() + if project: + owners_by_cwd_type[(cwd, service_type)] = ( + f"{project}:{service_type}", + f"{project}:services:{service_type}-launch", + f"{project}:services", + ) + raw_port_lock_session = getattr(service, "port_lock_session", None) + port_lock_session = raw_port_lock_session.strip() if isinstance(raw_port_lock_session, str) else "" + if port_lock_session: + key = (cwd, service_type) + port_lock_sessions_by_cwd_type[key] = tuple( + dict.fromkeys((*port_lock_sessions_by_cwd_type.get(key, ()), port_lock_session)) + ) for attr_name in ("actual_port", "requested_port"): port = getattr(service, attr_name, None) if isinstance(port, int) and port > 0: @@ -357,6 +559,8 @@ def restart_orphan_listener_scan( return RestartOrphanListenerScan( ports_by_type=ports_by_type, selected_by_cwd=selected_by_cwd, + owners_by_cwd_type=owners_by_cwd_type, + port_lock_sessions_by_cwd_type=port_lock_sessions_by_cwd_type, ) @@ -377,7 +581,14 @@ def restart_matching_orphan_listeners( if process_cwd(pid) != cwd: continue seen_pids.add(pid) - matches.append(RestartOrphanListenerMatch(pid=pid, port=port)) + matches.append( + RestartOrphanListenerMatch( + pid=pid, + port=port, + owner_candidates=scan.owners_by_cwd_type.get((cwd, service_type), ()), + port_lock_sessions=scan.port_lock_sessions_by_cwd_type.get((cwd, service_type), ()), + ) + ) return matches @@ -392,8 +603,8 @@ def terminate_restart_orphan_listeners( listener_pids_for_port: Callable[[int], Iterable[int]] | None, process_cwd: Callable[[int], str | None], terminate_pid: Callable[..., bool] | None, - release_port: Callable[[int], None], -) -> None: + port_planner: Any, +) -> set[int]: scan = restart_orphan_listener_scan( state, selected_services=selected_services, @@ -402,17 +613,29 @@ def terminate_restart_orphan_listeners( port_spacing=port_spacing, ) if not scan.selected_by_cwd: - return + return set() if not callable(listener_pids_for_port) or not callable(terminate_pid): - return + return set() matches = restart_matching_orphan_listeners( scan, listener_pids_for_port=listener_pids_for_port, process_cwd=process_cwd, ) + failed_pids: set[int] = set() for match in matches: if terminate_pid(match.pid, term_timeout=0.5 if aggressive else 2.0, kill_timeout=1.0): - release_port(match.port) + expected_sessions = match.port_lock_sessions or (None,) + for expected_session in expected_sessions: + if release_port_reservation( + port_planner, + match.port, + owner_candidates=match.owner_candidates, + expected_session=expected_session, + ): + break + else: + failed_pids.add(match.pid) + return failed_pids def terminate_restart_orphan_listeners_with_runtime( @@ -421,10 +644,10 @@ def terminate_restart_orphan_listeners_with_runtime( state: Any, selected_services: set[str], aggressive: bool, -) -> None: +) -> set[int]: process_runtime = resolve_process_runtime(runtime) port_allocator = resolve_port_allocator(runtime) - terminate_restart_orphan_listeners( + return terminate_restart_orphan_listeners( state=state, selected_services=selected_services, aggressive=aggressive, @@ -432,7 +655,7 @@ def terminate_restart_orphan_listeners_with_runtime( frontend_port_base=int(runtime.config.frontend_port_base), port_spacing=int(getattr(runtime.config, "port_spacing", 20) or 20), listener_pids_for_port=getattr(runtime, "_listener_pids_for_port", None), - process_cwd=process_cwd, + process_cwd=getattr(process_runtime, "process_cwd", process_cwd), terminate_pid=getattr(process_runtime, "terminate", None), - release_port=port_allocator.release, + port_planner=port_allocator, ) diff --git a/python/envctl_engine/startup/resume_orchestrator.py b/python/envctl_engine/startup/resume_orchestrator.py index c9773809c..6ef5a440f 100644 --- a/python/envctl_engine/startup/resume_orchestrator.py +++ b/python/envctl_engine/startup/resume_orchestrator.py @@ -6,7 +6,10 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_startup_support import mark_run_reused +from envctl_engine.runtime.lifecycle_operation_lease import release_lifecycle_operation +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.state.models import RunState +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.ui.spinner import spinner, spinner_enabled, use_spinner_policy from envctl_engine.ui.spinner_service import emit_spinner_policy, resolve_spinner_policy @@ -47,9 +50,17 @@ def emit_phase(phase: str, started_at: float, **extra: object) -> None: ) state_load_started = time.monotonic() - state = rt._try_load_existing_state( # type: ignore[attr-defined] + raw_state_projects = route.flags.get("_state_project_names") + state_projects = ( + [str(project).strip() for project in raw_state_projects if str(project).strip()] + if isinstance(raw_state_projects, list) + else list(route.projects) + ) + state = call_state_loader( + rt._try_load_existing_state, # type: ignore[attr-defined] mode=route.mode, strict_mode_match=rt._state_lookup_strict_mode_match(route), # type: ignore[attr-defined] + project_names=state_projects or None, ) emit_phase("state_load", state_load_started, found=state is not None) if state is None: @@ -108,17 +119,22 @@ def emit_phase(phase: str, started_at: float, **extra: object) -> None: ) restore_started = time.monotonic() restore_errors = self.restore_missing(state, missing_services, route=route) + restored_projects = { + project.casefold() + for name in missing_services + if ( + project := resolve_service_project_name( + name, + state.services.get(name), + project_name_from_service=rt._project_name_from_service, # type: ignore[attr-defined] + ) + ) + } emit_phase( "restore_missing", restore_started, status="error" if restore_errors else "ok", - project_count=len( - { - rt._project_name_from_service(name) - for name in missing_services - if rt._project_name_from_service(name) - } - ), # type: ignore[attr-defined] + project_count=len(restored_projects), error_count=len(restore_errors), ) if legacy_resume: @@ -156,6 +172,17 @@ def emit_phase(phase: str, started_at: float, **extra: object) -> None: ) enter_interactive = bool(rt._should_enter_resume_interactive(route)) # type: ignore[attr-defined] + if not rt._state_has_resumable_services(state): # type: ignore[attr-defined] + rt._emit( # type: ignore[attr-defined] + "resume.empty_state", + run_id=state.run_id, + reason="no_resumable_services", + ) + print("No active services to resume.") + return 1 + + release_lifecycle_operation(rt) + if not enter_interactive: current_session_id = getattr(rt, "_current_session_id", None) session_id = current_session_id() if callable(current_session_id) else None @@ -173,14 +200,6 @@ def emit_phase(phase: str, started_at: float, **extra: object) -> None: print("Warning: stale services detected during resume: " + ", ".join(missing_services)) if restore_errors: print("Warning: resume restore encountered errors: " + "; ".join(restore_errors)) - if not rt._state_has_resumable_services(state): # type: ignore[attr-defined] - rt._emit( # type: ignore[attr-defined] - "resume.empty_state", - run_id=state.run_id, - reason="no_resumable_services", - ) - print("No active services to resume.") - return 1 if enter_interactive: return rt._run_interactive_dashboard_loop(state) # type: ignore[attr-defined] return 0 diff --git a/python/envctl_engine/startup/resume_restore_execution.py b/python/envctl_engine/startup/resume_restore_execution.py index 6367c0661..54fdb6847 100644 --- a/python/envctl_engine/startup/resume_restore_execution.py +++ b/python/envctl_engine/startup/resume_restore_execution.py @@ -7,6 +7,8 @@ from typing import Callable, Mapping from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.docker_service_runtime import state_uses_docker_services +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.state.models import RunState from envctl_engine.startup.resume_restore_project import ResumeProjectRestoreRunner, _round_ms from envctl_engine.startup.resume_restore_results import apply_restore_result @@ -113,6 +115,7 @@ def _execute_restore_missing( use_single_spinner = use_spinner and not use_project_spinner_group suppress_timing_output = use_project_spinner_group or use_single_spinner route_for_startup = _route_for_resume_restore(route, state=state, suppress_timing_output=suppress_timing_output) + _apply_restore_runtime_mode(rt, route_for_startup) if use_single_spinner: rt._emit( # type: ignore[attr-defined] "ui.spinner.lifecycle", @@ -211,15 +214,24 @@ def _execute_restore_missing( def _projects_to_restore(rt: object, *, state: RunState, missing_services: list[str]) -> list[str]: - projects_to_restore: set[str] = set() - for service_name in missing_services: - project = rt._project_name_from_service(service_name) # type: ignore[attr-defined] + projects_to_restore: dict[str, str] = {} + + def add_project(raw_project: object) -> None: + project = str(raw_project or "").strip() if project: - projects_to_restore.add(project) + projects_to_restore.setdefault(project.casefold(), project) + + for service_name in missing_services: + project = resolve_service_project_name( + service_name, + state.services.get(service_name), + project_name_from_service=rt._project_name_from_service, # type: ignore[attr-defined] + ) + add_project(project) for project, requirements in state.requirements.items(): if not rt._requirements_ready(requirements): # type: ignore[attr-defined] - projects_to_restore.add(project) - return sorted(projects_to_restore) + add_project(getattr(requirements, "project", "") or project) + return sorted(projects_to_restore.values(), key=lambda project: (project.casefold(), project)) def _route_for_resume_restore( @@ -240,6 +252,8 @@ def _route_for_resume_restore( **route_for_startup.flags, "_resume_restore": True, } + if state_uses_docker_services(state): + flags["docker"] = True if suppress_timing_output: flags["_suppress_timing_print"] = True return Route( @@ -252,6 +266,11 @@ def _route_for_resume_restore( ) +def _apply_restore_runtime_mode(runtime: object, route: Route) -> None: + if bool(route.flags.get("docker")): + getattr(runtime, "env")["DOCKER_MODE"] = "true" + + def _run_restore_jobs( *, sorted_projects: list[str], diff --git a/python/envctl_engine/startup/resume_restore_policy.py b/python/envctl_engine/startup/resume_restore_policy.py index a751cc429..950b17a96 100644 --- a/python/envctl_engine/startup/resume_restore_policy.py +++ b/python/envctl_engine/startup/resume_restore_policy.py @@ -8,7 +8,33 @@ from envctl_engine.requirements.core import dependency_definitions from envctl_engine.shared.parsing import parse_bool from envctl_engine.shared.protocols import PortAllocator, StateRepository +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord +from envctl_engine.state.project_runtime import ( + requirement_key_for_project as _requirement_key_for_project, + requirements_for_project as _requirements_for_project, +) + + +def _store_requirements_for_project( + state: RunState, + *, + project: str, + requirements: RequirementsResult, +) -> str: + """Update one project without creating case-only duplicate keys.""" + + target = str(project).strip().casefold() + key = _requirement_key_for_project(state, project) or str(project).strip() + if not key: + raise ValueError("project is required when storing restored requirements") + state.requirements[key] = requirements + for alias in list(state.requirements): + if alias == key: + continue + if str(alias).strip().casefold() == target: + state.requirements.pop(alias, None) + return key def _restore_parallel_config( @@ -204,7 +230,7 @@ def context_for_project(orchestrator, state: RunState, project: str) -> object | rt = orchestrator.runtime discovered = rt._discover_projects(mode=state.mode) # type: ignore[attr-defined] for context in discovered: - if context.name.lower() == project.lower(): + if context.name.casefold() == project.casefold(): orchestrator.apply_ports_to_context(context, state) return context @@ -235,7 +261,12 @@ def project_root(orchestrator, state: RunState, project: str) -> Path | None: return rt.config.base_dir # type: ignore[attr-defined] for service_name, service in state.services.items(): - if rt._project_name_from_service(service_name).lower() != project.lower(): # type: ignore[attr-defined] + service_project = resolve_service_project_name( + service_name, + service, + project_name_from_service=rt._project_name_from_service, # type: ignore[attr-defined] + ) + if service_project.casefold() != project.casefold(): continue cwd_raw = getattr(service, "cwd", None) if not isinstance(cwd_raw, str) or not cwd_raw.strip(): @@ -258,7 +289,7 @@ def apply_ports_to_context(orchestrator, context: object, state: RunState) -> No if not isinstance(context_ports, dict): return - requirements = state.requirements.get(project) + requirements = _requirements_for_project(state, project) if requirements is not None: for definition in dependency_definitions(): plan = context_ports.get(definition.resources[0].legacy_port_key) @@ -267,7 +298,12 @@ def apply_ports_to_context(orchestrator, context: object, state: RunState) -> No rt._set_plan_port_from_component(plan, requirements.component(definition.id)) # type: ignore[attr-defined] for service_name, service in state.services.items(): - if rt._project_name_from_service(service_name).lower() != project.lower(): # type: ignore[attr-defined] + service_project = resolve_service_project_name( + service_name, + service, + project_name_from_service=rt._project_name_from_service, # type: ignore[attr-defined] + ) + if service_project.casefold() != project.casefold(): continue service_type = (getattr(service, "type", "") or "").strip().lower() port = getattr(service, "actual_port", None) diff --git a/python/envctl_engine/startup/resume_restore_project.py b/python/envctl_engine/startup/resume_restore_project.py index efb52a0a4..94c14465e 100644 --- a/python/envctl_engine/startup/resume_restore_project.py +++ b/python/envctl_engine/startup/resume_restore_project.py @@ -5,9 +5,12 @@ import time from envctl_engine.runtime.command_router import Route -from envctl_engine.state.models import RunState, ServiceRecord +from envctl_engine.runtime.lifecycle_service_termination import release_service_ports +from envctl_engine.shared.services import resolve_service_project_name +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord from envctl_engine.startup.resume_restore_policy import ( _configured_restore_service_types, + _requirements_for_project, _requirements_reuse_decision, _reserve_application_service_ports, _resume_terminate_aggressive, @@ -16,6 +19,21 @@ ) +class _StaleServiceTerminationError(RuntimeError): + def __init__( + self, + *, + unconfirmed_service_names: list[str], + confirmed_service_names: list[str], + ) -> None: + self.unconfirmed_service_names = list(unconfirmed_service_names) + self.confirmed_service_names = list(confirmed_service_names) + super().__init__( + "stale service termination could not be confirmed: " + + ", ".join(sorted(self.unconfirmed_service_names)) + ) + + @dataclass(slots=True) class ResumeProjectRestoreRunner: orchestrator: object @@ -101,15 +119,20 @@ def mark_step( ) project_service_names = [ name - for name in list(self.state.services.keys()) - if self.rt._project_name_from_service(name).lower() == project.lower() # type: ignore[attr-defined] + for name, service in list(self.state.services.items()) + if resolve_service_project_name( + name, + service, + project_name_from_service=self.rt._project_name_from_service, # type: ignore[attr-defined] + ).casefold() + == project.casefold() ] original_services: dict[str, ServiceRecord] = { name: self.state.services[name] for name in project_service_names if name in self.state.services } missing_set = set(self.missing_services) missing_project_service_names = [name for name in project_service_names if name in missing_set] - original_requirements = self.state.requirements.get(project) + original_requirements = _requirements_for_project(self.state, project) context_root = getattr(context, "root", None) requirements_reused, requirements_reuse_reason = _requirements_reuse_decision( self.orchestrator, @@ -144,21 +167,42 @@ def mark_step( elif configured_service_types: selected_service_types = set(configured_service_types) - existing_requirements = original_requirements - if existing_requirements is not None and not requirements_reused: + confirmed_old_service_names: list[str] = [] + replacement_requirements: RequirementsResult | None = None + try: self._report_progress( project, - f"{prefix} Releasing previous requirement ports...", - single_message=f"{prefix} Releasing previous requirement ports for {project}...", + f"{prefix} Stopping stale services...", + single_message=f"{prefix} Stopping stale services for {project}...", ) - release_started = time.monotonic() - self.rt._release_requirement_ports(existing_requirements) # type: ignore[attr-defined] + stop_started = time.monotonic() + terminated_count = self._stop_stale_services( + service_names=selected_service_names, + original_services=original_services, + ) + confirmed_old_service_names = list(selected_service_names) mark_step( - "release_requirement_ports", - duration_ms=_round_ms(time.monotonic() - release_started), + "stop_stale_services", + duration_ms=_round_ms(time.monotonic() - stop_started), + stale_service_count=len(selected_service_names), + terminated_count=terminated_count, + aggressive=_resume_terminate_aggressive(self.orchestrator, self.rt), ) - try: + existing_requirements = original_requirements + if existing_requirements is not None and not requirements_reused: + self._report_progress( + project, + f"{prefix} Releasing previous requirement ports...", + single_message=f"{prefix} Releasing previous requirement ports for {project}...", + ) + release_started = time.monotonic() + self.rt._release_requirement_ports(existing_requirements) # type: ignore[attr-defined] + mark_step( + "release_requirement_ports", + duration_ms=_round_ms(time.monotonic() - release_started), + ) + if requirements_reused and requirements_for_services is not None: requirements = requirements_for_services mark_step("start_requirements", duration_ms=0.0, status="reused") @@ -168,6 +212,7 @@ def mark_step( requirements = self.rt._start_requirements_for_project( context, mode=self.state.mode, route=project_route_for_startup ) # type: ignore[attr-defined] + replacement_requirements = requirements mark_step( "start_requirements", duration_ms=_round_ms(time.monotonic() - requirements_started), @@ -194,26 +239,10 @@ def mark_step( "error": error_message, "steps": step_durations_ms, "total_ms": project_total, + "remove_service_names": confirmed_old_service_names, + "requirements": requirements, } - self._report_progress( - project, - f"{prefix} Stopping stale services...", - single_message=f"{prefix} Stopping stale services for {project}...", - ) - stop_started = time.monotonic() - terminated_count = self._stop_stale_services( - service_names=selected_service_names, - original_services=original_services, - ) - mark_step( - "stop_stale_services", - duration_ms=_round_ms(time.monotonic() - stop_started), - stale_service_count=len(selected_service_names), - terminated_count=terminated_count, - aggressive=_resume_terminate_aggressive(self.orchestrator, self.rt), - ) - self._report_progress( project, f"{prefix} Reserving service ports...", @@ -274,7 +303,25 @@ def mark_step( "requirements": requirements, "remove_service_names": selected_service_names, } - except RuntimeError as exc: + except Exception as exc: # noqa: BLE001 + confirmed_from_error = getattr(exc, "confirmed_service_names", None) + if isinstance(confirmed_from_error, list): + confirmed_old_service_names = [ + name for name in confirmed_from_error if isinstance(name, str) and name + ] + exception_requirements = getattr(exc, "requirements", None) + if isinstance(exception_requirements, RequirementsResult): + replacement_requirements = exception_requirements + raw_unterminated = getattr(exc, "unterminated_services", None) + unterminated_services = ( + { + name: service + for name, service in raw_unterminated.items() + if isinstance(name, str) and isinstance(service, ServiceRecord) + } + if isinstance(raw_unterminated, dict) + else {} + ) mark_step("exception", status="error", error=str(exc)) self._mark_failure( project, @@ -294,6 +341,9 @@ def mark_step( "error": str(exc), "steps": step_durations_ms, "total_ms": project_total, + "remove_service_names": confirmed_old_service_names, + "requirements": replacement_requirements, + "unterminated_services": unterminated_services, } def _route_with_spinner_updates(self, project: str, prefix: str) -> Route: @@ -337,6 +387,7 @@ def _stop_stale_services( original_services: dict[str, ServiceRecord], ) -> int: terminated_count = 0 + unconfirmed: list[str] = [] aggressive_terminate = _resume_terminate_aggressive(self.orchestrator, self.rt) for service_name in service_names: service = original_services.get(service_name) @@ -349,9 +400,16 @@ def _stop_stale_services( ) if terminated: terminated_count += 1 - port = self.rt._service_port(service) # type: ignore[attr-defined] - if port is not None: - self.port_allocator.release(port) # type: ignore[attr-defined] + release_service_ports(self.rt, name=service_name, service=service) + else: + unconfirmed.append(service_name) + if unconfirmed: + raise _StaleServiceTerminationError( + unconfirmed_service_names=unconfirmed, + confirmed_service_names=[ + name for name in service_names if name in original_services and name not in unconfirmed + ], + ) return terminated_count def _report_progress( diff --git a/python/envctl_engine/startup/resume_restore_results.py b/python/envctl_engine/startup/resume_restore_results.py index 2ba46d10d..1a589c15e 100644 --- a/python/envctl_engine/startup/resume_restore_results.py +++ b/python/envctl_engine/startup/resume_restore_results.py @@ -1,7 +1,12 @@ from __future__ import annotations from collections.abc import Mapping +from dataclasses import replace +from envctl_engine.startup.resume_restore_policy import ( + _requirements_for_project, + _store_requirements_for_project, +) from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord @@ -22,29 +27,95 @@ def apply_restore_result( if timing_enabled: timing_lines.append(format_project_timing_line(project, step_map, project_total)) - error_message = str(result.get("error") or "").strip() - if error_message: - mark_restore_failure_requirements(state, project=project, error=error_message) - errors.append(f"{project}: {error_message}") - return - remove_names_raw = result.get("remove_service_names") if isinstance(remove_names_raw, list): for service_name in remove_names_raw: if isinstance(service_name, str): + existing = state.services.get(service_name) + existing_project = str(getattr(existing, "project", "") or "").strip() + if existing_project and existing_project.casefold() != project.casefold(): + continue state.services.pop(service_name, None) project_services_raw = result.get("project_services") if isinstance(project_services_raw, dict): - for service_name, record in project_services_raw.items(): - if isinstance(service_name, str) and isinstance(record, ServiceRecord): - state.services[service_name] = record + _store_project_services(state, project=project, services=project_services_raw) requirements_result = result.get("requirements") if isinstance(requirements_result, RequirementsResult): - state.requirements[project] = requirements_result + _store_requirements_for_project( + state, + project=project, + requirements=requirements_result, + ) + raw_unterminated = result.get("unterminated_services") + if isinstance(raw_unterminated, dict): + stored_names = _store_unterminated_services(state, raw_unterminated) + if stored_names: + existing_failed = state.metadata.get("termination_failed_services") + failed_names = ( + [name for name in existing_failed if isinstance(name, str)] + if isinstance(existing_failed, list) + else [] + ) + state.metadata["termination_failed_services"] = sorted(set(failed_names).union(stored_names)) + + error_message = str(result.get("error") or "").strip() + if error_message: + mark_restore_failure_requirements(state, project=project, error=error_message) + errors.append(f"{project}: {error_message}") + + +def _store_unterminated_services(state: RunState, services: dict[object, object]) -> list[str]: + occupied = set(state.services) + stored_names: list[str] = [] + for raw_name, service in services.items(): + if not isinstance(service, ServiceRecord): + continue + name = str(raw_name).strip() or service.name + stored_name = name + if stored_name in occupied: + stored_name = _unique_resume_collision_name(name, service, occupied) + occupied.add(stored_name) + state.services[stored_name] = replace( + service, + name=stored_name, + status="termination_failed", + degraded=True, + ) + stored_names.append(stored_name) + return stored_names + + +def _store_project_services(state: RunState, *, project: str, services: dict[object, object]) -> None: + occupied = set(state.services) + for raw_name, service in services.items(): + if not isinstance(service, ServiceRecord): + continue + name = str(raw_name).strip() or service.name + stored_service = service if str(service.project or "").strip() else replace(service, project=project) + stored_name = name + if stored_name in occupied: + stored_name = _unique_resume_collision_name(name, stored_service, occupied) + stored_service = replace(stored_service, name=stored_name) + elif stored_service.name != stored_name: + stored_service = replace(stored_service, name=stored_name) + occupied.add(stored_name) + state.services[stored_name] = stored_service + + +def _unique_resume_collision_name(name: str, service: ServiceRecord, occupied: set[str]) -> str: + pid = service.pid + suffix = f"Resume Collision {pid}" if isinstance(pid, int) and pid > 0 else "Resume Collision" + base = f"{name} {suffix}" + candidate = base + index = 2 + while candidate in occupied: + candidate = f"{base} {index}" + index += 1 + return candidate def mark_restore_failure_requirements(state: RunState, *, project: str, error: str) -> None: - existing = state.requirements.get(project) + existing = _requirements_for_project(state, project) if not isinstance(existing, RequirementsResult): return existing.health = "degraded" diff --git a/python/envctl_engine/startup/resume_restore_support.py b/python/envctl_engine/startup/resume_restore_support.py index 03b9f73cb..0f89d1376 100644 --- a/python/envctl_engine/startup/resume_restore_support.py +++ b/python/envctl_engine/startup/resume_restore_support.py @@ -9,10 +9,6 @@ ResumeRestoreDependencies, ResumeRestoreRunner, ) -from envctl_engine.startup.resume_restore_results import ( - format_project_timing_line as _format_project_timing_line, # noqa: F401 - mark_restore_failure_requirements as _mark_restore_failure_requirements, # noqa: F401 -) from envctl_engine.startup.resume_restore_policy import ( _configured_restore_service_types as _configured_restore_service_types, _port_allocator as _port_allocator, diff --git a/python/envctl_engine/startup/run_reuse_dashboard_restore.py b/python/envctl_engine/startup/run_reuse_dashboard_restore.py index 05e23a34f..515af1e59 100644 --- a/python/envctl_engine/startup/run_reuse_dashboard_restore.py +++ b/python/envctl_engine/startup/run_reuse_dashboard_restore.py @@ -4,8 +4,11 @@ from dataclasses import dataclass from typing import Any +from envctl_engine.dashboard_metadata import normalize_dashboard_service_types from envctl_engine.runtime.command_router import Route +from envctl_engine.shared.services import service_display_name from envctl_engine.startup.run_reuse_decision import mark_run_reused +from envctl_engine.startup.session import metadata_with_state_sources def dashboard_stopped_service_entries(state: object) -> list[dict[str, str]]: @@ -17,15 +20,16 @@ def dashboard_stopped_service_entries(state: object) -> list[dict[str, str]]: if not isinstance(item, Mapping): continue project = str(item.get("project", "") or "").strip() - service_type = str(item.get("type", "") or "").strip().lower() + normalized_types = normalize_dashboard_service_types([item.get("type", "")]) + service_type = normalized_types[0] if normalized_types else "" name = str(item.get("name", "") or "").strip() - if not project or service_type not in {"backend", "frontend"}: + if not project or not service_type: continue entries.append( { "project": project, "type": service_type, - "name": name or f"{project} {service_type.title()}", + "name": name or f"{project} {service_display_name(service_type)}", } ) return entries @@ -34,19 +38,32 @@ def dashboard_stopped_service_entries(state: object) -> list[dict[str, str]]: def metadata_without_dashboard_stopped_services( metadata: Mapping[str, object], *, - restored_service_names: set[str], + restored_service_names: set[str] | None = None, + restored_service_types_by_project: Mapping[str, object] | None = None, ) -> dict[str, object]: updated = dict(metadata) raw = updated.get("dashboard_stopped_services") if not isinstance(raw, list): return updated + restored_names = { + str(selector).strip().casefold().removeprefix("service:").strip() + for selector in (restored_service_names or set()) + if str(selector).strip() + } + restored_types = { + str(project).strip().casefold(): set(normalize_dashboard_service_types(service_types)) + for project, service_types in (restored_service_types_by_project or {}).items() + if str(project).strip() and isinstance(service_types, (list, tuple, set, frozenset)) + } remaining: list[object] = [] for item in raw: if not isinstance(item, Mapping): remaining.append(item) continue name = str(item.get("name", "") or "").strip() - if name in restored_service_names: + project = str(item.get("project", "") or "").strip() + service_type = str(item.get("type", "") or "").strip().lower() + if name.casefold() in restored_names or service_type in restored_types.get(project.casefold(), set()): continue remaining.append(dict(item)) if remaining: @@ -104,9 +121,9 @@ def _apply_restore(self, *, restore_entries: list[dict[str, str]], contexts_to_s stopped_service_names = sorted({entry["name"] for entry in restore_entries}) stopped_service_types = sorted({entry["type"] for entry in restore_entries}) target_project_names = sorted({entry["project"] for entry in restore_entries}, key=str.casefold) - self.session.base_metadata = metadata_without_dashboard_stopped_services( + self.session.base_metadata = metadata_with_state_sources( mark_run_reused(self.candidate_state.metadata, reason="restore_stopped_services"), - restored_service_names=set(stopped_service_names), + self.candidate_state, ) self.session.preserved_services = dict(self.candidate_state.services) self.session.preserved_requirements = dict(self.candidate_state.requirements) @@ -115,6 +132,16 @@ def _apply_restore(self, *, restore_entries: list[dict[str, str]], contexts_to_s target_project_names=target_project_names, stopped_service_names=stopped_service_names, stopped_service_types=stopped_service_types, + stopped_service_types_by_project={ + project: sorted( + { + entry["type"] + for entry in restore_entries + if entry["project"].casefold() == project.casefold() + } + ) + for project in target_project_names + }, ) self._emit_restore( target_project_names=target_project_names, @@ -127,6 +154,7 @@ def _apply_restart_route( target_project_names: list[str], stopped_service_names: list[str], stopped_service_types: list[str], + stopped_service_types_by_project: dict[str, list[str]], ) -> None: route = self.session.effective_route self.session.effective_route = Route( @@ -141,39 +169,47 @@ def _apply_restart_route( "_restore_dashboard_stopped_services": True, "services": stopped_service_names, "restart_service_types": stopped_service_types, + "_restart_service_types_by_project": stopped_service_types_by_project, "restart_include_requirements": False, }, ) def _emit_restore(self, *, target_project_names: list[str], stopped_service_names: list[str]) -> None: route = self.session.effective_route - self.emit_phase( - self.session, - "auto_resume_evaluate", - self.reuse_started, - status="restore_stopped_services", - match_mode="exact" if self.decision_kind == "resume_exact" else "subset", - stopped_service_count=len(stopped_service_names), - target_projects=target_project_names, - ) - self.runtime._emit( - "state.auto_resume.restore_stopped_services", - run_id=self.candidate_state.run_id, - mode=self.session.runtime_mode, - command=route.command, - projects=target_project_names, - services=stopped_service_names, - ) - self.runtime._emit( - "state.run_reuse.applied", - run_id=self.candidate_state.run_id, - mode=self.session.runtime_mode, - command=route.command, - decision_kind="restore_stopped_services", - reason="dashboard_stopped_services", - restored_projects=target_project_names, - restored_services=stopped_service_names, + diagnostics = ( + lambda: self.emit_phase( + self.session, + "auto_resume_evaluate", + self.reuse_started, + status="restore_stopped_services", + match_mode="exact" if self.decision_kind == "resume_exact" else "subset", + stopped_service_count=len(stopped_service_names), + target_projects=target_project_names, + ), + lambda: self.runtime._emit( + "state.auto_resume.restore_stopped_services", + run_id=self.candidate_state.run_id, + mode=self.session.runtime_mode, + command=route.command, + projects=target_project_names, + services=stopped_service_names, + ), + lambda: self.runtime._emit( + "state.run_reuse.applied", + run_id=self.candidate_state.run_id, + mode=self.session.runtime_mode, + command=route.command, + decision_kind="restore_stopped_services", + reason="dashboard_stopped_services", + restored_projects=target_project_names, + restored_services=stopped_service_names, + ), ) + for diagnostic in diagnostics: + try: + diagnostic() + except Exception: # noqa: BLE001 + pass def prepare_dashboard_stopped_service_restore( diff --git a/python/envctl_engine/startup/run_reuse_decision.py b/python/envctl_engine/startup/run_reuse_decision.py index c97bcfc65..2cf58de0b 100644 --- a/python/envctl_engine/startup/run_reuse_decision.py +++ b/python/envctl_engine/startup/run_reuse_decision.py @@ -5,8 +5,11 @@ from typing import Any, Mapping from envctl_engine.runtime.command_router import Route +from envctl_engine.shared.services import resolve_service_project_name +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RunState from envctl_engine.startup.run_reuse_identity import ( + ProjectIdentity, _auto_resume_start_enabled, _identity_keys, _root_mismatches, @@ -56,20 +59,37 @@ def evaluate(self) -> RunReuseDecision: selected_payload=selected_payload, startup_enabled=startup_enabled, ) - if not _auto_resume_start_enabled(self.route): + auto_resume_enabled = _auto_resume_start_enabled(self.route) + explicit_fresh_start = bool(self.route.flags.get("no_resume")) + if not auto_resume_enabled and not explicit_fresh_start: return self._fresh_without_state( reason="auto_resume_disabled", selected_payload=selected_payload, startup_enabled=startup_enabled, ) - candidate = self.runtime._try_load_existing_state(mode=self.runtime_mode, strict_mode_match=True) + candidate = call_state_loader( + self.runtime._try_load_existing_state, + mode=self.runtime_mode, + strict_mode_match=True, + project_names=[identity.name for identity in selected_identities], + ) if candidate is None: return self._fresh_without_state( reason="no_matching_state", selected_payload=selected_payload, startup_enabled=startup_enabled, ) + if explicit_fresh_start: + state_identities = project_identities_from_state(self.runtime, candidate) + return RunReuseDecision( + candidate_state=candidate, + decision_kind="fresh_run", + reason="explicit_no_resume", + selected_projects=selected_payload, + state_projects=identities_to_payload(state_identities), + startup_enabled=startup_enabled, + ) return self._evaluate_candidate( candidate=candidate, selected_identities=selected_identities, @@ -97,7 +117,7 @@ def _evaluate_candidate( self, *, candidate: RunState, - selected_identities: list[object], + selected_identities: list[ProjectIdentity], selected_payload: list[dict[str, str | None]], startup_enabled: bool, ) -> RunReuseDecision: @@ -200,7 +220,11 @@ def _initial_candidate_rejection( ) return None - def _startup_identity_mismatch(self, candidate: RunState, state_identities: list[object]) -> dict[str, object]: + def _startup_identity_mismatch( + self, + candidate: RunState, + state_identities: list[ProjectIdentity], + ) -> dict[str, object]: previous_identity = candidate.metadata.get("startup_identity") if not isinstance(previous_identity, dict): return {} @@ -317,8 +341,8 @@ def mark_run_reused(metadata: Mapping[str, object] | None, *, reason: str) -> di updated = dict(metadata or {}) raw_count = updated.get("run_reuse_count", 0) try: - count = int(raw_count) - except (TypeError, ValueError): + count = int(raw_count) if isinstance(raw_count, (bool, float, int, str)) else 0 + except (OverflowError, TypeError, ValueError): count = 0 updated["run_reuse_count"] = max(0, count) + 1 updated["last_reopened_at"] = datetime.now(tz=UTC).isoformat() @@ -343,7 +367,11 @@ def evaluate_run_reuse( def state_has_resumable_services(runtime: Any, state: RunState) -> bool: for service_name, service in state.services.items(): - project_name = runtime._project_name_from_service(service_name) + project_name = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) if not project_name: continue service_type = str(getattr(service, "type", "")).strip().lower() diff --git a/python/envctl_engine/startup/run_reuse_fresh_start.py b/python/envctl_engine/startup/run_reuse_fresh_start.py index 2cb75fc60..fae3b4e76 100644 --- a/python/envctl_engine/startup/run_reuse_fresh_start.py +++ b/python/envctl_engine/startup/run_reuse_fresh_start.py @@ -5,42 +5,62 @@ from typing import Any from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.lifecycle_service_termination import failed_listener_pids, unconfirmed_service_names from envctl_engine.shared.services import service_project_name, service_slug_from_record +from envctl_engine.startup.session import metadata_with_state_sources from envctl_engine.startup.startup_selection_support import _restart_service_types_for_project def fresh_start_replacement_services( *, - route: Route, selected_contexts: list[object], candidate_state: object, - configured_service_types: set[str], - additional_services: tuple[object, ...], project_name_from_service: Callable[[str], str], + route: Route | None = None, + additional_services: tuple[object, ...] = (), ) -> set[str]: - target_projects = {str(context.name).strip().lower() for context in selected_contexts} - target_projects.discard("") - if not target_projects: - return set() - selected_by_project = { - str(context.name).strip().lower(): _restart_service_types_for_project( - route=route, - project_name=str(context.name), - default_service_types=configured_service_types, - additional_services=additional_services, - ) + """Select old services that overlap the requested fresh-start scope.""" + + target_projects = { + str(context.name).strip().casefold(): str(context.name).strip() for context in selected_contexts if str(context.name).strip() } - selected: set[str] = set() + if not target_projects: + return set() + + services_by_project: dict[str, list[tuple[str, object, str]]] = { + project: [] for project in target_projects + } for service_name, service in getattr(candidate_state, "services", {}).items(): - project = service_project_name(service) or project_name_from_service(service_name) - project_key = str(project).strip().lower() - if project_key not in target_projects: + project = str(service_project_name(service) or project_name_from_service(service_name)).strip().casefold() + if project not in target_projects: + continue + services_by_project[project].append( + (service_name, service, _service_type(service_name, service)) + ) + + selected: set[str] = set() + for project, project_services in services_by_project.items(): + if route is None or not _route_has_app_service_scope(route): + selected.update(name for name, _service, _service_type_name in project_services) continue - service_type = service_slug_from_record(service) - if service_type and service_type in selected_by_project.get(project_key, set()): - selected.add(service_name) + configured_types = { + service_type_name + for _name, _service, service_type_name in project_services + if service_type_name + } + selected_types = _restart_service_types_for_project( + route=route, + project_name=target_projects[project], + default_service_types=configured_types, + additional_services=additional_services, + ) + selected.update( + name + for name, _service, service_type_name in project_services + if service_type_name in selected_types + ) return selected @@ -50,43 +70,162 @@ class FreshStartServiceReplacer: session: Any candidate_state: Any reason: str - fresh_start_replacement_services: Callable[..., set[str]] announce_session_identifiers: Callable[[Any], None] report_progress: Callable[[Route, str], None] - terminate_restart_orphan_listeners: Callable[..., None] + terminate_restart_orphan_listeners: Callable[..., set[int]] def replace(self) -> None: - if self.reason != "startup_fingerprint_mismatch": - return route = self.session.effective_route - if route.flags.get("runtime_scope") == "dependencies": - return - selected_services = self.fresh_start_replacement_services(self.session, candidate_state=self.candidate_state) - if not selected_services: - return - self.announce_session_identifiers(self.session) - self.report_progress( - route, - f"Startup selection changed; replacing {len(selected_services)} existing service(s)...", - ) - self.runtime._emit( - "state.run_reuse.replace_existing_services", - run_id=self.candidate_state.run_id, - mode=self.session.runtime_mode, - reason=self.reason, - selected_services=sorted(selected_services), - ) - self.runtime._terminate_services_from_state( + previous_preserve_flag = bool(getattr(self.session, "preserve_existing_state_on_failure", False)) + source_services = { + **dict(self.candidate_state.services), + **dict(self.session.preserved_services), + } + source_requirements = { + **dict(self.candidate_state.requirements), + **dict(self.session.preserved_requirements), + } + source_metadata = metadata_with_state_sources( + { + **dict(self.candidate_state.metadata), + **dict(self.session.base_metadata), + }, self.candidate_state, - selected_services=selected_services, - aggressive=False, - verify_ownership=True, ) - self.terminate_restart_orphan_listeners( - state=self.candidate_state, - selected_services=selected_services, - aggressive=True, + # Bind failure finalization to the existing run before resolving any + # legacy selector or emitting diagnostics. Those adapters are allowed + # to fail, but their failure must never supersede active authority. + self.session.preserved_services = source_services + self.session.preserved_requirements = source_requirements + self.session.base_metadata = source_metadata + self.session.preserve_existing_state_on_failure = True + dependencies_only = route.flags.get("runtime_scope") == "dependencies" + selected_services = ( + set() + if dependencies_only + else fresh_start_replacement_services( + selected_contexts=list(self.session.selected_contexts), + candidate_state=self.candidate_state, + project_name_from_service=self.runtime._project_name_from_service, + route=route, + additional_services=tuple( + getattr(getattr(self.runtime, "config", None), "additional_services", ()) or () + ), + ) ) + target_projects = { + str(context.name).strip().casefold() + for context in self.session.selected_contexts + if str(context.name).strip() + } + preserved_services = ( + dict(self.candidate_state.services) + if dependencies_only + else { + name: service + for name, service in self.candidate_state.services.items() + if name not in selected_services + } + ) + replace_requirements = _fresh_start_replaces_requirements(route) + if replace_requirements: + preserved_requirements, requirements_to_release = _split_requirements( + self.candidate_state, + target_projects=target_projects, + ) + else: + preserved_requirements = dict(self.candidate_state.requirements) + requirements_to_release = {} + try: + if selected_services: + self.announce_session_identifiers(self.session) + progress_message = ( + f"Auto-resume disabled; replacing {len(selected_services)} existing service(s)..." + if self.reason == "explicit_no_resume" + else f"Startup selection changed; replacing {len(selected_services)} existing service(s)..." + ) + self.report_progress(route, progress_message) + self.runtime._emit( + "state.run_reuse.replace_existing_services", + run_id=self.candidate_state.run_id, + mode=self.session.runtime_mode, + reason=self.reason, + selected_services=sorted(selected_services), + ) + if selected_services: + termination_result = self.runtime._terminate_services_from_state( + self.candidate_state, + selected_services=selected_services, + aggressive=False, + verify_ownership=True, + ) + failed_services = unconfirmed_service_names(termination_result, selected_services) + if failed_services: + raise RuntimeError( + "Fresh start aborted because existing services could not be stopped: " + + ", ".join(sorted(failed_services)) + ) + orphan_result = self.terminate_restart_orphan_listeners( + state=self.candidate_state, + selected_services=selected_services, + aggressive=True, + ) + failed_orphan_pids = failed_listener_pids(orphan_result) + if failed_orphan_pids is None: + raise RuntimeError("Fresh start aborted because orphan listener cleanup was not confirmed") + if failed_orphan_pids: + raise RuntimeError( + "Fresh start aborted because orphan listeners could not be stopped: " + + ", ".join(str(pid) for pid in sorted(failed_orphan_pids)) + ) + for requirements in requirements_to_release.values(): + self.runtime._release_requirement_ports(requirements) + self._rebind_replacement_ports(selected_services) + except Exception: + self.session.preserved_services = source_services + self.session.preserved_requirements = source_requirements + self.session.base_metadata = source_metadata + self.session.preserve_existing_state_on_failure = True + raise + self.session.preserved_services = preserved_services + self.session.preserved_requirements = preserved_requirements + self.session.preserve_existing_state_on_failure = previous_preserve_flag + + def _rebind_replacement_ports(self, selected_services: set[str]) -> None: + port_planner = getattr(self.runtime, "port_planner", None) + release = getattr(port_planner, "release", None) + if not callable(release): + return + contexts = { + str(getattr(context, "name", "")).casefold(): context + for context in getattr(self.session, "selected_contexts", []) + } + for name in sorted(selected_services): + service = getattr(self.candidate_state, "services", {}).get(name) + if service is None or not bool(getattr(service, "listener_expected", True)): + continue + project = service_project_name(service) or self.runtime._project_name_from_service(name) + service_type = service_slug_from_record(service) + context = contexts.get(str(project).casefold()) + plans = getattr(context, "ports", {}) if context is not None else {} + plan = plans.get(service_type) if service_type else None + old_port = getattr(service, "actual_port", None) or getattr(service, "requested_port", None) + current_port = getattr(plan, "final", None) + if plan is None or not isinstance(old_port, int) or old_port <= 0: + continue + owner = f"{getattr(context, 'name', project)}:{service_type}" + if isinstance(current_port, int) and current_port > 0: + release(current_port, owner=owner) + plan.assigned = old_port + plan.final = old_port + plan.source = "fresh_start_replacement" + self.runtime._emit( + "port.rebound.after_fresh_start_cleanup", + project=project, + service=service_type, + previous_port=old_port, + port=old_port, + ) def replace_existing_project_services_for_fresh_start( @@ -95,17 +234,15 @@ def replace_existing_project_services_for_fresh_start( session: Any, candidate_state: Any, reason: str, - fresh_start_replacement_services: Callable[..., set[str]], announce_session_identifiers: Callable[[Any], None], report_progress: Callable[[Route, str], None], - terminate_restart_orphan_listeners: Callable[..., None], + terminate_restart_orphan_listeners: Callable[..., set[int]], ) -> None: FreshStartServiceReplacer( runtime=runtime, session=session, candidate_state=candidate_state, reason=reason, - fresh_start_replacement_services=fresh_start_replacement_services, announce_session_identifiers=announce_session_identifiers, report_progress=report_progress, terminate_restart_orphan_listeners=terminate_restart_orphan_listeners, @@ -118,26 +255,64 @@ def replace_existing_project_services_for_fresh_start_with_defaults( session: Any, candidate_state: Any, reason: str, - configured_service_types: set[str], - additional_services: tuple[object, ...], announce_session_identifiers: Callable[[Any], None], report_progress: Callable[[Route, str], None], - terminate_restart_orphan_listeners: Callable[..., None], + terminate_restart_orphan_listeners: Callable[..., set[int]], ) -> None: replace_existing_project_services_for_fresh_start( runtime=runtime, session=session, candidate_state=candidate_state, reason=reason, - fresh_start_replacement_services=lambda _, *, candidate_state: fresh_start_replacement_services( - route=session.effective_route, - selected_contexts=list(session.selected_contexts), - candidate_state=candidate_state, - configured_service_types=configured_service_types, - additional_services=additional_services, - project_name_from_service=runtime._project_name_from_service, - ), announce_session_identifiers=announce_session_identifiers, report_progress=report_progress, terminate_restart_orphan_listeners=terminate_restart_orphan_listeners, ) + + +def _split_requirements( + state: object, + *, + target_projects: set[str], +) -> tuple[dict[str, object], dict[str, object]]: + preserved: dict[str, object] = {} + replaced: dict[str, object] = {} + for name, requirements in getattr(state, "requirements", {}).items(): + project = str(getattr(requirements, "project", "") or name).strip().casefold() + destination = replaced if project in target_projects else preserved + destination[name] = requirements + return preserved, replaced + + +def _route_has_app_service_scope(route: Route) -> bool: + runtime_scope = str(route.flags.get("runtime_scope") or "").strip().lower() + if runtime_scope in {"backend", "frontend", "fullstack", "dependencies"}: + return True + if isinstance(route.flags.get("services"), list) and bool(route.flags.get("services")): + return True + if isinstance(route.flags.get("restart_service_types"), list): + return True + return route.flags.get("launch_backend") is False or route.flags.get("launch_frontend") is False + + +def _fresh_start_replaces_requirements(route: Route) -> bool: + runtime_scope = str(route.flags.get("runtime_scope") or "").strip().lower() + if runtime_scope in {"dependencies", "entire-system"}: + return True + if _route_has_app_service_scope(route): + return False + if route.flags.get("launch_dependencies") is False: + return False + return True + + +def _service_type(service_name: str, service: object) -> str: + explicit = service_slug_from_record(service) + if explicit: + return explicit + lowered = str(service_name).strip().casefold() + for suffix in ("backend", "frontend"): + if lowered.endswith(f" {suffix}"): + return suffix + parts = lowered.rsplit(" ", 1) + return parts[-1] if len(parts) == 2 else "" diff --git a/python/envctl_engine/startup/run_reuse_identity.py b/python/envctl_engine/startup/run_reuse_identity.py index bd8e66f43..8aa294248 100644 --- a/python/envctl_engine/startup/run_reuse_identity.py +++ b/python/envctl_engine/startup/run_reuse_identity.py @@ -9,6 +9,7 @@ from envctl_engine.requirements.core import dependency_ids from envctl_engine.requirements.external import dependency_external_mode from envctl_engine.runtime.command_router import Route +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.state.models import RunState @@ -27,19 +28,31 @@ def build_startup_identity_metadata( route: Route | None = None, ) -> dict[str, object]: metadata = dict(base_metadata or {}) - project_roots = { - str(getattr(context, "name", "")).strip(): root - for context, root in ( - (context, normalize_project_root(getattr(context, "root", None))) for context in project_contexts - ) - if str(getattr(context, "name", "")).strip() and root - } + roots_by_key: dict[str, tuple[str, str]] = {} + existing_roots = metadata.get("project_roots") + if isinstance(existing_roots, Mapping): + for raw_name, raw_root in existing_roots.items(): + name = str(raw_name).strip() + root = normalize_project_root(raw_root) + if name and root: + roots_by_key[name.casefold()] = (name, root) + for context in project_contexts: + name = str(getattr(context, "name", "")).strip() + root = normalize_project_root(getattr(context, "root", None)) + if name and root: + roots_by_key[name.casefold()] = (name, root) + project_roots = {name: root for name, root in (roots_by_key[key] for key in sorted(roots_by_key))} identity_payload = _startup_identity_payload( runtime, runtime_mode=runtime_mode, project_contexts=project_contexts, route=route, ) + identity_payload["projects"] = [{"name": name, "root": root} for name, root in project_roots.items()] + identity_payload.pop("fingerprint", None) + identity_payload["fingerprint"] = hashlib.sha256( + json.dumps(identity_payload, sort_keys=True).encode("utf-8") + ).hexdigest() metadata["project_roots"] = project_roots metadata["startup_identity"] = identity_payload return metadata @@ -65,7 +78,9 @@ def _startup_identity_mismatch(previous: Mapping[str, object], current: Mapping[ def _startup_identity_comparison_payload(identity: Mapping[str, object]) -> dict[str, object]: - return {str(key): value for key, value in identity.items() if str(key) != "fingerprint"} + payload = {str(key): value for key, value in identity.items() if str(key) != "fingerprint"} + payload.setdefault("application_runtime", "process") + return payload def project_identities_from_contexts(contexts: list[object]) -> list[ProjectIdentity]: @@ -87,13 +102,17 @@ def project_identities_from_state(runtime: Any, state: RunState) -> list[Project if not normalized_name: continue names[normalized_name] = normalize_project_root(root_value) - for project_name in state.requirements: - normalized_name = str(project_name).strip() + for storage_name, requirements in state.requirements.items(): + normalized_name = str(getattr(requirements, "project", "") or storage_name).strip() if not normalized_name: continue names.setdefault(normalized_name, normalize_project_root(roots.get(normalized_name))) - for service_name in state.services: - project_name = runtime._project_name_from_service(service_name) + for service_name, service in state.services.items(): + project_name = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) normalized_name = str(project_name).strip() if not normalized_name: continue @@ -166,6 +185,12 @@ def _startup_identity_payload( } payload = { "mode": runtime_mode, + "application_runtime": ( + "docker" + if bool(route is not None and route.flags.get("docker")) + or str(getattr(runtime, "env", {}).get("DOCKER_MODE", "")).strip().lower() == "true" + else "process" + ), "projects": identities_to_payload(project_identities_from_contexts(project_contexts)), "startup_enabled": startup_enabled, "services": services, diff --git a/python/envctl_engine/startup/run_reuse_resolution.py b/python/envctl_engine/startup/run_reuse_resolution.py index cb9beb3c2..69b52a315 100644 --- a/python/envctl_engine/startup/run_reuse_resolution.py +++ b/python/envctl_engine/startup/run_reuse_resolution.py @@ -21,7 +21,7 @@ ) from envctl_engine.startup.run_reuse_fresh_start import replace_existing_project_services_for_fresh_start_with_defaults from envctl_engine.startup.run_reuse_resume import StartupRunReuseResumeHandler -from envctl_engine.startup.session import StartupSession +from envctl_engine.startup.session import StartupSession, metadata_with_state_sources from envctl_engine.startup.session_lifecycle import announce_session_identifiers, emit_startup_phase from envctl_engine.startup.service_bootstrap_domain import configured_service_types_for_mode from envctl_engine.startup.startup_progress import report_progress @@ -108,17 +108,19 @@ def _resolve_auto_reuse(self) -> int | None: def _emit_reuse_evaluation(self, decision: RunReuseDecision) -> None: candidate_state = decision.candidate_state - self.runtime._emit( - "state.run_reuse.evaluate", - run_id=candidate_state.run_id if candidate_state is not None else None, - mode=self.runtime_mode, - command=self.route.command, - decision_kind=decision.decision_kind, - reason=decision.reason, - selected_projects=decision.selected_projects, - state_projects=decision.state_projects, - weak_identity=decision.weak_identity, - mismatch_details=decision.mismatch_details, + self._best_effort_diagnostic( + lambda: self.runtime._emit( + "state.run_reuse.evaluate", + run_id=candidate_state.run_id if candidate_state is not None else None, + mode=self.runtime_mode, + command=self.route.command, + decision_kind=decision.decision_kind, + reason=decision.reason, + selected_projects=decision.selected_projects, + state_projects=decision.state_projects, + weak_identity=decision.weak_identity, + mismatch_details=decision.mismatch_details, + ) ) def _prepare_dashboard_stopped_restore(self, *, decision: RunReuseDecision, reuse_started: float) -> bool: @@ -137,7 +139,10 @@ def _apply_reuse_expand(self, *, decision: RunReuseDecision, candidate_state: An self.session.run_id = None self.session.preserved_services = dict(candidate_state.services) self.session.preserved_requirements = dict(candidate_state.requirements) - self.session.base_metadata = mark_run_reused(candidate_state.metadata, reason="reuse_expand") + self.session.base_metadata = metadata_with_state_sources( + mark_run_reused(candidate_state.metadata, reason="reuse_expand"), + candidate_state, + ) state_project_names = { str(project.get("name", "")).strip().casefold() for project in decision.state_projects @@ -149,53 +154,63 @@ def _apply_reuse_expand(self, *, decision: RunReuseDecision, candidate_state: An for context in self.session.selected_contexts if str(getattr(context, "name", "")).strip().casefold() not in state_project_names ] - self.emit_phase( - self.session, - "auto_resume_evaluate", - reuse_started, - status="expand", - match_mode="expand", - state_project_count=len(decision.state_projects), - selected_project_count=len(self.session.selected_contexts), + self._best_effort_diagnostic( + lambda: self.emit_phase( + self.session, + "auto_resume_evaluate", + reuse_started, + status="expand", + match_mode="expand", + state_project_count=len(decision.state_projects), + selected_project_count=len(self.session.selected_contexts), + ) ) - self.runtime._emit( - "state.run_reuse.applied", - run_id=candidate_state.run_id, - mode=self.runtime_mode, - command=self.route.command, - decision_kind=decision.decision_kind, - reason=decision.reason, - preserved_services=sorted(self.session.preserved_services), + self._best_effort_diagnostic( + lambda: self.runtime._emit( + "state.run_reuse.applied", + run_id=candidate_state.run_id, + mode=self.runtime_mode, + command=self.route.command, + decision_kind=decision.decision_kind, + reason=decision.reason, + preserved_services=sorted(self.session.preserved_services), + ) ) def _handle_no_reuse(self, *, decision: RunReuseDecision, reuse_started: float) -> None: candidate_state = decision.candidate_state - self.emit_phase( - self.session, - "auto_resume_evaluate", - reuse_started, - status="skipped" if candidate_state is not None else "none", - reason=decision.reason if candidate_state is not None else None, - state_project_count=len(decision.state_projects), - selected_project_count=len(self.session.selected_contexts), + self._best_effort_diagnostic( + lambda: self.emit_phase( + self.session, + "auto_resume_evaluate", + reuse_started, + status="skipped" if candidate_state is not None else "none", + reason=decision.reason if candidate_state is not None else None, + state_project_count=len(decision.state_projects), + selected_project_count=len(self.session.selected_contexts), + ) ) if candidate_state is None: return - self.runtime._emit( - "state.auto_resume.skipped", - reason=decision.reason, - state_projects=[project["name"] for project in decision.state_projects], - selected_projects=[project["name"] for project in decision.selected_projects], - mode=self.runtime_mode, - command=self.route.command, + self._best_effort_diagnostic( + lambda: self.runtime._emit( + "state.auto_resume.skipped", + reason=decision.reason, + state_projects=[project["name"] for project in decision.state_projects], + selected_projects=[project["name"] for project in decision.selected_projects], + mode=self.runtime_mode, + command=self.route.command, + ) ) - self.runtime._emit( - "state.run_reuse.skipped", - run_id=candidate_state.run_id, - reason=decision.reason, - mode=self.runtime_mode, - command=self.route.command, - mismatch_details=decision.mismatch_details, + self._best_effort_diagnostic( + lambda: self.runtime._emit( + "state.run_reuse.skipped", + run_id=candidate_state.run_id, + reason=decision.reason, + mode=self.runtime_mode, + command=self.route.command, + mismatch_details=decision.mismatch_details, + ) ) self.replace_existing_project_services_for_fresh_start( self.session, @@ -221,14 +236,23 @@ def _run_planning_prs_if_requested(self) -> int | None: return code def _emit_startup_branch_snapshot(self, debug_orch_groups: set[str]) -> None: - self.emit_snapshot( - self.session, - "startup_branch_enter", - command=self.requested_command, - mode=self.runtime_mode, - orch_group=sorted(debug_orch_groups) or None, + self._best_effort_diagnostic( + lambda: self.emit_snapshot( + self.session, + "startup_branch_enter", + command=self.requested_command, + mode=self.runtime_mode, + orch_group=sorted(debug_orch_groups) or None, + ) ) + @staticmethod + def _best_effort_diagnostic(callback: Callable[[], object]) -> None: + try: + callback() + except Exception: # noqa: BLE001 + pass + def _resume_matching_run( self, *, @@ -321,7 +345,7 @@ def resolve_startup_run_reuse_with_runtime( runtime: Any, session: StartupSession, *, - terminate_restart_orphan_listeners: Callable[..., None], + terminate_restart_orphan_listeners: Callable[..., set[int]], validate_attach_target_fn: Callable[..., Any], attach_plan_agent_terminal: Callable[..., Any], progress_lock: Any, @@ -362,13 +386,11 @@ def report_progress_fn(route: Route, message: str) -> None: session=session, validate_plan_agent_handoff=validate_plan_agent_handoff, attach_plan_agent_terminal=attach_plan_agent_terminal, - print_headless_plan_session_summary=lambda session, *, attach_target: ( - print_headless_plan_session_summary( - session, - validate_plan_agent_handoff=validate_plan_agent_handoff, - print_fn=print_fn, - attach_target=attach_target, - ) + print_headless_plan_session_summary=lambda session, *, attach_target: print_headless_plan_session_summary( + session, + validate_plan_agent_handoff=validate_plan_agent_handoff, + print_fn=print_fn, + attach_target=attach_target, ), ), print_headless_plan_session_summary=lambda session: print_headless_plan_session_summary( @@ -391,8 +413,6 @@ def report_progress_fn(route: Route, message: str) -> None: session=session, candidate_state=candidate_state, reason=reason, - configured_service_types=set(configured_service_types_for_mode(runtime.config, session.runtime_mode)), - additional_services=tuple(getattr(runtime.config, "additional_services", ()) or ()), announce_session_identifiers=lambda session: announce_session_identifiers( runtime, session, diff --git a/python/envctl_engine/startup/run_reuse_resume.py b/python/envctl_engine/startup/run_reuse_resume.py index bed5d1840..237bc755f 100644 --- a/python/envctl_engine/startup/run_reuse_resume.py +++ b/python/envctl_engine/startup/run_reuse_resume.py @@ -5,12 +5,17 @@ from typing import Any, cast from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.docker_service_runtime import state_uses_docker_services from envctl_engine.runtime.engine_runtime_startup_support import mark_run_reused +from envctl_engine.runtime.lifecycle_operation_lease import ( + lifecycle_operation_active, + release_lifecycle_operation, +) from envctl_engine.runtime.runtime_context import resolve_state_repository from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.startup.finalization import build_planning_dashboard_state from envctl_engine.startup.run_reuse_decision import RunReuseDecision -from envctl_engine.startup.session import StartupSession +from envctl_engine.startup.session import StartupSession, metadata_with_state_sources @dataclass(frozen=True, slots=True) @@ -35,26 +40,52 @@ class StartupRunReuseResumeHandler: def resume_matching_run(self) -> int | None: previous_run_id = self.session.run_id previous_identifiers_announced = self.session.identifiers_announced - self.session.run_id = self.candidate_state.run_id - self.announce_session_identifiers(self.session) - self._emit_resume_match_phase() - self._emit_resume_match_events() - - resume_code = self.runtime._resume(self._resume_route()) - if int(resume_code) == 0: - return self._resume_success_result() - self._handle_resume_failure( - resume_code=int(resume_code), - previous_run_id=previous_run_id, - previous_identifiers_announced=previous_identifiers_announced, - ) - return None + previous_authority = self._session_authority() + try: + self.session.run_id = self.candidate_state.run_id + self.announce_session_identifiers(self.session) + self._emit_resume_match_phase() + self._emit_resume_match_events() + lifecycle_lease_was_active = lifecycle_operation_active(self.runtime) + resume_code = self.runtime._resume(self._resume_route()) + if int(resume_code) == 0: + result = self._resume_success_result() + self._restore_session_authority(previous_authority) + return result + lifecycle_lease_was_released = lifecycle_lease_was_active and not lifecycle_operation_active(self.runtime) + self._handle_resume_failure( + resume_code=int(resume_code), + previous_run_id=previous_run_id, + previous_identifiers_announced=previous_identifiers_announced, + allow_fresh_fallback=not lifecycle_lease_was_released, + ) + return int(resume_code) if lifecycle_lease_was_released else None + except Exception: + self.session.run_id = previous_run_id + self.session.identifiers_announced = previous_identifiers_announced + self._bind_candidate_failure_authority(previous_authority) + raise def resume_dashboard_run(self) -> int | None: + previous_run_id = self.session.run_id + previous_identifiers_announced = self.session.identifiers_announced + previous_authority = self._session_authority() + try: + result = self._resume_dashboard_run_guarded() + except Exception: + self.session.run_id = previous_run_id + self.session.identifiers_announced = previous_identifiers_announced + self._bind_candidate_failure_authority(previous_authority) + raise + self._restore_session_authority(previous_authority) + return result + + def _resume_dashboard_run_guarded(self) -> int | None: self.session.run_id = self.candidate_state.run_id self.announce_session_identifiers(self.session) self._persist_dashboard_resume_metadata() self._emit_dashboard_resume_events() + release_lifecycle_operation(self.runtime) if self.headless_plan_output_only(self.session): self.print_headless_plan_session_summary(self.session) @@ -73,6 +104,48 @@ def resume_dashboard_run(self) -> int | None: return self.runtime._run_interactive_dashboard_loop(self.candidate_state) return 0 + def _session_authority( + self, + ) -> tuple[dict[str, object], dict[str, object], dict[str, object], bool]: + return ( + dict(self.session.preserved_services), + dict(self.session.preserved_requirements), + dict(self.session.base_metadata), + bool(self.session.preserve_existing_state_on_failure), + ) + + def _restore_session_authority( + self, + authority: tuple[dict[str, object], dict[str, object], dict[str, object], bool], + ) -> None: + services, requirements, metadata, preserve = authority + self.session.preserved_services = dict(services) # type: ignore[assignment] + self.session.preserved_requirements = dict(requirements) # type: ignore[assignment] + self.session.base_metadata = dict(metadata) + self.session.preserve_existing_state_on_failure = preserve + + def _bind_candidate_failure_authority( + self, + prior: tuple[dict[str, object], dict[str, object], dict[str, object], bool], + ) -> None: + prior_services, prior_requirements, prior_metadata, _prior_preserve = prior + self.session.preserved_services = { # type: ignore[assignment] + **dict(self.candidate_state.services), + **prior_services, + } + self.session.preserved_requirements = { # type: ignore[assignment] + **dict(self.candidate_state.requirements), + **prior_requirements, + } + self.session.base_metadata = metadata_with_state_sources( + { + **dict(self.candidate_state.metadata), + **prior_metadata, + }, + self.candidate_state, + ) + self.session.preserve_existing_state_on_failure = True + @property def _match_mode(self) -> str: return "exact" if self.decision.decision_kind == "resume_exact" else "subset" @@ -119,9 +192,13 @@ def _resume_route(self) -> Route: **self.route.flags, "_resume_source_command": self.route.command, "_run_reuse_reason": self.decision.decision_kind, + "_state_project_names": [str(name) for name in self._selected_project_names], } if self._attach_plan_agent_after_resume(): resume_flags["batch"] = True + if state_uses_docker_services(self.candidate_state): + resume_flags["docker"] = True + self.runtime.env["DOCKER_MODE"] = "true" return Route( command="resume", mode=self.runtime_mode, @@ -150,6 +227,7 @@ def _handle_resume_failure( resume_code: int, previous_run_id: str | None, previous_identifiers_announced: bool, + allow_fresh_fallback: bool = True, ) -> None: self.session.run_id = previous_run_id self.session.identifiers_announced = previous_identifiers_announced @@ -170,11 +248,12 @@ def _handle_resume_failure( mode=self.runtime_mode, command=self.route.command, ) - self.replace_existing_project_services_for_fresh_start( - self.session, - candidate_state=self.candidate_state, - reason=self.decision.reason, - ) + if allow_fresh_fallback: + self.replace_existing_project_services_for_fresh_start( + self.session, + candidate_state=self.candidate_state, + reason=self.decision.reason, + ) def _persist_dashboard_resume_metadata(self) -> None: self.candidate_state.metadata = build_planning_dashboard_state( diff --git a/python/envctl_engine/startup/selected_context_startup.py b/python/envctl_engine/startup/selected_context_startup.py index 7be9b3132..f30a0c411 100644 --- a/python/envctl_engine/startup/selected_context_startup.py +++ b/python/envctl_engine/startup/selected_context_startup.py @@ -2,12 +2,18 @@ import concurrent.futures from contextlib import nullcontext +from dataclasses import replace from typing import Any, Callable, cast from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_env import effective_dependency_scope from envctl_engine.startup.protocols import ProjectContextLike, StartupRuntime -from envctl_engine.startup.session import ProjectStartupResult, StartupSession +from envctl_engine.startup.session import ( + ProjectStartupResult, + StartupSession, + track_startup_failure, +) +from envctl_engine.state.models import RequirementsResult, ServiceRecord from envctl_engine.startup.startup_progress import suppress_progress_output as default_suppress_progress_output @@ -23,14 +29,205 @@ def record_project_startup( context: ProjectContextLike, result: ProjectStartupResult, ) -> None: + existing_service_names = set(session.preserved_services) + existing_service_names.update(session.unterminated_services) + for project_services in session.services_by_project.values(): + existing_service_names.update(project_services) + collisions = sorted(existing_service_names.intersection(result.services)) + project_collision = ( + context.name in session.services_by_project + or context.name in session.requirements_by_project + ) + if collisions or project_collision: + # The replacement processes are already live at this point. Give every + # one a unique durable identity before raising so parallel projects and + # preserved state can never overwrite one another during cleanup. + preserved_collisions = set(session.preserved_services).intersection(collisions) + session.preserve_existing_state_on_failure = bool(preserved_collisions) + session.service_state_collisions.update(collisions) + occupied_names = existing_service_names.union(result.services) + for name, service in result.services.items(): + stored_name = name + stored_service = ( + service + if not isinstance(service, ServiceRecord) or service.project + else replace(service, project=context.name) + ) + if name in existing_service_names: + stored_name = _unique_collision_service_name(name, service, occupied_names) + occupied_names.add(stored_name) + stored_service = replace(stored_service, name=stored_name) + session.service_state_collision_rows.append( + { + "original_name": name, + "replacement_name": stored_name, + "replacement_pid": getattr(service, "pid", None), + "replacement_project": getattr(service, "project", None) or context.name, + } + ) + session.unterminated_services[stored_name] = stored_service + _record_collision_requirements(session, context.name, result.requirements) + collision_labels = collisions or [f"project:{context.name}"] + raise RuntimeError( + "Refusing to overwrite preserved service state or tracked startup state " + "with newly started services: " + + ", ".join(collision_labels) + ) session.requirements_by_project[context.name] = result.requirements - session.services_by_project[context.name] = result.services + session.services_by_project[context.name] = { + name: ( + service + if not isinstance(service, ServiceRecord) or service.project + else replace(service, project=context.name) + ) + for name, service in result.services.items() + } session.started_context_names.append(context.name) for warning in result.warnings: if "No local app system is configured" in warning: session.warnings.append(warning) +def _unique_collision_service_name(name: str, service: object, occupied: set[str]) -> str: + pid = getattr(service, "pid", None) + suffix = f"Restart Collision {pid}" if isinstance(pid, int) and pid > 0 else "Restart Collision" + base = f"{name} {suffix}" + candidate = base + index = 2 + while candidate in occupied: + candidate = f"{base} {index}" + index += 1 + return candidate + + +def _record_collision_requirements( + session: StartupSession, + project: str, + requirements: RequirementsResult, +) -> None: + # A preserved requirement with the same storage key is remapped by the + # failure finalizer, which can also record whether it remained managed. + if project in session.preserved_requirements: + session.requirements_by_project[project] = requirements + return + if project not in session.requirements_by_project: + session.requirements_by_project[project] = requirements + return + occupied = set(session.preserved_requirements).union(session.requirements_by_project) + base = f"{project} Restart Collision" + storage_key = base + index = 2 + while storage_key in occupied: + storage_key = f"{base} {index}" + index += 1 + session.requirements_by_project[storage_key] = requirements + session.requirement_state_collision_rows.append( + { + "original_project": project, + "replacement_project": storage_key, + "replacement_requirements_retained": True, + } + ) + + +class _UnrecordedProjectStartup(RuntimeError): + """Adapter used to feed a returned startup result into failure tracking.""" + + def __init__( + self, + *, + project: str, + requirements: RequirementsResult | None, + services: dict[str, ServiceRecord], + ) -> None: + super().__init__(f"startup result for {project} was not committed to the session") + self.project = project + self.requirements = requirements + self.unterminated_services = services + + +def _retain_unrecorded_project_startup( + session: StartupSession, + context: ProjectContextLike, + result: ProjectStartupResult, +) -> None: + """Retain every resource returned before a callback/cancellation failure. + + ``record_project_startup`` is normally the ownership handoff. A signal can + arrive after project startup returns but before (or during) that handoff. + Feed only resources that are not already represented in the session through + the same collision-aware failure tracker used by lower startup layers. + """ + + tracked_requirements = [ + *session.preserved_requirements.values(), + *session.requirements_by_project.values(), + ] + requirements = ( + None + if any(existing is result.requirements for existing in tracked_requirements) + else result.requirements + ) + tracked_services = [ + *session.preserved_services.values(), + *session.unterminated_services.values(), + *( + service + for project_services in session.services_by_project.values() + for service in project_services.values() + ), + ] + untracked_services = { + name: service + for name, service in result.services.items() + if isinstance(service, ServiceRecord) + and not any(_same_runtime_service(service, existing) for existing in tracked_services) + } + if requirements is None and not untracked_services: + return + track_startup_failure( + session, + _UnrecordedProjectStartup( + project=str(context.name), + requirements=requirements, + services=untracked_services, + ), + ) + + +def _same_runtime_service(left: ServiceRecord, right: ServiceRecord) -> bool: + if left is right: + return True + for attribute in ("container_launch_token", "container_id"): + left_value = str(getattr(left, attribute, "") or "").strip() + right_value = str(getattr(right, attribute, "") or "").strip() + if left_value and left_value == right_value: + return True + left_pid = getattr(left, "pid", None) + right_pid = getattr(right, "pid", None) + return ( + isinstance(left_pid, int) + and not isinstance(left_pid, bool) + and left_pid > 0 + and left_pid == right_pid + ) + + +def _retain_unrecorded_project_startup_safely( + session: StartupSession, + context: ProjectContextLike, + result: ProjectStartupResult, + triggering_error: BaseException, +) -> None: + try: + _retain_unrecorded_project_startup(session, context, result) + except BaseException as retention_error: # noqa: BLE001 - never replace the triggering cancellation + try: + triggering_error.add_note(f"startup result retention also failed: {retention_error}") + except Exception: # noqa: BLE001 + pass + + def start_selected_contexts( *, runtime: StartupRuntime, @@ -138,8 +335,11 @@ def start_selected_contexts( project_spinner_group=project_spinner_group, use_project_spinner_group=use_project_spinner_group, ) - except RuntimeError: - _fail_single_spinner(runtime=runtime, active_spinner=active_spinner, enabled=use_single_spinner) + except BaseException: + try: + _fail_single_spinner(runtime=runtime, active_spinner=active_spinner, enabled=use_single_spinner) + except BaseException: # noqa: BLE001 - presentation must not mask cancellation + pass raise _succeed_single_spinner( runtime=runtime, @@ -228,51 +428,140 @@ def _start_contexts_parallel( ) -> None: completed: dict[str, ProjectStartupResult] = {} failures: list[str] = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=parallel_workers) as executor: - future_map = { - executor.submit( - runtime._start_project_context, - context=context, - mode=session.runtime_mode, - route=route_for_execution, - run_id=resolved_run_id(session), - ): context - for context in session.contexts_to_start - } - for future in concurrent.futures.as_completed(future_map): - context = future_map[future] - try: - result = cast(ProjectStartupResult, future.result()) - completed[context.name] = result - _record_parallel_success( - runtime=runtime, - session=session, + ui_failures: list[Exception] = [] + future_map: dict[concurrent.futures.Future[Any], ProjectContextLike] = {} + try: + # Let the executor context finish its wait before handling a main-thread + # cancellation. Once control reaches the outer handler, every submitted + # worker has a terminal result that can be transferred to the session. + with concurrent.futures.ThreadPoolExecutor(max_workers=parallel_workers) as executor: + for context in session.contexts_to_start: + future = executor.submit( + runtime._start_project_context, context=context, - result=result, - route_for_execution=route_for_execution, - render_project_startup_warnings=render_project_startup_warnings, - active_spinner=active_spinner, - use_single_spinner=use_single_spinner, - project_spinner_group=project_spinner_group, - use_project_spinner_group=use_project_spinner_group, - completed_count=len(completed), + mode=session.runtime_mode, + route=route_for_execution, + run_id=resolved_run_id(session), ) - except RuntimeError as exc: - if should_degrade_to_plan_agent_handoff(session, str(exc)): - record_plan_agent_handoff_local_startup_failure(session, project_name=context.name, error=str(exc)) - if use_project_spinner_group: - project_spinner_group.mark_success(context.name, "AI session running; local startup failed") + future_map[future] = context + for future in concurrent.futures.as_completed(future_map): + context = future_map[future] + try: + result = cast(ProjectStartupResult, future.result()) + except RuntimeError as exc: + track_startup_failure(session, exc) + try: + degrade = should_degrade_to_plan_agent_handoff(session, str(exc)) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + degrade = False + if degrade: + try: + record_plan_agent_handoff_local_startup_failure( + session, + project_name=context.name, + error=str(exc), + ) + if use_project_spinner_group: + project_spinner_group.mark_success( + context.name, + "AI session running; local startup failed", + ) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + continue + failures.append(str(exc)) + try: + runtime._emit("startup.project.failed", project=context.name, error=str(exc)) + if use_project_spinner_group: + project_spinner_group.mark_failure(context.name, str(exc)) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) continue - failures.append(str(exc)) - runtime._emit("startup.project.failed", project=context.name, error=str(exc)) - if use_project_spinner_group: - project_spinner_group.mark_failure(context.name, str(exc)) - for context in session.contexts_to_start: - result = completed.get(context.name) - if result is not None: - record_project_startup(session, context, result) + except Exception as exc: # noqa: BLE001 + track_startup_failure(session, exc) + failures.append(str(exc)) + try: + runtime._emit("startup.project.failed", project=context.name, error=str(exc)) + if use_project_spinner_group: + project_spinner_group.mark_failure(context.name, str(exc)) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + continue + + try: + record_project_startup(session, context, result) + except Exception as exc: # noqa: BLE001 + _retain_unrecorded_project_startup_safely(session, context, result, exc) + track_startup_failure(session, exc) + failures.append(str(exc)) + try: + runtime._emit("startup.project.failed", project=context.name, error=str(exc)) + if use_project_spinner_group: + project_spinner_group.mark_failure(context.name, str(exc)) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + # Keep draining the executor. Other futures may already have + # launched resources that must be recorded for final cleanup. + continue + completed[context.name] = result + try: + _record_parallel_success( + runtime=runtime, + session=session, + context=context, + result=result, + route_for_execution=route_for_execution, + render_project_startup_warnings=render_project_startup_warnings, + active_spinner=active_spinner, + use_single_spinner=use_single_spinner, + project_spinner_group=project_spinner_group, + use_project_spinner_group=use_project_spinner_group, + completed_count=len(completed), + ) + except Exception as exc: # noqa: BLE001 + ui_failures.append(exc) + except BaseException as exc: # noqa: BLE001 - cancellation requires a complete ownership drain + _drain_parallel_startup_futures( + session=session, + future_map=future_map, + triggering_error=exc, + ) + raise if failures: raise RuntimeError("; ".join(failures)) + if ui_failures: + raise ui_failures[0] + + +def _drain_parallel_startup_futures( + *, + session: StartupSession, + future_map: dict[concurrent.futures.Future[Any], ProjectContextLike], + triggering_error: BaseException, +) -> None: + """Transfer every completed worker result after a main-thread interruption.""" + + for future, context in future_map.items(): + try: + result = cast(ProjectStartupResult, future.result()) + except BaseException as worker_error: # noqa: BLE001 - retain any failure-owned resources + try: + track_startup_failure(session, worker_error) + except BaseException as retention_error: # noqa: BLE001 + try: + triggering_error.add_note( + f"startup failure retention also failed for {context.name}: {retention_error}" + ) + except Exception: # noqa: BLE001 + pass + continue + _retain_unrecorded_project_startup_safely( + session, + context, + result, + triggering_error, + ) def _record_parallel_success( @@ -335,13 +624,22 @@ def _start_contexts_sequential( ), ) except RuntimeError as exc: + track_startup_failure(session, exc) if should_degrade_to_plan_agent_handoff(session, str(exc)): record_plan_agent_handoff_local_startup_failure(session, project_name=context.name, error=str(exc)) if use_project_spinner_group: project_spinner_group.mark_success(context.name, "AI session running; local startup failed") continue raise - record_project_startup(session, context, result) + except BaseException as exc: # noqa: BLE001 - persist failure-owned resources before propagation + track_startup_failure(session, exc) + raise + try: + record_project_startup(session, context, result) + except BaseException as exc: # noqa: BLE001 - result may already own live resources + _retain_unrecorded_project_startup_safely(session, context, result, exc) + track_startup_failure(session, exc) + raise render_project_startup_warnings( context=context, warnings=result.warnings, diff --git a/python/envctl_engine/startup/service_attach_execution.py b/python/envctl_engine/startup/service_attach_execution.py index 1421c745b..bd3d3606e 100644 --- a/python/envctl_engine/startup/service_attach_execution.py +++ b/python/envctl_engine/startup/service_attach_execution.py @@ -1,15 +1,26 @@ from __future__ import annotations -from collections.abc import Callable, Mapping -from dataclasses import dataclass -import os +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field from pathlib import Path import time from typing import Any, cast -from envctl_engine.runtime.service_manager import ServiceStartDescriptor +from envctl_engine.runtime.docker_service_runtime import ( + DockerServiceLaunch, + DockerServiceLaunchCleanupError, + DockerServiceRuntime, + docker_service_container_command_source, + docker_service_mode_enabled, +) +from envctl_engine.runtime.service_manager import ServiceCleanupError, ServiceStartDescriptor +from envctl_engine.runtime.service_truth_diagnostics import service_listener_failure_class +from envctl_engine.shared.reason_codes import ServiceFailureReason +from envctl_engine.shared.process_runner import ProcessHandoffCleanupError +from envctl_engine.shared.services import service_display_name from envctl_engine.startup.service_execution_policy import ordered_service_layers from envctl_engine.startup.service_execution_records import PreparedServiceLaunch +from envctl_engine.startup.session import unconfirmed_service_names from envctl_engine.state.models import PortPlan, ServiceRecord @@ -34,6 +45,11 @@ class ServiceAttachRunner: additional_services: tuple[Any, ...] backend_listener_expected: bool rebound_delta: int + _active_service_ports: dict[str, int] = field(default_factory=dict, init=False, repr=False) + _rebound_launch_ports: dict[str, int] = field(default_factory=dict, init=False, repr=False) + docker_mode: bool = False + refresh_cache: bool = False + _container_launches: dict[str, DockerServiceLaunch] = field(default_factory=dict, init=False, repr=False) def start( self, @@ -42,24 +58,113 @@ def start( on_service_retry: Callable[[str, int, int, int, str | None], None], ) -> dict[str, ServiceRecord]: descriptors = self._service_descriptors() - generic_attach = getattr(self.runtime.services, "start_services_with_attach", None) - if callable(generic_attach): - return self._start_with_generic_attach( - descriptors=descriptors, - attach_parallel=attach_parallel, - on_service_retry=on_service_retry, - generic_attach=cast(Callable[..., dict[str, ServiceRecord]], generic_attach), - ) - if self.selected_service_types <= {"backend", "frontend"}: - return self._start_core_services_with_legacy_attach( - attach_parallel=attach_parallel, - on_service_retry=on_service_retry, + self._active_service_ports = { + descriptor.service_type: descriptor.requested_port + for descriptor in descriptors + if descriptor.requested_port > 0 + } + + def release_failed_port_then_report( + service_type: str, + failed_port: int, + retry_port: int, + attempt: int, + error: str | None, + ) -> None: + self._release_failed_service_port(service_type, failed_port) + self._release_rebound_launch_port(service_type) + if retry_port > 0: + self._active_service_ports[service_type] = retry_port + on_service_retry(service_type, failed_port, retry_port, attempt, error) + + try: + if hasattr(self.runtime.services, "process_runner"): + self.runtime.services.process_runner = self.process_runtime + generic_attach = getattr(self.runtime.services, "start_services_with_attach", None) + if callable(generic_attach): + records = self._start_with_generic_attach( + descriptors=descriptors, + attach_parallel=attach_parallel, + on_service_retry=release_failed_port_then_report, + generic_attach=cast(Callable[..., dict[str, ServiceRecord]], generic_attach), + ) + elif self.selected_service_types <= {"backend", "frontend"}: + records = self._start_core_services_with_legacy_attach( + attach_parallel=attach_parallel, + on_service_retry=release_failed_port_then_report, + ) + else: + raise RuntimeError("Service manager does not support additional app services") + self._annotate_container_records(records) + self._record_port_lock_session(records.values()) + return records + except BaseException as exc: + container_cleanup_error: BaseException | None = None + try: + self._cleanup_container_launches() + except BaseException as cleanup_exc: # noqa: BLE001 - cancellation must not skip cleanup + container_cleanup_error = cleanup_exc + unterminated = dict(getattr(exc, "unterminated_services", {}) or {}) + if isinstance(container_cleanup_error, ServiceCleanupError): + unterminated.update(container_cleanup_error.unterminated_services) + tracked_error: BaseException = ( + ServiceCleanupError(str(exc), unterminated) if unterminated else exc ) - raise RuntimeError("Service manager does not support additional app services") + self._record_port_lock_session(unterminated.values()) + self._release_failed_start_reservations(tracked_error) + if container_cleanup_error is not None: + if unterminated: + raise ServiceCleanupError( + f"{exc}; Docker cleanup error: {container_cleanup_error}", + unterminated, + ) from exc + raise RuntimeError(f"{exc}; Docker cleanup error: {container_cleanup_error}") from exc + raise + + def _record_port_lock_session(self, services: Iterable[object]) -> None: + raw_session_id = getattr(self.port_allocator, "session_id", None) + session_id = raw_session_id.strip() if isinstance(raw_session_id, str) else "" + if not session_id: + return + for service in services: + if isinstance(service, ServiceRecord): + service.port_lock_session = session_id def reserve_next(self, port: int) -> int: return self.port_allocator.reserve_next(port, owner=f"{self.project_name}:services") + def _release_failed_service_port(self, service_type: str, port: int) -> None: + release = getattr(self.port_allocator, "release", None) + if not callable(release): + return + for owner in ( + f"{self.project_name}:{service_type}", + f"{self.project_name}:services", + f"{self.project_name}:services:{service_type}-launch", + ): + release(port, owner=owner) + + def _release_rebound_launch_port(self, service_type: str) -> None: + port = self._rebound_launch_ports.pop(service_type, None) + if port is None: + return + release = getattr(self.port_allocator, "release", None) + if callable(release): + release(port, owner=f"{self.project_name}:services:{service_type}-launch") + + def _release_failed_start_reservations(self, error: BaseException) -> None: + preserved_service_types = { + str(getattr(service, "type", "") or "").strip().lower() + for service in dict(getattr(error, "unterminated_services", {}) or {}).values() + if _service_has_recorded_runtime_identity(service) + } + for service_type, port in tuple(self._active_service_ports.items()): + if service_type in preserved_service_types: + continue + self._release_failed_service_port(service_type, port) + self._active_service_ports.pop(service_type, None) + self._release_rebound_launch_port(service_type) + def _start_with_generic_attach( self, *, @@ -70,18 +175,42 @@ def _start_with_generic_attach( ) -> dict[str, ServiceRecord]: descriptor_by_type = {descriptor.service_type: descriptor for descriptor in descriptors} records: dict[str, ServiceRecord] = {} - for layer in ordered_service_layers( - tuple(descriptor.service_type for descriptor in descriptors), - self.additional_services, - ): - layer_records = generic_attach( - project=self.project_name, - descriptors=tuple(descriptor_by_type[name] for name in layer), - reserve_next=self.reserve_next, - on_retry=on_service_retry, - parallel_start=attach_parallel and len(layer) > 1, - ) - records.update(layer_records) + try: + for layer in ordered_service_layers( + tuple(descriptor.service_type for descriptor in descriptors), + self.additional_services, + ): + layer_records = generic_attach( + project=self.project_name, + descriptors=tuple(descriptor_by_type[name] for name in layer), + reserve_next=self.reserve_next, + on_retry=on_service_retry, + parallel_start=attach_parallel and len(layer) > 1, + ) + records.update(layer_records) + except BaseException as exc: + unterminated = dict(getattr(exc, "unterminated_services", {}) or {}) + terminator = getattr(self.runtime, "_terminate_started_services", None) + if records: + if callable(terminator): + try: + failed_names = unconfirmed_service_names( + terminator(records), + records, + ) + except BaseException as cleanup_exc: # noqa: BLE001 - cancellation must not skip cleanup + failed_names = set(records) + exc = RuntimeError(f"{exc}; cleanup error: {cleanup_exc}") + else: + failed_names = set(records) + unterminated.update((name, service) for name, service in records.items() if name in failed_names) + if unterminated: + names = ", ".join(sorted(unterminated)) + raise ServiceCleanupError( + f"{exc}; cleanup could not confirm exit for: {names}", + unterminated, + ) from exc + raise return records def _start_core_services_with_legacy_attach( @@ -109,6 +238,7 @@ def _start_core_services_with_legacy_attach( def _service_descriptors(self) -> list[ServiceStartDescriptor]: descriptors: list[ServiceStartDescriptor] = [] + pid_required = not self._docker_mode_enabled() if "backend" in self.selected_service_types: descriptors.append( ServiceStartDescriptor( @@ -119,6 +249,7 @@ def _service_descriptors(self) -> list[ServiceStartDescriptor]: detect_actual=self.detect_backend_actual, listener_expected=self.backend_listener_expected, log_path=self.backend_log_path, + pid_required=pid_required, ) ) if "frontend" in self.selected_service_types: @@ -131,6 +262,7 @@ def _service_descriptors(self) -> list[ServiceStartDescriptor]: detect_actual=self.detect_frontend_actual, listener_expected=True, log_path=self.frontend_log_path, + pid_required=pid_required, ) ) for service in sorted(self.additional_services, key=lambda item: (item.start_order, item.name)): @@ -140,9 +272,7 @@ def _service_descriptors(self) -> list[ServiceStartDescriptor]: service_type=service.name, cwd=str(launch.cwd), requested_port=launch.requested_port, - start=lambda port, service_name=service.name: self.start_additional_service( - service_name, port - ), + start=lambda port, service_name=service.name: self.start_additional_service(service_name, port), detect_actual=lambda pid, requested, service_name=service.name: self.detect_additional_actual( service_name, pid, requested ), @@ -151,6 +281,7 @@ def _service_descriptors(self) -> list[ServiceStartDescriptor]: log_path=launch.log_path, public_url=launch.env.get(f"ENVCTL_SOURCE_SERVICE_{service.env_suffix}_PUBLIC_URL"), health_url=launch.env.get(f"ENVCTL_SOURCE_SERVICE_{service.env_suffix}_HEALTH_URL"), + pid_required=pid_required, ) ) return descriptors @@ -197,35 +328,239 @@ def _launch_prepared_process( launch_port: int, event_port: int, env_extra: dict[str, str], - pid_fallback: int, ) -> tuple[bool, str | None, int | None]: launch_started = time.monotonic() - process = self._start_process( - command, - cwd=str(launch.cwd), - env=self.command_env_builder(port=launch_port, extra=env_extra), - log_path=launch.log_path, + service_env = self.command_env_builder(port=launch_port, extra=env_extra) + if self._docker_mode_enabled(): + try: + container_launch = DockerServiceRuntime( + self.runtime, + self.process_runtime, + refresh_cache=self.refresh_cache, + ).launch( + project_name=self.project_name, + project_root=self.project_root, + service_name=service_name, + cwd=launch.cwd, + command=command, + env=service_env, + host_port=event_port, + container_port=launch_port, + listener_expected=launch.listener_expected, + log_path=launch.log_path, + on_confirmed=lambda confirmed: self._container_launches.__setitem__( + service_name, + confirmed, + ), + ) + except DockerServiceLaunchCleanupError as exc: + self._container_launches[service_name] = exc.launch + self._emit_attach_phase(service_name, "container_launch", launch_started) + detail = str(exc) + self.runtime._emit( + "service.failure", + project=self.project_name, + service=service_name, + failure_class="container_cleanup_unconfirmed", + requested_port=event_port, + detail=detail, + ) + return False, detail, None + except (OSError, RuntimeError) as exc: + self._emit_attach_phase(service_name, "container_launch", launch_started) + detail = str(exc) + self.runtime._emit( + "service.failure", + project=self.project_name, + service=service_name, + failure_class="container_start_failed", + requested_port=event_port, + detail=detail, + ) + return False, detail, None + self._container_launches[service_name] = container_launch + process_pid = None + phase = "container_launch" + else: + try: + process = self._start_process( + command, + cwd=str(launch.cwd), + env=service_env, + log_path=launch.log_path, + ) + except ProcessHandoffCleanupError as exc: + detail = str(exc) + self.runtime._emit( + "service.failure", + project=self.project_name, + service=service_name, + failure_class="process_cleanup_unconfirmed", + requested_port=event_port, + detail=detail, + ) + return False, detail, exc.pid + except OSError as exc: + executable = command[0] if command else "" + detail = f"process spawn failed for {executable}: {exc}" + self.runtime._emit( + "service.failure", + project=self.project_name, + service=service_name, + failure_class=ServiceFailureReason.PROCESS_SPAWN_FAILED.value, + requested_port=event_port, + detail=detail, + ) + return False, detail, None + pid = getattr(process, "pid", None) + process_pid = pid if isinstance(pid, int) and pid > 0 else None + phase = "process_launch" + try: + self._emit_attach_phase(service_name, phase, launch_started) + self.runtime._emit( + "service.start", + project=self.project_name, + service=service_name, + port=event_port, + retry=False, + ) + except Exception: # noqa: BLE001 + pass + return True, None, process_pid + + def _container_actual_port( + self, + service_name: str, + requested: int, + *, + listener_expected: bool = True, + ) -> int | None: + launch = self._container_launches.get(service_name) + if launch is None: + return None + ready = DockerServiceRuntime(self.runtime, self.process_runtime).wait_until_ready( + launch, + port=requested, + listener_expected=listener_expected, ) - self._emit_attach_phase(service_name, "process_launch", launch_started) + if ready: + return requested if listener_expected else None + detail = f"container {launch.container_name} exited or did not become ready" + cleanup_detail = self._remove_failed_container_launch(service_name, launch) + if cleanup_detail: + detail = f"{detail}; {cleanup_detail}" self.runtime._emit( - "service.start", + "service.failure", project=self.project_name, service=service_name, - port=event_port, - retry=False, + failure_class="container_not_ready", + requested_port=requested if listener_expected else None, + detail=detail, ) - return True, None, getattr(process, "pid", pid_fallback) + raise RuntimeError(detail) + + def _remove_failed_container_launch( + self, + service_name: str, + launch: DockerServiceLaunch, + ) -> str | None: + try: + stop_kwargs = ( + {"expected_launch_token": launch.launch_token} if launch.launch_token else {} + ) + if launch.cleanup_pending_since is not None: + stop_kwargs["pending_cleanup_since"] = launch.cleanup_pending_since + removed = DockerServiceRuntime(self.runtime, self.process_runtime).stop( + launch.container_id, + **stop_kwargs, + ) + except BaseException as exc: # noqa: BLE001 - preserve the launch in state for a later stop retry + return f"failed to remove container: {exc}" + if not removed: + return "failed to remove container" + if self._container_launches.get(service_name) == launch: + self._container_launches.pop(service_name, None) + return None + + def _docker_mode_enabled(self) -> bool: + return self.docker_mode or docker_service_mode_enabled(self.runtime) + + def _annotate_container_records(self, records: dict[str, ServiceRecord]) -> None: + for record in records.values(): + service_name = str(record.service_slug or record.type) + launch = self._container_launches.get(service_name) + if launch is None: + continue + record.runtime_kind = "docker" + record.container_id = launch.container_id + record.container_name = launch.container_name + record.container_image = launch.image + record.container_launch_token = launch.launch_token or None + record.container_cleanup_pending_since = launch.cleanup_pending_since + + def _cleanup_container_launches(self) -> None: + docker_runtime = DockerServiceRuntime(self.runtime, self.process_runtime) + unconfirmed: dict[str, ServiceRecord] = {} + details: list[str] = [] + for service_name, launch in tuple(self._container_launches.items()): + try: + stop_kwargs = ( + {"expected_launch_token": launch.launch_token} if launch.launch_token else {} + ) + if launch.cleanup_pending_since is not None: + stop_kwargs["pending_cleanup_since"] = launch.cleanup_pending_since + stopped = docker_runtime.stop(launch.container_id, **stop_kwargs) + except BaseException as exc: # noqa: BLE001 - cancellation must not abandon authority + stopped = False + details.append(f"{service_name}: {exc}") + if stopped: + self._container_launches.pop(service_name, None) + continue + details.append(f"{service_name}: removal unconfirmed") + record = self._container_service_record(service_name, launch) + unconfirmed[record.name] = record + if unconfirmed: + raise ServiceCleanupError( + "Docker cleanup could not confirm container removal: " + "; ".join(dict.fromkeys(details)), + unconfirmed, + ) + + def _container_service_record( + self, + service_name: str, + launch: DockerServiceLaunch, + ) -> ServiceRecord: + prepared = self.prepared_launches.get(service_name) + listener_expected = bool(getattr(prepared, "listener_expected", True)) + port = self._active_service_ports.get(service_name) + record = ServiceRecord( + name=f"{self.project_name} {service_display_name(service_name)}", + type=service_name, + cwd=str(getattr(prepared, "cwd", self.project_root)), + requested_port=port if listener_expected else None, + actual_port=port if listener_expected else None, + log_path=launch.log_path, + status="termination_failed", + listener_expected=listener_expected, + project=self.project_name, + service_slug=service_name, + failure_detail="Docker container removal was not confirmed", + degraded=True, + runtime_kind="docker", + container_id=launch.container_id, + container_name=launch.container_name, + container_image=launch.image, + container_launch_token=launch.launch_token or None, + container_cleanup_pending_since=launch.cleanup_pending_since, + ) + return record def start_backend(self, port: int) -> tuple[bool, str | None, int | None]: conflict_result = self._conflicting_start_result("backend", port) if conflict_result is not None: return conflict_result command_resolve_started = time.monotonic() - command, _resolved_source = self.runtime._service_start_command_resolved( - service_name="backend", - project_root=self.project_root, - port=port, - ) + command = self._core_service_command("backend", port) self._emit_attach_phase("backend", "command_resolution", command_resolve_started) return self._launch_prepared_process( service_name="backend", @@ -234,7 +569,6 @@ def start_backend(self, port: int) -> tuple[bool, str | None, int | None]: launch_port=port, event_port=port, env_extra=self.backend_env_extra, - pid_fallback=os.getpid(), ) def start_frontend(self, port: int) -> tuple[bool, str | None, int | None]: @@ -247,22 +581,40 @@ def start_frontend(self, port: int) -> tuple[bool, str | None, int | None]: launch_port, owner=f"{self.project_name}:services:frontend-launch", ) - command_resolve_started = time.monotonic() + self._rebound_launch_ports["frontend"] = launch_port + try: + command_resolve_started = time.monotonic() + command = self._core_service_command("frontend", launch_port) + self._emit_attach_phase("frontend", "command_resolution", command_resolve_started) + result = self._launch_prepared_process( + service_name="frontend", + command=command, + launch=self.prepared_launches["frontend"], + launch_port=launch_port, + event_port=port, + env_extra=self.frontend_env_extra, + ) + except BaseException: + self._release_rebound_launch_port("frontend") + raise + if not isinstance(result[2], int) or isinstance(result[2], bool) or result[2] <= 0: + self._release_rebound_launch_port("frontend") + return result + + def _core_service_command(self, service_name: str, port: int) -> list[str]: + container_command_source = docker_service_container_command_source( + self.runtime, + service_name, + docker_mode=self._docker_mode_enabled(), + ) + if container_command_source is not None: + return [] command, _resolved_source = self.runtime._service_start_command_resolved( - service_name="frontend", + service_name=service_name, project_root=self.project_root, - port=launch_port, - ) - self._emit_attach_phase("frontend", "command_resolution", command_resolve_started) - return self._launch_prepared_process( - service_name="frontend", - command=command, - launch=self.prepared_launches["frontend"], - launch_port=launch_port, - event_port=port, - env_extra=self.frontend_env_extra, - pid_fallback=os.getpid() + 1, + port=port, ) + return command def start_additional_service(self, service_name: str, port: int) -> tuple[bool, str | None, int | None]: conflict_result = self._conflicting_start_result(service_name, port) @@ -273,16 +625,23 @@ def start_additional_service(self, service_name: str, port: int) -> tuple[bool, if service is None: return False, f"unknown additional service {service_name}", None command_resolve_started = time.monotonic() - command = self.runtime._split_command( - service.start_cmd, - port=port, - replacements={ - "project_root": str(self.project_root), - "service_dir": str(launch.cwd), - "service_name": service_name, - }, - cwd=launch.cwd, - ) + if docker_service_container_command_source( + self.runtime, + service_name, + docker_mode=self._docker_mode_enabled(), + ) is not None: + command = [] + else: + command = self.runtime._split_command( + service.start_cmd, + port=port, + replacements={ + "project_root": str(self.project_root), + "service_dir": str(launch.cwd), + "service_name": service_name, + }, + cwd=launch.cwd, + ) self._emit_attach_phase(service_name, "command_resolution", command_resolve_started) return self._launch_prepared_process( service_name=service_name, @@ -291,10 +650,15 @@ def start_additional_service(self, service_name: str, port: int) -> tuple[bool, launch_port=port, event_port=port, env_extra=launch.env, - pid_fallback=os.getpid(), ) def detect_backend_actual(self, pid: int | None, requested: int) -> int | None: + if "backend" in self._container_launches: + return self._container_actual_port( + "backend", + requested, + listener_expected=self.backend_listener_expected, + ) if not self.backend_listener_expected: self.runtime._emit( "service.bind.skipped", @@ -334,6 +698,9 @@ def detect_backend_actual(self, pid: int | None, requested: int) -> int | None: return actual def detect_frontend_actual(self, pid: int | None, requested: int) -> int | None: + container_actual = self._container_actual_port("frontend", requested) + if "frontend" in self._container_launches: + return container_actual self.runtime._emit( "service.bind.requested", project=self.project_name, @@ -374,6 +741,12 @@ def detect_frontend_actual(self, pid: int | None, requested: int) -> int | None: def detect_additional_actual(self, service_name: str, pid: int | None, requested: int) -> int | None: launch = self.prepared_launches[service_name] + if service_name in self._container_launches: + return self._container_actual_port( + service_name, + requested, + listener_expected=launch.listener_expected, + ) if not launch.listener_expected: self.runtime._emit( "service.bind.skipped", @@ -431,17 +804,22 @@ def _resolve_detected_actual( if self.runtime._listener_truth_enforced(): detail = self.runtime._service_listener_failure_detail(log_path=log_path, pid=pid) error_suffix = f" ({detail})" if detail else "" + failure_class = service_listener_failure_class(detail) self.runtime._emit( "service.failure", project=self.project_name, service=service_name, - failure_class="listener_not_detected", + failure_class=failure_class, requested_port=requested, detail=detail, ) - raise RuntimeError( - f"{service_name} listener not detected for {self.project_name} on port {failure_port}{error_suffix}" - ) + if failure_class == ServiceFailureReason.DEPENDENCY_MISSING.value: + failure_summary = f"{service_name} is missing a required executable or module" + elif failure_class == ServiceFailureReason.PROCESS_EXITED.value: + failure_summary = f"{service_name} process exited before opening its listener" + else: + failure_summary = f"{service_name} listener not detected" + raise RuntimeError(f"{failure_summary} for {self.project_name} on port {failure_port}{error_suffix}") self.runtime._emit( "service.failure", project=self.project_name, @@ -466,3 +844,13 @@ def _emit_attach_phase(self, service_name: str, phase: str, started: float) -> N phase=phase, duration_ms=round((time.monotonic() - started) * 1000.0, 2), ) + + +def _service_has_recorded_runtime_identity(service: object) -> bool: + if str(getattr(service, "runtime_kind", "process") or "process").strip().lower() == "docker": + if str(getattr(service, "container_id", "") or getattr(service, "container_name", "")).strip(): + return True + listener_pids = getattr(service, "listener_pids", None) + listeners = listener_pids if isinstance(listener_pids, (list, tuple, set, frozenset)) else () + candidates = [getattr(service, "pid", None), *listeners] + return any(isinstance(pid, int) and not isinstance(pid, bool) and pid > 0 for pid in candidates) diff --git a/python/envctl_engine/startup/service_project_execution.py b/python/envctl_engine/startup/service_project_execution.py index 5602ed65c..89077ea3f 100644 --- a/python/envctl_engine/startup/service_project_execution.py +++ b/python/envctl_engine/startup/service_project_execution.py @@ -6,6 +6,10 @@ from typing import cast from envctl_engine.runtime.command_resolution import CommandExists, suggest_service_start_command +from envctl_engine.runtime.docker_service_runtime import ( + docker_service_container_command_source, + docker_service_mode_enabled, +) from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.runtime_context import resolve_process_runtime from envctl_engine.startup.startup_selection_support import _restart_service_types_for_project @@ -281,12 +285,18 @@ def project_env_for_service(service_name: str) -> dict[str, str]: reason="all_services_disabled", ) return {} - if route is not None and route.command == "plan" and _no_local_app_system_configured( - rt, - mode=effective_mode, - project_root=context.root, - selected_service_types=selected_service_types, - configured_additional_services=configured_additional_services, + use_docker_services = bool(route is not None and route.flags.get("docker")) or docker_service_mode_enabled(rt) + if ( + route is not None + and route.command == "plan" + and not use_docker_services + and _no_local_app_system_configured( + rt, + mode=effective_mode, + project_root=context.root, + selected_service_types=selected_service_types, + configured_additional_services=configured_additional_services, + ) ): rt._emit( "service.attach.skipped", @@ -320,28 +330,34 @@ def project_env_for_service(service_name: str) -> dict[str, str]: def prepare_backend_runtime() -> float: started = time.monotonic() - rt._prepare_backend_runtime( - context=context, - backend_cwd=backend_cwd, - backend_log_path=backend_log_path, - project_env_base=project_env_internal, - route=route, - backend_env_file=backend_env_file, - backend_env_is_default=backend_env_is_default, - ) + if use_docker_services: + rt._emit("service.prepare.skipped", project=context.name, service="backend", reason="docker_image") + else: + rt._prepare_backend_runtime( + context=context, + backend_cwd=backend_cwd, + backend_log_path=backend_log_path, + project_env_base=project_env_internal, + route=route, + backend_env_file=backend_env_file, + backend_env_is_default=backend_env_is_default, + ) return round((time.monotonic() - started) * 1000.0, 2) def prepare_frontend_runtime() -> float: started = time.monotonic() - rt._prepare_frontend_runtime( - context=context, - frontend_cwd=frontend_cwd, - frontend_log_path=frontend_log_path, - project_env_base=frontend_project_env_base, - frontend_env_file=frontend_env_file, - backend_port=backend_plan.final, - route=route, - ) + if use_docker_services: + rt._emit("service.prepare.skipped", project=context.name, service="frontend", reason="docker_image") + else: + rt._prepare_frontend_runtime( + context=context, + frontend_cwd=frontend_cwd, + frontend_log_path=frontend_log_path, + project_env_base=frontend_project_env_base, + frontend_env_file=frontend_env_file, + backend_port=backend_plan.final, + route=route, + ) return round((time.monotonic() - started) * 1000.0, 2) if prepare_backend and prepare_frontend and prep_parallel: @@ -379,18 +395,30 @@ def prepare_frontend_runtime() -> float: rebound_delta = parse_int(rt.env.get("ENVCTL_TEST_FRONTEND_REBOUND_DELTA"), 0) backend_command_source = None if "backend" in selected_service_types: - backend_command_source = rt._service_command_source( - service_name="backend", - project_root=context.root, - port=backend_plan.final, + backend_command_source = docker_service_container_command_source( + rt, + "backend", + docker_mode=use_docker_services, ) + if backend_command_source is None: + backend_command_source = rt._service_command_source( + service_name="backend", + project_root=context.root, + port=backend_plan.final, + ) frontend_command_source = None if "frontend" in selected_service_types: - frontend_command_source = rt._service_command_source( - service_name="frontend", - project_root=context.root, - port=(frontend_plan.final + rebound_delta if rebound_delta > 0 else frontend_plan.final), + frontend_command_source = docker_service_container_command_source( + rt, + "frontend", + docker_mode=use_docker_services, ) + if frontend_command_source is None: + frontend_command_source = rt._service_command_source( + service_name="frontend", + project_root=context.root, + port=(frontend_plan.final + rebound_delta if rebound_delta > 0 else frontend_plan.final), + ) prepared_launches: dict[str, PreparedServiceLaunch] = {} command_env_builder = resolve_command_env_builder(rt) @@ -501,6 +529,8 @@ def prepare_frontend_runtime() -> float: additional_services=tuple(additional_services), backend_listener_expected=backend_listener_expected, rebound_delta=rebound_delta, + docker_mode=use_docker_services, + refresh_cache=bool(route is not None and route.flags.get("refresh_cache")), ).start(attach_parallel=attach_parallel, on_service_retry=on_service_retry) attach_duration_ms = round((time.monotonic() - attach_started) * 1000.0, 2) diff --git a/python/envctl_engine/startup/session.py b/python/envctl_engine/startup/session.py index 9cd91a696..d50043143 100644 --- a/python/envctl_engine/startup/session.py +++ b/python/envctl_engine/startup/session.py @@ -1,7 +1,8 @@ from __future__ import annotations +from collections.abc import Collection, Mapping import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Any, cast from envctl_engine.runtime.command_router import Route @@ -54,11 +55,17 @@ class StartupSession: restart_state: RunState | None = None requirements_by_project: dict[str, RequirementsResult] = field(default_factory=dict) services_by_project: dict[str, dict[str, ServiceRecord]] = field(default_factory=dict) + unterminated_services: dict[str, ServiceRecord] = field(default_factory=dict) started_context_names: list[str] = field(default_factory=list) dashboard_after_start: bool = False disabled_startup_mode: bool = False used_project_spinner_group: bool = False strict_truth_failed: bool = False + authority_committed: bool = False + preserve_existing_state_on_failure: bool = False + service_state_collisions: set[str] = field(default_factory=set) + service_state_collision_rows: list[dict[str, object]] = field(default_factory=list) + requirement_state_collision_rows: list[dict[str, object]] = field(default_factory=list) failure_message: str | None = None errors: list[str] = field(default_factory=list) warnings: list[str] = field(default_factory=list) @@ -79,15 +86,61 @@ class StartupSession: @property def merged_services(self) -> dict[str, ServiceRecord]: + replacement_names = set(self.unterminated_services) + for project_services in self.services_by_project.values(): + replacement_names.update(project_services) + collisions = sorted(set(self.preserved_services).intersection(replacement_names)) + if collisions: + self.service_state_collisions.update(collisions) + self.preserve_existing_state_on_failure = True + raise RuntimeError( + "Refusing to overwrite preserved service state with newly started services: " + + ", ".join(collisions) + ) services = dict(self.preserved_services) for project_services in self.services_by_project.values(): services.update(project_services) + services.update(self.unterminated_services) return services @property def merged_requirements(self) -> dict[str, RequirementsResult]: requirements = dict(self.preserved_requirements) - requirements.update(self.requirements_by_project) + collisions: list[str] = [] + for project, replacement in self.requirements_by_project.items(): + # App-only restart reuses the exact persisted RequirementsResult. + # Its durable storage key can be a collision alias, while project + # startup naturally reports the canonical project name. Retain the + # existing storage identity instead of serializing the same stack + # twice under two keys. + if any(existing is replacement for existing in requirements.values()): + continue + preserved_key = _preserved_requirement_key(requirements, project) + preserved = requirements.get(preserved_key) if preserved_key is not None else None + if preserved is not None and preserved == replacement: + # Preserve the existing storage key, including a collision + # alias, when deserialization produced an equal object. + continue + if ( + preserved is not None + and self.effective_route.flags.get("launch_dependencies") is False + and not _requirements_have_enabled_components(replacement) + ): + # App-only and no-infrastructure starts still return a + # RequirementsResult containing disabled port projections. + # Those projections do not own resources and must not collide + # with, duplicate, or replace the preserved dependency stack. + continue + if preserved is not None: + collisions.append(project) + continue + requirements[project] = replacement + if collisions: + self.preserve_existing_state_on_failure = True + raise RuntimeError( + "Refusing to overwrite preserved requirement state with newly started requirements: " + + ", ".join(sorted(collisions)) + ) return requirements @property @@ -111,3 +164,158 @@ def plan_agent_launch_degraded(self) -> bool: if launch_result is None: return False return str(getattr(launch_result, "status", "")).strip().lower() == "partial" + + +def unconfirmed_service_names( + termination_result: object, + services: Mapping[str, object], +) -> set[str]: + """Return services whose exit was not positively confirmed. + + The termination contract returns a collection of names that remain alive; + an explicit empty collection therefore confirms complete cleanup. Legacy + adapters and failed mocks may return ``None`` or another scalar. Treating + those values as success loses process and port ownership, so they retain + every service for a later verified cleanup pass. + """ + + if not isinstance(termination_result, Collection) or isinstance( + termination_result, + (str, bytes, Mapping), + ): + return set(services) + raw_reported = list(termination_result) + if any(not str(raw_name).strip() for raw_name in raw_reported): + return set(services) + reported = {str(raw_name).strip() for raw_name in raw_reported} + if not reported.issubset(services): + # An adapter that reports an unknown identity has not positively + # confirmed the disposition of any tracked process. Fail closed so a + # stale/case-mismatched name cannot make every real service disappear + # from the durable recovery state. + return set(services) + return reported + + +def _preserved_requirement_key( + requirements: Mapping[str, RequirementsResult], + project: str, +) -> str | None: + target = str(project).strip().casefold() + exact = [key for key in requirements if str(key).strip().casefold() == target] + if len(exact) == 1: + return exact[0] + if len(exact) > 1: + raise RuntimeError( + f"Ambiguous preserved requirements authority for {project}: " + + ", ".join(sorted(exact)) + ) + aliases = [ + key + for key, value in requirements.items() + if str(getattr(value, "project", "") or "").strip().casefold() == target + ] + if len(aliases) == 1: + return aliases[0] + if len(aliases) > 1: + raise RuntimeError( + f"Ambiguous preserved requirements authority for {project}: " + + ", ".join(sorted(aliases)) + ) + return None + + +def _requirements_have_enabled_components(requirements: RequirementsResult) -> bool: + return any( + bool(component.get("enabled", False)) + for component in requirements.components.values() + if isinstance(component, Mapping) + ) + + +def metadata_with_state_sources(metadata: dict[str, Any], state: RunState) -> dict[str, Any]: + """Return metadata that records every state replaced by a new startup run.""" + + updated = dict(metadata) + source_run_ids = {state.run_id} + for source in ( + updated.get("state_source_run_ids"), + state.metadata.get("state_source_run_ids"), + ): + if not isinstance(source, list): + continue + source_run_ids.update(str(run_id).strip() for run_id in source if str(run_id).strip()) + updated["state_source_run_ids"] = sorted(source_run_ids) + return updated + + +def track_startup_failure(session: StartupSession, failure: BaseException) -> None: + project = str(getattr(failure, "project", "") or "").strip() + requirements = getattr(failure, "requirements", None) + if project and isinstance(requirements, RequirementsResult): + requirement_key = project + if project in session.requirements_by_project: + occupied_projects = set(session.preserved_requirements).union(session.requirements_by_project) + base = f"{project} Restart Collision" + requirement_key = base + index = 2 + while requirement_key in occupied_projects: + requirement_key = f"{base} {index}" + index += 1 + session.requirement_state_collision_rows.append( + { + "original_project": project, + "replacement_project": requirement_key, + "replacement_requirements_retained": True, + } + ) + session.requirements_by_project[requirement_key] = requirements + services = getattr(failure, "unterminated_services", None) + if not isinstance(services, dict): + return + occupied_names = set(session.preserved_services).union(session.unterminated_services) + for project_services in session.services_by_project.values(): + occupied_names.update(project_services) + tracked_identities = { + id(service) + for service in session.unterminated_services.values() + } + tracked_identities.update( + id(service) + for project_services in session.services_by_project.values() + for service in project_services.values() + ) + for name, service in services.items(): + if isinstance(name, str) and isinstance(service, ServiceRecord): + if id(service) in tracked_identities: + continue + stored_name = name + stored_service = service if service.project or not project else replace(service, project=project) + if stored_name in occupied_names: + pid = getattr(service, "pid", None) + suffix = f"Restart Collision {pid}" if isinstance(pid, int) and pid > 0 else "Restart Collision" + base = f"{name} {suffix}" + stored_name = base + index = 2 + while stored_name in occupied_names or stored_name in services: + stored_name = f"{base} {index}" + index += 1 + stored_service = replace(stored_service, name=stored_name) + session.service_state_collisions.add(name) + session.service_state_collision_rows.append( + { + "original_name": name, + "replacement_name": stored_name, + "replacement_pid": service.pid, + "replacement_project": service.project or project or None, + } + ) + if name in session.preserved_services: + session.preserve_existing_state_on_failure = True + occupied_names.add(stored_name) + tracked_identities.add(id(service)) + session.unterminated_services[stored_name] = stored_service + + +def track_unterminated_services(session: StartupSession, failure: BaseException) -> None: + track_startup_failure(session, failure) diff --git a/python/envctl_engine/startup/startup_execution_support.py b/python/envctl_engine/startup/startup_execution_support.py index 631fb4a7e..7641298b9 100644 --- a/python/envctl_engine/startup/startup_execution_support.py +++ b/python/envctl_engine/startup/startup_execution_support.py @@ -18,6 +18,7 @@ ) from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_env import effective_dependency_scope +from envctl_engine.shared.services import service_slug_from_record from envctl_engine.startup.protocols import ProjectContextLike, StartupOrchestratorLike from envctl_engine.startup.requirements_execution import ( REQUIREMENTS_PROGRESS_PROJECT_FLAG, @@ -29,8 +30,9 @@ startup_breakdown_enabled, ) from envctl_engine.startup.service_execution import start_project_services as start_project_services_impl -from envctl_engine.startup.session import ProjectStartupResult -from envctl_engine.state.models import PortPlan, RequirementsResult +from envctl_engine.startup.session import ProjectStartupResult, unconfirmed_service_names +from envctl_engine.state.models import PortPlan, RequirementsResult, ServiceRecord +from envctl_engine.state.project_runtime import requirements_for_project @dataclass(slots=True) @@ -40,6 +42,23 @@ class _SharedProjectContext: ports: Mapping[str, PortPlan] +class ProjectStartupFailure(RuntimeError): + """A project failure that preserves already-started runtime resources.""" + + def __init__( + self, + message: str, + *, + project: str, + requirements: RequirementsResult, + unterminated_services: dict[str, ServiceRecord] | None = None, + ) -> None: + super().__init__(message) + self.project = project + self.requirements = requirements + self.unterminated_services = dict(unterminated_services or {}) + + def start_project_context( orchestrator: StartupOrchestratorLike, *, @@ -55,50 +74,79 @@ def start_project_context( report_progress_fn(route, f"Starting project {context.name}...", project=context.name) _reserve_project_ports(rt, context, route=route) requirements = _requirements_for_project_context(orchestrator, context=context, mode=mode, route=route) - if not rt._requirements_ready(requirements): - raise RuntimeError(_requirements_failure_message(context.name, requirements)) - _sync_supabase_auth_users_before_services( - orchestrator, - context=context, - mode=mode, - route=route, - requirements=requirements, - ) - requirements_progress_message = _requirements_ready_progress_message( - orchestrator, - context=context, - mode=mode, - route=route, - requirements=requirements, - ) - if requirements_progress_message is not None: - report_progress_fn(route, requirements_progress_message, project=context.name) - project_services = orchestrator.start_project_services( - context, - requirements=requirements, - run_id=run_id, - route=route, - ) + project_services: dict[str, ServiceRecord] = {} try: + if not rt._requirements_ready(requirements): + raise RuntimeError(_requirements_failure_message(context.name, requirements)) + _sync_supabase_auth_users_before_services( + orchestrator, + context=context, + mode=mode, + route=route, + requirements=requirements, + ) + requirements_progress_message = _requirements_ready_progress_message( + orchestrator, + context=context, + mode=mode, + route=route, + requirements=requirements, + ) + if requirements_progress_message is not None: + report_progress_fn(route, requirements_progress_message, project=context.name) + project_services = orchestrator.start_project_services( + context, + requirements=requirements, + run_id=run_id, + route=route, + ) rt._assert_project_services_post_start_truth(context=context, services=project_services) - except RuntimeError: - rt._terminate_started_services(project_services) - raise - report_progress_fn( - route, - f"Services ready for {context.name}: " - + " ".join( - f"{service_type}={_service_ready_label(project_services.get(f'{context.name} {service_type.title()}'))}" - for service_type in ("backend", "frontend") - if f"{context.name} {service_type.title()}" in project_services - ), - project=context.name, - ) - return ProjectStartupResult( - requirements=requirements, - services=project_services, - warnings=list(rt._consume_project_startup_warnings(context.name)), - ) + services_by_type = { + service_slug_from_record(service): service + for service in project_services.values() + if service_slug_from_record(service) + } + report_progress_fn( + route, + f"Services ready for {context.name}: " + + " ".join( + f"{service_type}={_service_ready_label(services_by_type.get(service_type))}" + for service_type in ("backend", "frontend") + if service_type in services_by_type + ), + project=context.name, + ) + return ProjectStartupResult( + requirements=requirements, + services=project_services, + warnings=list(rt._consume_project_startup_warnings(context.name)), + ) + except Exception as exc: + unterminated = dict(getattr(exc, "unterminated_services", {}) or {}) + cleanup_error: Exception | None = None + if project_services: + try: + failed_services = unconfirmed_service_names( + rt._terminate_started_services(project_services), + project_services, + ) + except Exception as cleanup_exc: # noqa: BLE001 + cleanup_error = cleanup_exc + failed_services = set(project_services) + unterminated.update( + (name, service) for name, service in project_services.items() if name in failed_services + ) + message = str(exc) + if unterminated and "cleanup could not confirm exit for:" not in message: + message = f"{message}; cleanup could not confirm exit for: {', '.join(sorted(unterminated))}" + if cleanup_error is not None: + message = f"{message}; cleanup error: {cleanup_error}" + raise ProjectStartupFailure( + message, + project=context.name, + requirements=requirements, + unterminated_services=unterminated, + ) from exc def _reserve_project_ports(rt: Any, context: ProjectContextLike, *, route: Route | None) -> None: @@ -281,7 +329,7 @@ def _load_or_start_shared_main_requirements( rt = orchestrator.runtime existing = rt._try_load_existing_state(mode="main", strict_mode_match=True) if existing is not None: - requirements = existing.requirements.get("Main") + requirements = requirements_for_project(existing, "Main") if isinstance(requirements, RequirementsResult) and rt._requirements_ready(requirements): reconciled = _annotate_shared_main_requirements(orchestrator, requirements) if _shared_requirements_runtime_ready(reconciled): @@ -335,10 +383,13 @@ def _annotate_shared_main_requirements( ) supabase = cloned.component("supabase") if bool(supabase.get("enabled", False)): - supabase["container_name"] = build_supabase_project_name( - project_root=project_root, - project_name="Main", - ) + "-supabase-db-1" + supabase["container_name"] = ( + build_supabase_project_name( + project_root=project_root, + project_name="Main", + ) + + "-supabase-db-1" + ) return _reconcile_shared_main_requirements(orchestrator, cloned) diff --git a/python/envctl_engine/startup/startup_selection_support.py b/python/envctl_engine/startup/startup_selection_support.py index 2235e19a6..e83872f50 100644 --- a/python/envctl_engine/startup/startup_selection_support.py +++ b/python/envctl_engine/startup/startup_selection_support.py @@ -1,13 +1,21 @@ from __future__ import annotations from collections import OrderedDict +from collections.abc import Mapping from pathlib import Path +from envctl_engine.dashboard_metadata import ( + DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY, + DASHBOARD_STOPPED_SERVICES_KEY, + dashboard_project_configured_services_from_metadata, + normalize_dashboard_service_types, +) from envctl_engine.planning import list_planning_files, planning_existing_counts, select_projects_for_plan_files from envctl_engine.runtime.command_router import MODE_TREE_TOKENS, Route from envctl_engine.runtime.runtime_context import resolve_port_allocator, resolve_process_runtime from envctl_engine.shared.protocols import PortAllocator, ProcessRuntime from envctl_engine.shared.services import ( + resolve_service_project_name, service_display_name, service_matches_selector, service_project_name, @@ -34,17 +42,39 @@ def project_app_ports_text(context: ProjectContextLike) -> str: def state_project_names(*, runtime: StartupRuntime, state: RunState) -> set[str]: - names = {str(name).strip() for name in state.requirements.keys() if str(name).strip()} - for service_name in state.services: - project_name = runtime._project_name_from_service(service_name) - if isinstance(project_name, str) and project_name.strip(): - names.add(project_name.strip()) - metadata_roots = state.metadata.get("project_roots") + requirements = getattr(state, "requirements", {}) + names = { + str(getattr(requirement, "project", "") or storage_name).strip() + for storage_name, requirement in requirements.items() + if str(getattr(requirement, "project", "") or storage_name).strip() + } + for service_name, service in getattr(state, "services", {}).items(): + normalized = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) + if normalized: + names.add(normalized) + metadata = getattr(state, "metadata", {}) + metadata_roots = metadata.get("project_roots") if isinstance(metadata, Mapping) else None if isinstance(metadata_roots, dict): for project_name in metadata_roots: normalized = str(project_name).strip() if normalized: names.add(normalized) + if isinstance(metadata, Mapping): + names.update(dashboard_project_configured_services_from_metadata(metadata)) + raw_stopped = metadata.get(DASHBOARD_STOPPED_SERVICES_KEY) + if isinstance(raw_stopped, list): + for item in raw_stopped: + if not isinstance(item, Mapping): + continue + project_name = str(item.get("project", "") or "").strip() + if project_name: + names.add(project_name) + if not names and str(getattr(state, "mode", "")).strip().lower() == "main": + names.add("Main") return names @@ -97,13 +127,19 @@ def trees_start_selection_required(*, route: Route, runtime_mode: str) -> bool: def tree_preselected_projects_from_state( *, runtime: StartupRuntime, project_contexts: list[ProjectContextLike] ) -> list[str]: - state = runtime._try_load_existing_state(mode="trees", strict_mode_match=True) + repository = getattr(runtime, "state_repository", None) + load_all = getattr(repository, "load_all", None) + states = [state for state in load_all(mode="trees") if isinstance(state, RunState)] if callable(load_all) else [] + if not states: + state = runtime._try_load_existing_state(mode="trees", strict_mode_match=True) + states = [state] if state is not None else [] available = {str(context.name).strip().lower(): str(context.name).strip() for context in project_contexts} - if state is None: + if not states: return _tree_preselected_projects_from_plans(runtime=runtime, project_contexts=project_contexts) preselected: list[str] = [] seen: set[str] = set() - for name in sorted(state_project_names(runtime=runtime, state=state)): + active_names = {name for state in states for name in state_project_names(runtime=runtime, state=state)} + for name in sorted(active_names): normalized = str(name).strip().lower() resolved = available.get(normalized) if not resolved: @@ -138,9 +174,7 @@ def _tree_preselected_projects_from_plans( ] existing_counts = planning_existing_counts(projects=projects, planning_files=planning_files) plan_counts = OrderedDict( - (plan_file, count) - for plan_file in planning_files - if (count := int(existing_counts.get(plan_file, 0))) > 0 + (plan_file, count) for plan_file in planning_files if (count := int(existing_counts.get(plan_file, 0))) > 0 ) if not plan_counts: return [] @@ -228,6 +262,9 @@ def restart_include_requirements(route: Route) -> bool: return False if runtime_scope in {"dependencies", "entire-system"}: return True + launch_dependencies = route.flags.get("launch_dependencies") + if launch_dependencies is not None: + return bool(launch_dependencies) if bool(route.flags.get("all")): return True services = route.flags.get("services") @@ -241,6 +278,18 @@ def restart_include_requirements(route: Route) -> bool: def _restart_selected_services(*, state: RunState, route: Route) -> set[str]: service_filters = route.flags.get("services") selected: set[str] = set() + project_keys = { + str(project).strip().casefold() + for project in route.projects + if str(project).strip() + } + + def in_selected_projects(name: str, service: object) -> bool: + if not project_keys: + return True + project = service_project_name(service) or _project_name_from_service_name(name) + return bool(project and project.casefold() in project_keys) + if isinstance(service_filters, list): filters = [str(value).strip() for value in service_filters if str(value).strip()] for selector in filters: @@ -248,7 +297,8 @@ def _restart_selected_services(*, state: RunState, route: Route) -> set[str]: { name for name, service in state.services.items() - if name == selector or service_matches_selector(service, selector) + if (name == selector or service_matches_selector(service, selector)) + and in_selected_projects(name, service) } ) return selected @@ -276,53 +326,313 @@ def _restart_selected_services(*, state: RunState, route: Route) -> set[str]: if selected: return selected + runtime_scope = route.flags.get("runtime_scope") + if runtime_scope in {"backend", "frontend", "fullstack", "dependencies"}: + selected_types = { + "backend", + "frontend", + } if runtime_scope == "fullstack" else ({str(runtime_scope)} if runtime_scope != "dependencies" else set()) + return { + name + for name, service in state.services.items() + if service_slug_from_record(service) in selected_types and in_selected_projects(name, service) + } + if route.projects: project_set = { project.strip().lower() for project in route.projects if isinstance(project, str) and project.strip() } - for name in state.services: - project = _project_name_from_service_name(name) + for name, service in state.services.items(): + project = service_project_name(service) or _project_name_from_service_name(name) if project and project.lower() in project_set: selected.add(name) - if selected: - return selected + return selected return set(state.services.keys()) def restart_target_projects(*, state: RunState, route: Route, runtime: StartupRuntime) -> set[str]: targets: set[str] = set() + known_projects = {name.casefold(): name for name in state_project_names(runtime=runtime, state=state)} for value in route.projects: if isinstance(value, str) and value.strip(): - targets.add(value.strip()) + normalized = value.strip() + targets.add(known_projects.get(normalized.casefold(), normalized)) if targets: return targets services = route.flags.get("services") if isinstance(services, list): - for service_name in services: - if not isinstance(service_name, str): + for selector in services: + if not isinstance(selector, str): continue - matched = False - for name, service in state.services.items(): - if name == service_name or service_matches_selector(service, service_name): - matched = True - project = service_project_name(service) or _project_name_from_service_name(name) - if project: - targets.add(project) - if matched: + observed = set( + _observed_service_types_for_selector( + state=state, + runtime=runtime, + selector=selector, + ) + ) + if observed: + targets.update(observed) continue - project = _project_name_from_service_name(service_name) - if project: - targets.add(project) - if targets: + configured_projects = _configured_service_projects_for_selector( + state=state, + runtime=runtime, + selector=selector, + ) + if len(configured_projects) == 1: + targets.update(configured_projects) return targets - for service_name in state.services: - project = runtime._project_name_from_service(service_name) - if project: - targets.add(project) - return targets + return state_project_names(runtime=runtime, state=state) + + +def _stopped_service_projects_for_selector(state: RunState, selector: str) -> set[str]: + target = str(selector).strip().lower().removeprefix("service:").strip() + if not target: + return set() + metadata = getattr(state, "metadata", {}) + raw_stopped = metadata.get(DASHBOARD_STOPPED_SERVICES_KEY) if isinstance(metadata, Mapping) else None + if not isinstance(raw_stopped, list): + return set() + projects: set[str] = set() + for item in raw_stopped: + if not isinstance(item, Mapping): + continue + project = str(item.get("project", "") or "").strip() + name = str(item.get("name", "") or "").strip() + service_type = str(item.get("type", "") or "").strip().lower() + candidates = {name.lower(), service_type, service_display_name(service_type).lower()} - {""} + if project and target in candidates: + projects.add(project) + return projects + + +def _stopped_service_entries(state: RunState) -> list[tuple[str, str, str]]: + metadata = getattr(state, "metadata", {}) + raw_stopped = metadata.get(DASHBOARD_STOPPED_SERVICES_KEY) if isinstance(metadata, Mapping) else None + if not isinstance(raw_stopped, list): + return [] + entries: list[tuple[str, str, str]] = [] + for item in raw_stopped: + if not isinstance(item, Mapping): + continue + project = str(item.get("project", "") or "").strip() + normalized_types = normalize_dashboard_service_types([item.get("type", "")]) + service_type = normalized_types[0] if normalized_types else "" + name = str(item.get("name", "") or "").strip() + if project and service_type: + entries.append((project, service_type, name)) + return entries + + +def _additional_service_enabled_for_project( + service: object, + *, + state: RunState, + project_name: str, +) -> bool: + mode = str(getattr(state, "mode", "") or "main").strip().lower() + metadata = getattr(state, "metadata", {}) + roots = metadata.get("project_roots") if isinstance(metadata, Mapping) else None + project_root = None + if isinstance(roots, Mapping): + for raw_project, raw_root in roots.items(): + if str(raw_project).strip().casefold() == project_name.casefold(): + project_root = Path(str(raw_root)) + break + enabled_for_project = getattr(service, "enabled_for_project_root", None) + if callable(enabled_for_project) and project_root is not None: + return bool(enabled_for_project(mode, project_root)) + enabled_for_mode = getattr(service, "enabled_for_mode", None) + if callable(enabled_for_mode): + return bool(enabled_for_mode(mode)) + return True + + +def configured_service_types_for_project( + *, + state: RunState, + runtime: StartupRuntime, + project_name: str, +) -> set[str]: + metadata = getattr(state, "metadata", {}) + configured_by_project = ( + dashboard_project_configured_services_from_metadata(metadata) + if isinstance(metadata, Mapping) + else {} + ) + if isinstance(metadata, Mapping) and DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY in metadata: + snapshot_types: set[str] = set() + for project, service_types in configured_by_project.items(): + if project.casefold() == project_name.casefold(): + snapshot_types = set(service_types) + break + service_enabled = getattr(runtime, "_service_enabled_for_mode", None) + mode = str(getattr(state, "mode", "") or "main").strip().lower() + if callable(service_enabled): + snapshot_types = { + service_type + for service_type in snapshot_types + if service_type not in {"backend", "frontend"} + or bool(service_enabled(mode, service_type)) + } + config = getattr(runtime, "config", None) + if config is not None and hasattr(config, "additional_services"): + enabled_additional = { + str(getattr(service, "name", "") or "").strip().lower() + for service in tuple(getattr(config, "additional_services", ()) or ()) + if str(getattr(service, "name", "") or "").strip() + and _additional_service_enabled_for_project( + service, + state=state, + project_name=project_name, + ) + } + snapshot_types = { + service_type + for service_type in snapshot_types + if service_type in {"backend", "frontend"} or service_type in enabled_additional + } + return snapshot_types + + configured: set[str] = set() + service_enabled = getattr(runtime, "_service_enabled_for_mode", None) + mode = str(getattr(state, "mode", "") or "main").strip().lower() + for service_type in ("backend", "frontend"): + if not callable(service_enabled) or bool(service_enabled(mode, service_type)): + configured.add(service_type) + for service in tuple(getattr(getattr(runtime, "config", None), "additional_services", ()) or ()): + service_type = str(getattr(service, "name", "") or "").strip().lower() + if service_type and _additional_service_enabled_for_project( + service, + state=state, + project_name=project_name, + ): + configured.add(service_type) + for service_name, service in getattr(state, "services", {}).items(): + project = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) + service_type = _service_type_from_record_name(service_name, service) + if project and project.casefold() == project_name.casefold() and service_type: + configured.add(service_type) + for project, service_type, _service_name in _stopped_service_entries(state): + if project.casefold() == project_name.casefold(): + configured.add(service_type) + return configured + + +def _selector_key(selector: str) -> str: + return str(selector).strip().casefold().removeprefix("service:").strip() + + +def _selector_matches_service_type( + selector: str, + *, + project_name: str, + service_type: str, + service_name: str = "", +) -> bool: + target = _selector_key(selector) + if not target: + return False + display = service_display_name(service_type) + candidates = { + str(service_name).strip().casefold(), + service_type.casefold(), + display.casefold(), + f"{project_name} {display}".strip().casefold(), + } + candidates.discard("") + return target in candidates + + +def _observed_service_types_for_selector( + *, + state: RunState, + runtime: StartupRuntime, + selector: str, +) -> dict[str, set[str]]: + matches: dict[str, set[str]] = {} + for service_name, service in getattr(state, "services", {}).items(): + project = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) + service_type = _service_type_from_record_name(service_name, service) + if not project or not service_type: + continue + configured = configured_service_types_for_project( + state=state, + runtime=runtime, + project_name=project, + ) + if service_type not in configured: + continue + if service_matches_selector(service, selector) or _selector_matches_service_type( + selector, + project_name=project, + service_type=service_type, + service_name=service_name, + ): + matches.setdefault(project, set()).add(service_type) + for project, service_type, service_name in _stopped_service_entries(state): + configured = configured_service_types_for_project( + state=state, + runtime=runtime, + project_name=project, + ) + if service_type in configured and _selector_matches_service_type( + selector, + project_name=project, + service_type=service_type, + service_name=service_name, + ): + matches.setdefault(project, set()).add(service_type) + return matches + + +def _configured_service_projects_for_selector( + *, + state: RunState, + runtime: StartupRuntime, + selector: str, +) -> dict[str, set[str]]: + matches: dict[str, set[str]] = {} + for project in sorted(state_project_names(runtime=runtime, state=state), key=str.casefold): + for service_type in configured_service_types_for_project( + state=state, + runtime=runtime, + project_name=project, + ): + if _selector_matches_service_type( + selector, + project_name=project, + service_type=service_type, + ): + matches.setdefault(project, set()).add(service_type) + return matches + + +def _configured_service_target_projects( + *, + state: RunState, + runtime: StartupRuntime, + selector: str, +) -> set[str]: + matching_projects = set( + _configured_service_projects_for_selector( + state=state, + runtime=runtime, + selector=selector, + ) + ) + return matching_projects if len(matching_projects) == 1 else set() def restart_target_projects_for_selected_services( @@ -330,16 +640,17 @@ def restart_target_projects_for_selected_services( ) -> set[str]: targets: set[str] = set() for service_name in selected_services: - project = runtime._project_name_from_service(service_name) + service = state.services.get(service_name) + project = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) if project: targets.add(project) if targets: return targets - for service_name in state.services: - project = runtime._project_name_from_service(service_name) - if project: - targets.add(project) - return targets + return state_project_names(runtime=runtime, state=state) def _project_name_from_service_name(name: str) -> str | None: @@ -354,6 +665,17 @@ def _project_name_from_service_name(name: str) -> str | None: return None +def _service_type_from_record_name(name: str, service: object) -> str: + service_type = service_slug_from_record(service) + if service_type: + return service_type + lowered = str(name).strip().lower() + if lowered.endswith(" backend"): + return "backend" + if lowered.endswith(" frontend"): + return "frontend" + return "" + def _service_filter_types_for_project( *, route: Route, project_name: str, configured_service_types: set[str] @@ -415,6 +737,105 @@ def _expand_service_dependencies( return expanded +def restart_service_types_by_project( + *, + state: RunState, + route: Route, + runtime: StartupRuntime, + target_projects: set[str], +) -> dict[str, set[str]]: + additional_services = tuple( + getattr(getattr(runtime, "config", None), "additional_services", ()) or () + ) + active_types: dict[str, set[str]] = {} + for service_name, service in getattr(state, "services", {}).items(): + project = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) + service_type = _service_type_from_record_name(service_name, service) + if project and service_type: + active_types.setdefault(project.casefold(), set()).add(service_type) + + selectors = route.flags.get("services") + explicit_types = route.flags.get("restart_service_types") + selected_by_project: dict[str, set[str]] = {} + for project in sorted(target_projects, key=str.casefold): + configured = configured_service_types_for_project( + state=state, + runtime=runtime, + project_name=project, + ) + selected: set[str] + if isinstance(selectors, list): + selected = set() + for selector in selectors: + if not isinstance(selector, str) or not selector.strip(): + continue + observed = _observed_service_types_for_selector( + state=state, + runtime=runtime, + selector=selector, + ) + configured_matches = _configured_service_projects_for_selector( + state=state, + runtime=runtime, + selector=selector, + ) + for candidate_project, service_types in (*observed.items(), *configured_matches.items()): + if candidate_project.casefold() == project.casefold(): + selected.update(service_types) + selected = _expand_service_dependencies( + selected, + route=route, + project_name=project, + additional_services=additional_services, + ) + elif isinstance(explicit_types, list): + selected = { + str(service_type).strip().lower() + for service_type in explicit_types + if str(service_type).strip() + } + else: + selected = set(active_types.get(project.casefold(), set())) + runtime_scope = str(route.flags.get("runtime_scope") or "").strip().lower() + if runtime_scope in {"backend", "frontend"}: + selected.intersection_update({runtime_scope}) + elif runtime_scope == "fullstack": + selected.intersection_update({"backend", "frontend"}) + elif runtime_scope == "dependencies": + selected.clear() + selected.intersection_update(configured) + selected_by_project[project] = _apply_startup_service_launch_flags(selected, route=route) + return selected_by_project + + +def restart_selected_services_for_type_map( + *, + state: RunState, + runtime: StartupRuntime, + service_types_by_project: Mapping[str, set[str]], +) -> set[str]: + selected: set[str] = set() + type_map = { + str(project).strip().casefold(): {str(value).strip().lower() for value in service_types} + for project, service_types in service_types_by_project.items() + if str(project).strip() + } + for service_name, service in getattr(state, "services", {}).items(): + project = resolve_service_project_name( + service_name, + service, + project_name_from_service=runtime._project_name_from_service, + ) + service_type = _service_type_from_record_name(service_name, service) + if project and service_type in type_map.get(project.casefold(), set()): + selected.add(service_name) + return selected + + def _restart_service_types_for_project( *, route: Route | None, @@ -426,6 +847,24 @@ def _restart_service_types_for_project( if route is None: return configured_service_types + type_map = route.flags.get("_restart_service_types_by_project") + if isinstance(type_map, Mapping): + selected: set[str] = set() + for raw_project, raw_types in type_map.items(): + if str(raw_project).strip().casefold() != project_name.casefold(): + continue + if isinstance(raw_types, (list, tuple, set, frozenset)): + selected.update( + str(service_type).strip().lower() + for service_type in raw_types + if str(service_type).strip() + ) + break + return _apply_startup_service_launch_flags( + selected.intersection(configured_service_types), + route=route, + ) + services_value = route.flags.get("services") if isinstance(services_value, list) and services_value: service_types = _service_filter_types_for_project( @@ -468,10 +907,29 @@ def _restart_service_types_for_project( normalized.intersection(configured_service_types or normalized), route=route, ) + selected_services = route.flags.get("_restart_selected_services") + if isinstance(selected_services, list): + selected_names = {str(value).strip().casefold() for value in selected_services if str(value).strip()} + selected_types = { + service_type + for service_type in configured_service_types + if f"{project_name} {service_display_name(service_type)}".casefold() in selected_names + } + return _apply_startup_service_launch_flags(selected_types, route=route) return _apply_startup_service_launch_flags(configured_service_types, route=route) + def _apply_startup_service_launch_flags(service_types: set[str], *, route: Route) -> set[str]: + if route.flags.get("launch_backend") is False and route.flags.get("launch_frontend") is False: + return set() selected = set(service_types) + if "launch_backend" in route.flags or "launch_frontend" in route.flags: + explicitly_selected: set[str] = set() + if route.flags.get("launch_backend") is True: + explicitly_selected.add("backend") + if route.flags.get("launch_frontend") is True: + explicitly_selected.add("frontend") + return selected.intersection(explicitly_selected) if route.flags.get("launch_backend") is False: selected.discard("backend") if route.flags.get("launch_frontend") is False: diff --git a/python/envctl_engine/state/__init__.py b/python/envctl_engine/state/__init__.py index 37b34ba27..6120f032e 100644 --- a/python/envctl_engine/state/__init__.py +++ b/python/envctl_engine/state/__init__.py @@ -3,6 +3,7 @@ import json import re import shlex +from dataclasses import replace from pathlib import Path from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord @@ -30,6 +31,11 @@ def load_state(path: str, *, allowed_root: str | None = None) -> RunState: pid=svc.get("pid"), requested_port=svc.get("requested_port"), actual_port=svc.get("actual_port"), + port_lock_session=( + svc["port_lock_session"].strip() + if isinstance(svc.get("port_lock_session"), str) and svc["port_lock_session"].strip() + else None + ), log_path=svc.get("log_path"), status=svc.get("status", "unknown"), synthetic=bool(svc.get("synthetic", False)), @@ -43,6 +49,14 @@ def load_state(path: str, *, allowed_root: str | None = None) -> RunState: failure_detail=str(svc.get("failure_detail") or "") or None, critical=parse_bool(svc.get("critical"), True), degraded=parse_bool(svc.get("degraded"), False), + runtime_kind=str(svc.get("runtime_kind") or "process"), + container_id=str(svc.get("container_id") or "") or None, + container_name=str(svc.get("container_name") or "") or None, + container_image=str(svc.get("container_image") or "") or None, + container_launch_token=str(svc.get("container_launch_token") or "") or None, + container_cleanup_pending_since=parse_float_or_none( + svc.get("container_cleanup_pending_since") + ), ) for name, svc in data.get("services", {}).items() } @@ -58,7 +72,7 @@ def load_state(path: str, *, allowed_root: str | None = None) -> RunState: "n8n": payload.get("n8n", {}), } requirements[name] = RequirementsResult( - project=name, + project=str(payload.get("project") or name), components=components, health=str(payload.get("health", "unknown") or "unknown"), failures=list(payload.get("failures", [])), @@ -131,8 +145,36 @@ def merge_states(states: list[RunState]) -> RunState: merged_pointers: dict[str, str] = {} merged_metadata: dict[str, object] = {} for state in states: - merged_services.update(state.services) - merged_requirements.update(state.requirements) + for storage_name, service in state.services.items(): + existing = merged_services.get(storage_name) + existing_project = str(getattr(existing, "project", "") or "").strip().casefold() + incoming_project = str(getattr(service, "project", "") or "").strip().casefold() + if existing is not None and existing_project and incoming_project and existing_project != incoming_project: + base = f"{storage_name} State Collision {service.project}" + collision_name = base + index = 2 + occupied = set(merged_services).union(state.services) + while collision_name in occupied: + collision_name = f"{base} {index}" + index += 1 + merged_services[collision_name] = replace(service, name=collision_name) + else: + merged_services[storage_name] = service + for storage_key, requirements in state.requirements.items(): + existing = merged_requirements.get(storage_key) + existing_project = str(getattr(existing, "project", "") or "").strip().casefold() + incoming_project = str(requirements.project or "").strip().casefold() + if existing is not None and existing_project and incoming_project and existing_project != incoming_project: + base = f"{storage_key} State Collision {requirements.project}" + collision_key = base + index = 2 + occupied = set(merged_requirements).union(state.requirements) + while collision_key in occupied: + collision_key = f"{base} {index}" + index += 1 + merged_requirements[collision_key] = requirements + else: + merged_requirements[storage_key] = requirements merged_pointers.update(state.pointers) merged_metadata.update(state.metadata) latest = states[-1] @@ -169,6 +211,7 @@ def state_to_dict(state: RunState) -> dict[str, object]: "pid": svc.pid, "requested_port": svc.requested_port, "actual_port": svc.actual_port, + "port_lock_session": svc.port_lock_session, "log_path": svc.log_path, "status": svc.status, "synthetic": svc.synthetic, @@ -182,11 +225,18 @@ def state_to_dict(state: RunState) -> dict[str, object]: "failure_detail": svc.failure_detail, "critical": svc.critical, "degraded": svc.degraded, + "runtime_kind": svc.runtime_kind, + "container_id": svc.container_id, + "container_name": svc.container_name, + "container_image": svc.container_image, + "container_launch_token": svc.container_launch_token, + "container_cleanup_pending_since": svc.container_cleanup_pending_since, } for name, svc in state.services.items() }, "requirements": { project: { + "project": req.project, "components": {dependency_id: req.component(dependency_id) for dependency_id in dependency_ids()}, "db": req.db, "redis": req.redis, diff --git a/python/envctl_engine/state/action_command_support.py b/python/envctl_engine/state/action_command_support.py index c5e0708d4..690f28570 100644 --- a/python/envctl_engine/state/action_command_support.py +++ b/python/envctl_engine/state/action_command_support.py @@ -2,10 +2,17 @@ from dataclasses import dataclass, field from typing import Any, cast +from contextlib import contextmanager import json +import signal +import threading from envctl_engine.runtime.browser_diagnostics import build_runtime_diagnostics from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.docker_service_runtime import ( + refresh_container_log_snapshots, + stop_container_log_followers, +) from envctl_engine.shared.parsing import parse_float_or_none, parse_int from envctl_engine.state.action_log_support import DEFAULT_LOG_ISSUE_LIMIT from envctl_engine.state.models import RunState @@ -25,6 +32,26 @@ class StateActionReconciliation: reconciled: bool = False +@contextmanager +def _interrupt_log_follow_on_termination(enabled: bool): + if not enabled or threading.current_thread() is not threading.main_thread(): + yield + return + previous: dict[signal.Signals, Any] = {} + + def interrupt(_signum: int, _frame: object) -> None: + raise KeyboardInterrupt + + for signal_name in (signal.SIGHUP, signal.SIGTERM): + previous[signal_name] = signal.getsignal(signal_name) + signal.signal(signal_name, interrupt) + try: + yield + finally: + for signal_name, handler in previous.items(): + signal.signal(signal_name, handler) + + class StateActionCommandRunner: def __init__(self, orchestrator: Any) -> None: self.orchestrator = orchestrator @@ -200,6 +227,12 @@ def _run_logs(self, route: Route) -> int: if duration_seconds is not None and duration_seconds < 0: duration_seconds = 0.0 no_color = bool(route.flags.get("logs_no_color")) or self.orchestrator._human_output_no_color(route) + follower_pids = refresh_container_log_snapshots( + self.runtime.raw_runtime, + self.state, + tail=max(tail, 0), + follow=follow and not bool(route.flags.get("json")), + ) if bool(route.flags.get("json")): print( json.dumps( @@ -215,13 +248,17 @@ def _run_logs(self, route: Route) -> int: ) ) return self._finish_action(0) - self.runtime.print_logs( - self.state, - tail=max(tail, 0), - follow=follow, - duration_seconds=duration_seconds, - no_color=no_color, - ) + try: + with _interrupt_log_follow_on_termination(bool(follower_pids)): + self.runtime.print_logs( + self.state, + tail=max(tail, 0), + follow=follow, + duration_seconds=duration_seconds, + no_color=no_color, + ) + finally: + stop_container_log_followers(self.runtime.raw_runtime, follower_pids) return self._finish_action(0) def _run_clear_logs(self, route: Route) -> int: @@ -450,6 +487,12 @@ def _run_errors(self, route: Route) -> int: parse_int(str(route.flags.get("logs_tail", DEFAULT_LOG_ISSUE_LIMIT)), DEFAULT_LOG_ISSUE_LIMIT), 0, ) + refresh_container_log_snapshots( + self.runtime.raw_runtime, + self.state, + tail=log_issue_limit, + follow=False, + ) log_issues = self.orchestrator._service_log_issues(self.state, max_matches=log_issue_limit) has_errors = bool(failed or reconciliation.requirement_issues or reconciliation.recent_failures or log_issues) if bool(route.flags.get("json")): diff --git a/python/envctl_engine/state/action_health_support.py b/python/envctl_engine/state/action_health_support.py index d61667e3f..d5048cc1a 100644 --- a/python/envctl_engine/state/action_health_support.py +++ b/python/envctl_engine/state/action_health_support.py @@ -155,7 +155,8 @@ def requirement_health_rows(self, state: RunState) -> list[dict[str, object]]: rows: list[dict[str, object]] = [] definitions = tuple(self._dependency_definitions()) component_order = {definition.id: index for index, definition in enumerate(definitions)} - for project, requirements in state.requirements.items(): + for storage_key, requirements in state.requirements.items(): + project = str(getattr(requirements, "project", "") or storage_key).strip() for definition in definitions: component = definition.id data = requirements.component(component) @@ -172,6 +173,7 @@ def requirement_health_rows(self, state: RunState) -> list[dict[str, object]]: rows.append( { "project": project, + "storage_key": storage_key, "component": component, "status": status, "port": dependency_display_port(component, data), @@ -205,6 +207,16 @@ def _service_row(service: ServiceRecord) -> dict[str, object]: "failure_detail": getattr(service, "failure_detail", None), "critical": getattr(service, "critical", True), "degraded": getattr(service, "degraded", False), + "runtime_kind": getattr(service, "runtime_kind", "process"), + "container_id": getattr(service, "container_id", None), + "container_name": getattr(service, "container_name", None), + "container_image": getattr(service, "container_image", None), + "container_launch_token": getattr(service, "container_launch_token", None), + "container_cleanup_pending_since": getattr( + service, + "container_cleanup_pending_since", + None, + ), } rows = self._parallel_service_map( diff --git a/python/envctl_engine/state/action_log_support.py b/python/envctl_engine/state/action_log_support.py index 851e6646e..64d8f6fc7 100644 --- a/python/envctl_engine/state/action_log_support.py +++ b/python/envctl_engine/state/action_log_support.py @@ -204,7 +204,34 @@ def clear_one(service: ServiceRecord) -> tuple[int, int, int, int, str | None]: if not raw_path: return 0, 0, 1, 0, (None if quiet else f"{service.name}: log=n/a") log_path = Path(raw_path) + is_docker = str(getattr(service, "runtime_kind", "process") or "process").lower() == "docker" if not log_path.is_file(): + if is_docker: + try: + Path(f"{log_path}.docker-since").touch() + line = f"{service.name}: Docker log cursor advanced at {log_path}" + return 1, 0, 0, 0, ( + None + if quiet + else render_paths_in_terminal_text( + line, + paths=[log_path], + env=env, + interactive_tty=interactive_tty, + ) + ) + except OSError as exc: + line = f"{service.name}: failed to clear Docker logs at {log_path} ({exc})" + return 0, 0, 0, 1, ( + None + if quiet + else render_paths_in_terminal_text( + line, + paths=[log_path], + env=env, + interactive_tty=interactive_tty, + ) + ) line = f"{service.name}: log missing at {log_path}" return 0, 1, 0, 0, ( None @@ -219,6 +246,8 @@ def clear_one(service: ServiceRecord) -> tuple[int, int, int, int, str | None]: try: with log_path.open("w", encoding="utf-8"): pass + if is_docker: + Path(f"{log_path}.docker-since").touch() line = f"{service.name}: log cleared at {log_path}" return 1, 0, 0, 0, ( None diff --git a/python/envctl_engine/state/action_orchestrator.py b/python/envctl_engine/state/action_orchestrator.py index a0989e80c..972c6ab6c 100644 --- a/python/envctl_engine/state/action_orchestrator.py +++ b/python/envctl_engine/state/action_orchestrator.py @@ -20,7 +20,7 @@ services_from_selection, ) from envctl_engine.ui.selection_types import TargetSelection -from envctl_engine.shared.services import service_matches_selector +from envctl_engine.shared.services import resolve_service_project_name, service_matches_selector class StateActionRuntimeFacade: @@ -216,24 +216,35 @@ def _resolve_selected_services(self, route: Route, state: RunState) -> set[str] services_flag = route.flags.get("services") project_filters = {name.lower() for name in route.projects} project_filters.update(self.runtime.selectors_from_passthrough(route.passthrough_args)) - selected: set[str] = set() + service_selected: set[str] | None = None + project_selected: set[str] | None = None if isinstance(services_flag, list): + service_selected = set() for raw in services_flag: target = str(raw).strip() if not target: continue for name, service in state.services.items(): if service_matches_selector(service, target): - selected.add(name) + service_selected.add(name) if project_filters: - for name in state.services: - project = self.runtime.project_name_from_service(name).lower() + project_selected = set() + for name, service in state.services.items(): + project = resolve_service_project_name( + name, + service, + project_name_from_service=self.runtime.project_name_from_service, + ).lower() if project and project in project_filters: - selected.add(name) - - if selected: - return selected + project_selected.add(name) + + if service_selected is not None and project_selected is not None: + return service_selected.intersection(project_selected) + if service_selected is not None: + return service_selected + if project_selected is not None: + return project_selected if project_filters or (isinstance(services_flag, list) and services_flag): return set() return None @@ -248,8 +259,20 @@ def _filtered_state(self, state: RunState, selected_services: set[str]) -> RunSt if not selected_services: return RunState(run_id=state.run_id, mode=state.mode) services = {name: svc for name, svc in state.services.items() if name in selected_services} - project_names = {self.runtime.project_name_from_service(name) for name in services} - requirements = {project: req for project, req in state.requirements.items() if project in project_names} + project_names = { + resolve_service_project_name( + name, + service, + project_name_from_service=self.runtime.project_name_from_service, + ) + for name, service in services.items() + } + project_keys = {str(project).strip().casefold() for project in project_names if str(project).strip()} + requirements = { + storage_key: req + for storage_key, req in state.requirements.items() + if str(getattr(req, "project", "") or storage_key).strip().casefold() in project_keys + } return RunState( run_id=state.run_id, mode=state.mode, diff --git a/python/envctl_engine/state/fingerprints.py b/python/envctl_engine/state/fingerprints.py new file mode 100644 index 000000000..30087f574 --- /dev/null +++ b/python/envctl_engine/state/fingerprints.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +from envctl_engine.state import state_to_dict +from envctl_engine.state.models import RunState + + +def text_fingerprint(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def file_fingerprint(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def state_fingerprint(state: RunState) -> str: + return text_fingerprint(json.dumps(state_to_dict(state), sort_keys=True)) diff --git a/python/envctl_engine/state/lookup.py b/python/envctl_engine/state/lookup.py new file mode 100644 index 000000000..ce5580bb1 --- /dev/null +++ b/python/envctl_engine/state/lookup.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import inspect +from collections.abc import Callable, Sequence +from typing import Any + + +def call_state_loader( + loader: Callable[..., Any], + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, +) -> Any: + """Call current and legacy state-loader adapters without injecting unsupported keywords.""" + + accepts_any_keyword, accepted_keywords = _accepted_keywords(loader) + options: dict[str, object] = {} + for keyword, value in ( + ("mode", mode), + ("strict_mode_match", strict_mode_match), + ): + if accepts_any_keyword or keyword in accepted_keywords: + options[keyword] = value + if project_names is not None: + if not accepts_any_keyword and "project_names" not in accepted_keywords: + raise TypeError("Targeted state lookup requires a loader that accepts project_names") + options["project_names"] = project_names + return loader(**options) + + +def _accepted_keywords(loader: Callable[..., Any]) -> tuple[bool, frozenset[str]]: + try: + parameters = tuple(inspect.signature(loader).parameters.values()) + except (TypeError, ValueError): + return True, frozenset() + return ( + any(parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters), + frozenset( + parameter.name + for parameter in parameters + if parameter.kind in {inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY} + ), + ) + + +__all__ = ["call_state_loader"] diff --git a/python/envctl_engine/state/models.py b/python/envctl_engine/state/models.py index cfcab4405..c581dae51 100644 --- a/python/envctl_engine/state/models.py +++ b/python/envctl_engine/state/models.py @@ -24,6 +24,7 @@ class ServiceRecord: pid: int | None = None requested_port: int | None = None actual_port: int | None = None + port_lock_session: str | None = None log_path: str | None = None status: str = "unknown" synthetic: bool = False @@ -37,6 +38,12 @@ class ServiceRecord: failure_detail: str | None = None critical: bool = True degraded: bool = False + runtime_kind: str = "process" + container_id: str | None = None + container_name: str | None = None + container_image: str | None = None + container_launch_token: str | None = None + container_cleanup_pending_since: float | None = None @dataclass(slots=True) diff --git a/python/envctl_engine/state/persistence.py b/python/envctl_engine/state/persistence.py new file mode 100644 index 000000000..6d8e29b64 --- /dev/null +++ b/python/envctl_engine/state/persistence.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import fcntl +import errno +import os +import re +import stat +import tempfile +import time +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + + +_ATOMIC_TEMP_TOKEN = re.compile(r"[a-z0-9_]{8}") + + +def atomic_write_text(path: Path, text: str) -> None: + """Replace a UTF-8 text file without exposing a partially written value.""" + + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + prefix=f".{path.name}.", + suffix=".tmp", + dir=path.parent, + delete=False, + ) as temporary_file: + temporary_path = Path(temporary_file.name) + temporary_file.write(text) + temporary_file.flush() + os.fsync(temporary_file.fileno()) + os.replace(temporary_path, path) + fsync_directory(path.parent) + except BaseException: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + raise + + +def scavenge_atomic_write_temps(path: Path) -> int: + """Remove crash leftovers for one exact atomic-write target. + + The caller must already hold the exclusive lock that serializes writes to + ``path``. Scanning is deliberately target-local: retired runtime history + can be unbounded, and unrelated ``*.tmp`` files must never be touched. + Directory and entry symlinks are not followed. + """ + + parent = path.parent + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + directory_fd = os.open(parent, flags) + removed = 0 + expected_prefix = f".{path.name}." + try: + if not stat.S_ISDIR(os.fstat(directory_fd).st_mode): + raise OSError(f"Atomic-write parent is not a real directory: {parent}") + with os.scandir(directory_fd) as entries: + for entry in entries: + name = entry.name + if not name.startswith(expected_prefix) or not name.endswith(".tmp"): + continue + token = name[len(expected_prefix) : -len(".tmp")] + if _ATOMIC_TEMP_TOKEN.fullmatch(token) is None: + continue + try: + metadata = entry.stat(follow_symlinks=False) + except FileNotFoundError: + continue + if not stat.S_ISREG(metadata.st_mode): + continue + try: + os.unlink(name, dir_fd=directory_fd) + except FileNotFoundError: + continue + removed += 1 + if removed: + fsync_directory(parent) + return removed + finally: + os.close(directory_fd) + + +def durable_mkdir(path: Path) -> None: + """Create a directory chain and durably publish every new directory entry.""" + + missing: list[Path] = [] + cursor = path + while not cursor.exists(): + missing.append(cursor) + if cursor.parent == cursor: + break + cursor = cursor.parent + if cursor.is_symlink() or not cursor.is_dir(): + raise OSError(f"Directory ancestor is not a real directory: {cursor}") + for directory in reversed(missing): + try: + directory.mkdir() + except FileExistsError: + if directory.is_symlink() or not directory.is_dir(): + raise + if directory.is_symlink() or not directory.is_dir(): + raise OSError(f"Created path is not a real directory: {directory}") + fsync_directory(directory) + fsync_directory(directory.parent) + + +def require_path_component(value: object, *, label: str) -> str: + """Return a single portable path component or reject ambiguous traversal input.""" + + raw = str(value) + if ( + not raw + or raw != raw.strip() + or raw in {".", ".."} + or "/" in raw + or "\\" in raw + or "\0" in raw + or Path(raw).is_absolute() + ): + raise ValueError(f"{label} must be a non-empty path component") + return raw + + +def fsync_directory(path: Path) -> None: + """Durably publish directory entry changes without following a replaced symlink.""" + + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + directory_fd = os.open(path, flags) + try: + if not stat.S_ISDIR(os.fstat(directory_fd).st_mode): + raise OSError(f"Path is not a real directory: {path}") + try: + os.fsync(directory_fd) + except OSError as exc: + if exc.errno not in {errno.EBADF, errno.EINVAL, errno.ENOTSUP, errno.EROFS}: + raise + finally: + os.close(directory_fd) + + +@contextmanager +def advisory_file_lock( + path: Path, + *, + exclusive: bool, + timeout_seconds: float | None = None, +) -> Iterator[None]: + """Hold a process-safe advisory lock backed by a stable lock-file inode.""" + + path.parent.mkdir(parents=True, exist_ok=True) + flags = os.O_CREAT | os.O_RDWR + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags, 0o600) + try: + lock_file = os.fdopen(descriptor, "a+b") + except BaseException: + os.close(descriptor) + raise + with lock_file: + if not stat.S_ISREG(os.fstat(lock_file.fileno()).st_mode): + raise OSError(f"Lock path is not a regular file: {path}") + operation = fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH + if timeout_seconds is None: + fcntl.flock(lock_file.fileno(), operation) + else: + deadline = time.monotonic() + max(0.0, timeout_seconds) + while True: + try: + fcntl.flock(lock_file.fileno(), operation | fcntl.LOCK_NB) + break + except BlockingIOError as exc: + if time.monotonic() >= deadline: + raise TimeoutError(f"Timed out acquiring lock: {path}") from exc + time.sleep(0.01) + try: + yield + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + +__all__ = [ + "advisory_file_lock", + "atomic_write_text", + "durable_mkdir", + "fsync_directory", + "require_path_component", + "scavenge_atomic_write_temps", +] diff --git a/python/envctl_engine/state/project_runtime.py b/python/envctl_engine/state/project_runtime.py index c3a69c4f5..1e89ec815 100644 --- a/python/envctl_engine/state/project_runtime.py +++ b/python/envctl_engine/state/project_runtime.py @@ -1,13 +1,34 @@ from __future__ import annotations +import json from dataclasses import dataclass from pathlib import Path from typing import Any, Mapping +from envctl_engine.dashboard_metadata import ( + DASHBOARD_CONFIGURED_SERVICE_TYPES_KEY, + DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY, + DASHBOARD_STOPPED_SERVICES_KEY, + normalize_dashboard_service_types, +) from envctl_engine.shared.services import service_project_name +from envctl_engine.state.fingerprints import text_fingerprint from envctl_engine.state.models import RequirementsResult, RunState +PROJECT_SCOPED_METADATA_MAP_KEYS = ( + "project_roots", + "project_pr_links", + "project_test_summaries", + "project_action_reports", + DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY, +) +_PROJECT_TEST_METADATA_COMPANION_KEYS = ( + "project_test_results_root", + "project_test_results_updated_at", +) + + @dataclass(slots=True) class ProjectRuntimeResolution: ok: bool @@ -59,10 +80,54 @@ def payload(self) -> dict[str, object]: } +def requirement_keys_for_project(state: RunState, project: str) -> list[str]: + target = str(project).strip().casefold() + if not target: + return [] + exact = [key for key in state.requirements if str(key).strip().casefold() == target] + owner_aliases = [ + key + for key, requirements in state.requirements.items() + if key not in exact + and str(getattr(requirements, "project", "") or "").strip().casefold() == target + ] + return [*exact, *owner_aliases] + + +def requirement_key_for_project(state: RunState, project: str) -> str | None: + target = str(project).strip().casefold() + if not target: + return None + exact = [key for key in state.requirements if str(key).strip().casefold() == target] + if len(exact) == 1: + return exact[0] + if len(exact) > 1: + raise RuntimeError( + f"Ambiguous requirements authority for {project}: " + ", ".join(sorted(exact)) + ) + owner_aliases = [ + key + for key, requirements in state.requirements.items() + if str(getattr(requirements, "project", "") or "").strip().casefold() == target + ] + if len(owner_aliases) == 1: + return owner_aliases[0] + if len(owner_aliases) > 1: + raise RuntimeError( + f"Ambiguous requirements authority for {project}: " + ", ".join(sorted(owner_aliases)) + ) + return None + + +def requirements_for_project(state: RunState, project: str) -> RequirementsResult | None: + key = requirement_key_for_project(state, project) + return state.requirements.get(key) if key is not None else None + + def active_project_names(state: RunState, *, runtime: Any | None = None) -> list[str]: names: set[str] = set() - for name in getattr(state, "requirements", {}) or {}: - normalized = str(name).strip() + for storage_key, requirements in (getattr(state, "requirements", {}) or {}).items(): + normalized = str(getattr(requirements, "project", "") or storage_key).strip() if normalized: names.add(normalized) for service_name, service in (getattr(state, "services", {}) or {}).items(): @@ -140,6 +205,7 @@ def resolve_requested_project_state( state=filter_state_to_projects(state, selected, runtime=runtime), ) + def _service_project_name(service_name: str, service: object, *, runtime: Any | None = None) -> str: explicit_project = str(getattr(service, "project", "") or "").strip() if explicit_project: @@ -171,10 +237,8 @@ def filter_state_to_projects( if _service_project_name(name, service, runtime=runtime) in selected } requirements = _filtered_requirements(state.requirements, selected) - metadata = dict(state.metadata) - roots = metadata.get("project_roots") - if isinstance(roots, Mapping): - metadata["project_roots"] = {str(name): value for name, value in roots.items() if str(name) in selected} + metadata = filter_project_scoped_metadata(state.metadata, selected) + metadata["project_names"] = sorted(selected, key=lambda value: (value.casefold(), value)) return RunState( run_id=state.run_id, mode=state.mode, @@ -187,11 +251,114 @@ def filter_state_to_projects( ) +def filter_project_scoped_metadata( + metadata: Mapping[str, object], + projects: list[str] | tuple[str, ...] | set[str] | frozenset[str], + *, + case_sensitive: bool = True, +) -> dict[str, object]: + """Return metadata containing only records owned by the selected projects. + + Runtime state has several independently evolved project-keyed metadata + surfaces. Keeping their filtering in one place prevents a selected state + from retaining dashboard, PR, test, or action data for another run. + """ + + selected = {str(project).strip() for project in projects if str(project).strip()} + normalized_selected = selected if case_sensitive else {project.casefold() for project in selected} + + def selected_project(project: object) -> bool: + value = str(project).strip() + if not value: + return False + return value in normalized_selected if case_sensitive else value.casefold() in normalized_selected + + filtered = dict(metadata) + for key in PROJECT_SCOPED_METADATA_MAP_KEYS: + raw = metadata.get(key) + if not isinstance(raw, Mapping): + if key in metadata: + filtered.pop(key, None) + continue + retained = {str(project): value for project, value in raw.items() if selected_project(project)} + if retained: + filtered[key] = retained + else: + filtered.pop(key, None) + + raw_stopped = metadata.get(DASHBOARD_STOPPED_SERVICES_KEY) + if isinstance(raw_stopped, list): + stopped: list[dict[str, object]] = [] + for item in raw_stopped: + if not isinstance(item, Mapping) or not selected_project(item.get("project", "")): + continue + if not str(item.get("name", "")).strip() or not str(item.get("type", "")).strip(): + continue + stopped.append(dict(item)) + if stopped: + filtered[DASHBOARD_STOPPED_SERVICES_KEY] = stopped + else: + filtered.pop(DASHBOARD_STOPPED_SERVICES_KEY, None) + elif DASHBOARD_STOPPED_SERVICES_KEY in metadata: + filtered.pop(DASHBOARD_STOPPED_SERVICES_KEY, None) + + raw_summaries = metadata.get("project_test_summaries") + retained_summaries = filtered.get("project_test_summaries") + summaries_were_narrowed = ( + isinstance(raw_summaries, Mapping) + and isinstance(retained_summaries, Mapping) + and len(retained_summaries) != len(raw_summaries) + ) + if not isinstance(retained_summaries, Mapping) or not retained_summaries or summaries_were_narrowed: + for key in _PROJECT_TEST_METADATA_COMPANION_KEYS: + filtered.pop(key, None) + + if DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY in metadata or DASHBOARD_STOPPED_SERVICES_KEY in metadata: + configured_types: set[str] = set() + configured = filtered.get(DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY) + if isinstance(configured, Mapping): + for service_types in configured.values(): + configured_types.update(normalize_dashboard_service_types(service_types)) + stopped = filtered.get(DASHBOARD_STOPPED_SERVICES_KEY) + if isinstance(stopped, list): + configured_types.update( + normalize_dashboard_service_types( + [item.get("type", "") for item in stopped if isinstance(item, Mapping)] + ) + ) + if configured_types: + filtered[DASHBOARD_CONFIGURED_SERVICE_TYPES_KEY] = sorted(configured_types) + else: + filtered.pop(DASHBOARD_CONFIGURED_SERVICE_TYPES_KEY, None) + + raw_identity = metadata.get("startup_identity") + if isinstance(raw_identity, Mapping): + identity = dict(raw_identity) + raw_projects = identity.get("projects") + identity_projects = raw_projects if isinstance(raw_projects, list) else [] + identity["projects"] = [ + dict(project) + for project in identity_projects + if isinstance(project, Mapping) and selected_project(project.get("name", "")) + ] + identity.pop("fingerprint", None) + serialized = json.dumps(identity, sort_keys=True) + identity["fingerprint"] = text_fingerprint(serialized) + filtered["startup_identity"] = identity + + return filtered + + def _filtered_requirements( requirements: Mapping[str, RequirementsResult], selected: set[str], ) -> dict[str, RequirementsResult]: - direct = {project: req for project, req in requirements.items() if project in selected} + selected_keys = {str(project).strip().casefold() for project in selected if str(project).strip()} + direct = { + storage_key: req + for storage_key, req in requirements.items() + if str(getattr(req, "project", "") or storage_key).strip().casefold() in selected_keys + } if direct: return direct # Shared tree dependency state often stores requirements under Main. When a @@ -281,7 +448,6 @@ def cwd_runtime_warnings( ] - def project_resolution_event_payload( resolution: ProjectRuntimeResolution, state: RunState, @@ -308,8 +474,8 @@ def project_resolution_event_payload( def _runtime_truth_project_names(state: RunState, *, runtime: Any | None = None) -> list[str]: names: set[str] = set() - for name in getattr(state, "requirements", {}) or {}: - normalized = str(name).strip() + for storage_key, requirements in (getattr(state, "requirements", {}) or {}).items(): + normalized = str(getattr(requirements, "project", "") or storage_key).strip() if normalized: names.add(normalized) for service_name, service in (getattr(state, "services", {}) or {}).items(): diff --git a/python/envctl_engine/state/repository.py b/python/envctl_engine/state/repository.py index b750117ba..0c0adbe14 100644 --- a/python/envctl_engine/state/repository.py +++ b/python/envctl_engine/state/repository.py @@ -1,17 +1,39 @@ from __future__ import annotations import json -import hashlib import shutil -from collections.abc import Mapping +import stat +import uuid +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Callable, cast +from envctl_engine.dashboard_metadata import DASHBOARD_STOPPED_SERVICES_KEY from envctl_engine.shared.artifact_names import safe_artifact_stem -from envctl_engine.state.models import RunState -from envctl_engine.state import dump_state, load_legacy_shell_state, load_state, load_state_from_pointer, state_to_dict +from envctl_engine.state import load_legacy_shell_state, load_state, load_state_from_pointer, state_to_dict +from envctl_engine.state.fingerprints import file_fingerprint, state_fingerprint, text_fingerprint +from envctl_engine.state.models import RunState, ServiceRecord +from envctl_engine.state.persistence import ( + advisory_file_lock, + atomic_write_text, + durable_mkdir, + fsync_directory, + require_path_component, + scavenge_atomic_write_temps, +) +from envctl_engine.state.project_runtime import filter_project_scoped_metadata +from envctl_engine.state.run_index import RunIndex, RunIndexCandidate, StateSelector +from envctl_engine.state.runtime_map import build_runtime_map +from envctl_engine.state.state_aggregation import ( + IndexedState as _IndexedState, + filter_state_to_owned_projects, + normalized_project_names, + select_indexed_owners, + state_from_indexed_owners, +) @dataclass(slots=True) @@ -24,6 +46,25 @@ class RuntimeStateRepository: COMPAT_READ_WRITE: str = "compat_read_write" COMPAT_READ_ONLY: str = "compat_read_only" SCOPED_ONLY: str = "scoped_only" + _RUNTIME_ARTIFACT_NAMES = frozenset( + { + "events.jsonl", + "runtime_readiness_report.json", + } + ) + _REVISION_ANCILLARY_ARTIFACT_NAMES = ( + "ports_manifest.json", + "error_report.json", + "events.jsonl", + "runtime_readiness_report.json", + ) + _RUN_ALIAS_ARTIFACT_NAMES = ( + "run_state.json", + "runtime_map.json", + *_REVISION_ANCILLARY_ARTIFACT_NAMES, + ) + _RUN_ALIAS_MANIFEST_NAME = ".revision_alias_manifest.json" + _REVISION_CLEANUP_DIR_NAME = ".revision-cleanup" def __init__( self, @@ -34,9 +75,18 @@ def __init__( runtime_scope_id: str, compat_mode: str, ) -> None: - self.runtime_root: Path = runtime_root - self.runtime_legacy_root: Path = runtime_legacy_root - self.runtime_dir: Path = runtime_dir + input_runtime_root = runtime_root.expanduser().absolute() + input_legacy_root = runtime_legacy_root.expanduser().absolute() + if input_runtime_root.is_symlink() or input_legacy_root.is_symlink(): + raise ValueError("runtime roots must not be symlinks") + self.runtime_dir: Path = runtime_dir.resolve() + self.runtime_root: Path = runtime_root.resolve() + self.runtime_legacy_root: Path = runtime_legacy_root.resolve() + try: + self.runtime_root.relative_to(self.runtime_dir) + self.runtime_legacy_root.relative_to(self.runtime_dir) + except ValueError as exc: + raise ValueError("runtime roots must be contained by runtime_dir") from exc self.runtime_scope_id: str = runtime_scope_id self.compat_mode: str = ( compat_mode @@ -48,6 +98,16 @@ def __init__( } else self.COMPAT_READ_WRITE ) + self.run_index = RunIndex( + runtime_root=self.runtime_root, + runtime_dir=self.runtime_dir, + runtime_scope_id=self.runtime_scope_id, + ) + self._lock_path = self.runtime_root / ".state_repository.lock" + self._registry_marker_path = self.runtime_root / "run_registry.json" + self._alias_manifest_path = self.runtime_root / "current_alias.json" + self._runtime_root_identity = self._existing_directory_identity(self.runtime_root) + self._legacy_root_identity = self._existing_directory_identity(self.runtime_legacy_root) def run_state_path(self) -> Path: return self.runtime_root / "run_state.json" @@ -61,25 +121,36 @@ def ports_manifest_path(self) -> Path: def error_report_path(self) -> Path: return self.runtime_root / "error_report.json" + def lifecycle_operation_lock_path(self) -> Path: + self._validate_runtime_roots() + return self.runtime_root / ".lifecycle_operation.lock" + + def ensure_runtime_roots(self) -> None: + self._validate_runtime_roots() + def run_dir_path(self, run_id: str | None) -> Path: + runs_root = self._contained_child(self.runtime_root, "runs", label="runs directory") if run_id is None: - return self.runtime_root / "runs" - return self.runtime_root / "runs" / run_id + return runs_root + normalized_run_id = require_path_component(run_id, label="run_id") + return self._contained_child(runs_root, normalized_run_id, label="run_id") def test_results_dir_path(self, run_id: str, stamp: str | None = None) -> Path: - root = self.run_dir_path(run_id) / "test-results" + root = self._contained_child(self.run_dir_path(run_id), "test-results", label="test results directory") if not stamp: return root - return root / stamp + normalized_stamp = require_path_component(stamp, label="test result stamp") + return self._contained_child(root, normalized_stamp, label="test result stamp") def tree_diffs_dir_path(self, run_id: str | None = None, name: str | None = None) -> Path: if run_id: - root = self.run_dir_path(run_id) / "tree-diffs" + root = self._contained_child(self.run_dir_path(run_id), "tree-diffs", label="tree diffs directory") else: - root = self.runtime_root / "tree-diffs" + root = self._contained_child(self.runtime_root, "tree-diffs", label="tree diffs directory") if not name: return root - return root / name + normalized_name = require_path_component(name, label="tree diff name") + return self._contained_child(root, normalized_name, label="tree diff name") def save_run( self, @@ -91,70 +162,59 @@ def save_run( emit: Callable[..., None], runtime_map_builder: Callable[[RunState], dict[str, object]], write_runtime_readiness_report: Callable[[Path], None] | None = None, + on_commit: Callable[[], None] | None = None, ) -> StateRepositoryPaths: - run_dir = self.run_dir_path(state.run_id) - run_dir.mkdir(parents=True, exist_ok=True) - - run_state_path = run_dir / "run_state.json" - run_state_text = self._write_run_state(state=state, run_state_path=run_state_path, emit=emit) - - runtime_map_text = self._write_runtime_map_text( - state=state, - runtime_map=runtime_map_builder(state), - runtime_map_path=run_dir / "runtime_map.json", - emit_path=run_dir / "runtime_map.json", - emit=emit, - ) - - ports_manifest = { - "run_id": state.run_id, - "mode": state.mode, - "projects": [self._context_ports_payload(context) for context in contexts], - } - ports_manifest_text = self._json_text(ports_manifest) - self._write_run_and_current_artifact( - run_path=run_dir / "ports_manifest.json", - current_path=self.ports_manifest_path(), - text=ports_manifest_text, + project_names = sorted( + { + *self._context_names(contexts), + *self._runtime_project_names_from_state(state), + }, + key=str.casefold, ) - - error_report = { - "run_id": state.run_id, - "errors": errors, - "generated_at": datetime.now(tz=UTC).isoformat(), + self._prepare_state_for_save(state, project_names=project_names, contexts=contexts) + runtime_map = runtime_map_builder(state) + artifacts = { + "run_state.json": self._json_text(state_to_dict(state)), + "runtime_map.json": self._json_text(runtime_map), + "ports_manifest.json": self._json_text( + { + "run_id": state.run_id, + "mode": state.mode, + "projects": [self._context_ports_payload(context) for context in contexts], + } + ), + "error_report.json": self._json_text( + { + "run_id": state.run_id, + "errors": errors, + "generated_at": datetime.now(tz=UTC).isoformat(), + } + ), + "events.jsonl": self._events_text(events), } - error_report_text = self._json_text(error_report) - self._write_run_and_current_artifact( - run_path=run_dir / "error_report.json", - current_path=self.error_report_path(), - text=error_report_text, - ) - - events_text = self._events_text(events) - self._write_run_and_current_artifact( - run_path=run_dir / "events.jsonl", - current_path=self.runtime_root / "events.jsonl", - text=events_text, - ) - - if write_runtime_readiness_report is not None: - write_runtime_readiness_report(run_dir) - - self._write_scoped_pointers(run_state_path=run_state_path, state=state, contexts=contexts) - - if self.compat_mode == self.COMPAT_READ_WRITE: - self._write_legacy_compat( - run_state_path=run_state_path, - run_state_text=run_state_text, - runtime_map_text=runtime_map_text, - ports_manifest_text=ports_manifest_text, - error_report_text=error_report_text, - events_text=events_text, + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + self._carry_forward_runtime_artifacts_unlocked(state, artifacts) + revision_state_path = self._commit_revision_unlocked( state=state, - contexts=contexts, + project_names=project_names, + artifacts=artifacts, + supersede_run_ids=self._source_run_ids(state), + on_commit=on_commit, ) - - return StateRepositoryPaths(run_dir=run_dir, run_state=run_state_path) + run_dir = self.run_dir_path(state.run_id) + if write_runtime_readiness_report is not None: + try: + write_runtime_readiness_report(run_dir) + except Exception as exc: + self._safe_emit(emit, "state.readiness_report.error", run_id=state.run_id, error=str(exc)) + self._emit_saved_state( + emit, + state=state, + state_path=revision_state_path, + runtime_map_text=artifacts["runtime_map.json"], + ) + return StateRepositoryPaths(run_dir=run_dir, run_state=run_dir / "run_state.json") def save_resume_state( self, @@ -175,65 +235,208 @@ def save_selected_stop_state( state: RunState, emit: Callable[..., None], runtime_map_builder: Callable[[RunState], dict[str, object]], + authoritative_project_names: Sequence[str] | None = None, ) -> dict[str, object]: return self._save_runtime_update( state=state, emit=emit, runtime_map_builder=runtime_map_builder, + authoritative_project_names=authoritative_project_names, ) - def load_latest(self, *, mode: str | None = None, strict_mode_match: bool = False) -> RunState | None: - allowed_root = str(self.runtime_dir) - - def load_for_mode(expected_mode: str | None) -> RunState | None: - def candidate_matches(candidate: RunState) -> bool: - if expected_mode is not None and candidate.mode != expected_mode: - return False - return self._state_matches_scope(candidate) + def load_latest( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, + ) -> RunState | None: + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + self._reconcile_current_aliases_unlocked() + return self._load_latest_unlocked( + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ) - candidate = self._load_state_file_candidate(self.run_state_path(), allowed_root=allowed_root) - if candidate is not None and candidate_matches(candidate): - return candidate + def load_all(self, *, mode: str | None = None) -> list[RunState]: + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + self._reconcile_current_aliases_unlocked() + candidates = self.run_index.candidates(StateSelector(mode=mode, project_names=())) + indexed_states = self._load_valid_index_states(candidates, allowed_root=str(self.runtime_dir)) + if len(indexed_states) != len(candidates): + invalid_run_ids = sorted( + {candidate.run_id for candidate in candidates} + - {indexed.candidate.run_id for indexed in indexed_states} + ) + raise RuntimeError( + "Active runtime state failed integrity validation for run(s): " + ", ".join(invalid_run_ids) + ) + active_states: list[RunState] = [] + for indexed_state in indexed_states: + candidate_projects = normalized_project_names(indexed_state.candidate.project_names) + active_states.append( + filter_state_to_owned_projects( + indexed_state.state, + candidate_projects, + service_project_name=self._service_project_name, + fallback_project=( + next(iter(candidate_projects)) if len(candidate_projects) == 1 else None + ), + ) + ) + return active_states + + def has_active_runs(self) -> bool: + """Return registry activity without trusting or loading mutable run payloads.""" + + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + return bool(self.run_index.candidates(StateSelector(mode=None, project_names=()))) + + def deactivate_run(self, run_id: str) -> bool: + return self.deactivate_runs([run_id]) + + def deactivate_runs(self, run_ids: Sequence[str]) -> bool: + normalized_run_ids = {require_path_component(run_id, label="run_id") for run_id in run_ids} + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + candidates = self.run_index.candidates(StateSelector(mode=None, project_names=())) + removed_candidates = [candidate for candidate in candidates if candidate.run_id in normalized_run_ids] + if not removed_candidates: + return False + removed_run_ids = sorted({candidate.run_id for candidate in removed_candidates}) + self.run_index.remove_many(removed_run_ids) + for candidate in removed_candidates: + self._remove_pointers_to_state(self.runtime_root, candidate.state_path) + if self.compat_mode == self.COMPAT_READ_WRITE: + self._remove_pointers_to_state(self.runtime_legacy_root, candidate.state_path) + try: + self._promote_latest_active_state() + finally: + for candidate in removed_candidates: + self._prune_or_enqueue_revision_cleanup_unlocked(candidate) + return True - candidate = self._load_legacy_shell_candidate( - self.runtime_root / "run_state.state", allowed_root=allowed_root + def write_runtime_artifact( + self, + *, + run_id: str | None, + artifact_name: str, + text: str, + ) -> bool: + """Persist a mutable runtime artifact without bypassing lifecycle fencing. + + Run-bound writers are accepted only while that exact run remains in the + active registry. An unbound writer is a deliberate scope-level diagnostic + write (for example, ``envctl doctor``) and is never guessed onto a run. + """ + safe_artifact_name = require_path_component(artifact_name, label="runtime artifact name") + if safe_artifact_name not in self._RUNTIME_ARTIFACT_NAMES: + raise ValueError(f"unsupported mutable runtime artifact: {safe_artifact_name}") + normalized_run_id = require_path_component(run_id, label="run_id") if run_id is not None else None + + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + if normalized_run_id is None: + candidates = self.run_index.candidates(StateSelector(mode=None, project_names=())) + if candidates: + self._write_scope_diagnostic_artifact(safe_artifact_name, text) + else: + self._remove_scope_diagnostic_artifact(safe_artifact_name) + self._write_scope_artifact_aliases(safe_artifact_name, text) + return True + + candidates = self.run_index.candidates(StateSelector(mode=None, project_names=())) + indexed_states = self._load_valid_index_states( + candidates, + allowed_root=str(self.runtime_dir), ) - if candidate is not None and candidate_matches(candidate): - return candidate - - candidate = self._load_pointer_candidates( - self._ordered_scoped_pointers(mode=expected_mode), - allowed_root=allowed_root, - predicate=candidate_matches, + indexed_state = next( + (candidate for candidate in indexed_states if candidate.candidate.run_id == normalized_run_id), + None, ) - if candidate is not None: - return candidate + if indexed_state is None: + return False + + candidate = indexed_state.candidate + self._write_text(candidate.state_path.parent / safe_artifact_name, text) + self._write_text(self.run_dir_path(candidate.run_id) / safe_artifact_name, text) + self._run_alias_manifest_path(candidate.run_id).unlink(missing_ok=True) + self._promote_latest_active_state() + return True + + def _write_scope_diagnostic_artifact(self, artifact_name: str, text: str) -> None: + diagnostics_dir = self._contained_child( + self.runtime_root, + "diagnostics", + label="diagnostics directory", + ) + durable_mkdir(diagnostics_dir) + self._write_text(diagnostics_dir / artifact_name, text) - if self.compat_mode == self.SCOPED_ONLY: - return None + def _remove_scope_diagnostic_artifact(self, artifact_name: str) -> None: + diagnostics_dir = self.runtime_root / "diagnostics" + if diagnostics_dir.is_symlink() or not diagnostics_dir.is_dir(): + return + (diagnostics_dir / artifact_name).unlink(missing_ok=True) + try: + diagnostics_dir.rmdir() + except OSError: + pass - candidate = self._load_state_file_candidate( - self.runtime_legacy_root / "run_state.json", - allowed_root=allowed_root, - ) - if candidate is not None and candidate_matches(candidate): - return candidate + def _write_scope_artifact_aliases(self, artifact_name: str, text: str) -> None: + self._write_text(self.runtime_root / artifact_name, text) + if self.compat_mode == self.COMPAT_READ_WRITE: + self._write_text(self.runtime_legacy_root / artifact_name, text) - candidate = self._load_legacy_shell_candidate( - self.runtime_legacy_root / "run_state.state", - allowed_root=allowed_root, - ) - if candidate is not None and candidate_matches(candidate): - return candidate + def _load_latest_unlocked( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, + ) -> RunState | None: + allowed_root = str(self.runtime_dir) + selected_projects = tuple(project_names or ()) - candidate = self._load_pointer_candidates( - self._ordered_legacy_pointers(mode=expected_mode), + def load_for_mode(expected_mode: str | None) -> RunState | None: + candidates = self.run_index.candidates(StateSelector(mode=expected_mode, project_names=())) + if expected_mode is None and candidates: + valid_candidates = self._load_valid_index_states( + candidates, + allowed_root=allowed_root, + ) + selected_project_keys = normalized_project_names(selected_projects) + if selected_project_keys: + valid_candidates = [ + indexed + for indexed in valid_candidates + if selected_project_keys.intersection( + normalized_project_names( + self._project_names_from_state(indexed.state) + ) + ) + ] + if not valid_candidates: + return None + newest_mode = max( + valid_candidates, + key=lambda indexed: indexed.candidate.activation_sequence, + ).candidate.mode + candidates = [ + indexed.candidate + for indexed in valid_candidates + if indexed.candidate.mode == newest_mode + ] + indexed_candidate = self._load_index_candidates( + candidates, allowed_root=allowed_root, - predicate=candidate_matches, + selected_projects=selected_projects, ) - if candidate is not None: - return candidate - return None + return indexed_candidate matched = load_for_mode(mode) if matched is not None: @@ -248,42 +451,650 @@ def _save_runtime_update( state: RunState, emit: Callable[..., None], runtime_map_builder: Callable[[RunState], dict[str, object]], + authoritative_project_names: Sequence[str] | None = None, ) -> dict[str, object]: - run_state_path = self._runtime_update_run_state_path(state) - run_state_text = self._write_run_state(state=state, run_state_path=run_state_path, emit=emit) - + project_names = ( + self._runtime_project_names_from_state(state) + if authoritative_project_names is None + else sorted( + { + str(project_name).strip() + for project_name in authoritative_project_names + if str(project_name).strip() + }, + key=str.casefold, + ) + ) + self._prepare_state_for_save(state, project_names=project_names, authoritative_projects=True) runtime_map = runtime_map_builder(state) - runtime_map_text = self._write_runtime_map_text( + if not project_names: + run_ids = [state.run_id, *self._source_run_ids(state)] + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + self.run_index.remove_many(run_ids) + self._promote_latest_active_state() + self._safe_emit(emit, "state.deactivate", run_ids=sorted(set(run_ids))) + return runtime_map + artifacts = { + "run_state.json": self._json_text(state_to_dict(state)), + "runtime_map.json": self._json_text(runtime_map), + } + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + self._carry_forward_runtime_artifacts_unlocked(state, artifacts) + revision_state_path = self._commit_revision_unlocked( + state=state, + project_names=project_names, + artifacts=artifacts, + supersede_run_ids=self._source_run_ids(state), + ) + self._emit_saved_state( + emit, state=state, - runtime_map=runtime_map, - runtime_map_path=run_state_path.parent / "runtime_map.json", - emit_path=self.runtime_map_path(), - emit=emit, + state_path=revision_state_path, + runtime_map_text=artifacts["runtime_map.json"], ) + return runtime_map + def _carry_forward_runtime_artifacts_unlocked( + self, + state: RunState, + artifacts: dict[str, str], + ) -> None: + source_run_ids = {state.run_id, *self._source_run_ids(state)} + source_candidates = [ + candidate + for candidate in self.run_index.candidates(StateSelector(mode=state.mode, project_names=())) + if candidate.run_id in source_run_ids + ] + current_ports_manifest = artifacts.get("ports_manifest.json") + artifacts["ports_manifest.json"] = self._merged_ports_manifest_text( + state, + source_candidates, + current_manifest_text=current_ports_manifest, + ) + artifacts.setdefault( + "error_report.json", + self._merged_error_report_text(state, source_candidates), + ) + if not source_candidates: + return + source_candidate = max( + source_candidates, + key=lambda candidate: candidate.activation_sequence, + ) + source_directories = ( + source_candidate.state_path.parent, + self.run_dir_path(source_candidate.run_id), + ) + for artifact_name in self._RUNTIME_ARTIFACT_NAMES: + if artifact_name in artifacts: + continue + for source_directory in source_directories: + source = source_directory / artifact_name + if source.is_file() and not source.is_symlink(): + try: + artifacts[artifact_name] = source.read_text(encoding="utf-8") + except (OSError, UnicodeError): + continue + break + + def _merged_ports_manifest_text( + self, + state: RunState, + source_candidates: Sequence[RunIndexCandidate], + *, + current_manifest_text: str | None = None, + ) -> str: project_names = self._project_names_from_state(state) - self._write_mode_pointers( - root=self.runtime_root, - run_state_path=run_state_path, - mode=state.mode, - project_names=project_names, + selected = normalized_project_names(project_names) + projects_by_key: dict[str, dict[str, object]] = {} + for candidate in sorted( + source_candidates, + key=lambda item: item.activation_sequence, + ): + payload = self._read_revision_json_artifact(candidate, "ports_manifest.json") + raw_projects = payload.get("projects") if isinstance(payload, Mapping) else None + if not isinstance(raw_projects, list): + continue + for raw_project in raw_projects: + if not isinstance(raw_project, Mapping): + continue + project_name = str(raw_project.get("project", "")).strip() + project_key = project_name.casefold() + if project_key and project_key in selected: + projects_by_key[project_key] = dict(raw_project) + + if current_manifest_text is not None: + try: + current_payload = json.loads(current_manifest_text) + except (TypeError, ValueError, json.JSONDecodeError): + current_payload = {} + raw_current_projects = current_payload.get("projects") if isinstance(current_payload, Mapping) else None + if isinstance(raw_current_projects, list): + for raw_project in raw_current_projects: + if not isinstance(raw_project, Mapping): + continue + project_name = str(raw_project.get("project", "")).strip() + project_key = project_name.casefold() + if project_key and project_key in selected: + projects_by_key[project_key] = dict(raw_project) + + roots = state.metadata.get("project_roots") + project_roots = roots if isinstance(roots, Mapping) else {} + for project_name in project_names: + project_key = project_name.casefold() + project_payload = projects_by_key.get(project_key) + if project_payload is None: + project_payload = { + "project": project_name, + "root": next( + ( + str(root) + for name, root in project_roots.items() + if str(name).strip().casefold() == project_key + ), + "", + ), + "ports": {}, + } + projects_by_key[project_key] = project_payload + raw_ports = project_payload.get("ports") + ports = dict(raw_ports) if isinstance(raw_ports, Mapping) else {} + for service_name, service in state.services.items(): + if self._service_project_name(service_name, service).casefold() != project_key: + continue + service_key = str(service.type or service_name).strip().casefold() or "service" + ports.setdefault( + service_key, + { + "requested": service.requested_port, + "assigned": service.actual_port or service.requested_port, + "final": service.actual_port, + "source": "runtime_state", + "retries": None, + }, + ) + project_payload["ports"] = ports + return self._json_text( + { + "run_id": state.run_id, + "mode": state.mode, + "projects": [projects_by_key[key] for key in sorted(projects_by_key)], + } ) - if self.compat_mode == self.COMPAT_READ_WRITE: - self.runtime_legacy_root.mkdir(parents=True, exist_ok=True) - self._write_text(self.runtime_legacy_root / "run_state.json", run_state_text) - self._write_text(self.runtime_legacy_root / "runtime_map.json", runtime_map_text) - self._write_mode_pointers( - root=self.runtime_legacy_root, - run_state_path=run_state_path, + def _merged_error_report_text( + self, + state: RunState, + source_candidates: Sequence[RunIndexCandidate], + ) -> str: + errors: list[str] = [] + for candidate in sorted(source_candidates, key=lambda item: item.activation_sequence): + payload = self._read_revision_json_artifact(candidate, "error_report.json") + raw_errors = payload.get("errors") if isinstance(payload, Mapping) else None + if not isinstance(raw_errors, list): + continue + for raw_error in raw_errors: + error = str(raw_error).strip() + if error and error not in errors: + errors.append(error) + return self._json_text( + { + "run_id": state.run_id, + "errors": errors, + "source_run_ids": sorted({candidate.run_id for candidate in source_candidates}), + "generated_at": datetime.now(tz=UTC).isoformat(), + } + ) + + @staticmethod + def _read_revision_json_artifact( + candidate: RunIndexCandidate, + artifact_name: str, + ) -> Mapping[str, object]: + path = candidate.state_path.parent / artifact_name + if path.is_symlink() or not path.is_file(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return {} + return payload if isinstance(payload, Mapping) else {} + + def _commit_revision_unlocked( + self, + *, + state: RunState, + project_names: Sequence[str], + artifacts: Mapping[str, str], + supersede_run_ids: Sequence[str] = (), + on_commit: Callable[[], None] | None = None, + ) -> Path: + superseded_run_ids = { + require_path_component(run_id, label="superseded run_id") + for run_id in supersede_run_ids + if str(run_id).strip() and str(run_id).strip() != state.run_id + } + superseded_candidates = [ + candidate + for candidate in self.run_index.candidates(StateSelector(mode=None, project_names=())) + if candidate.run_id in superseded_run_ids + ] + run_dir = self.run_dir_path(state.run_id) + revisions_dir = self._contained_child(run_dir, "revisions", label="revisions directory") + revision_id = f"{datetime.now(tz=UTC).strftime('%Y%m%dT%H%M%S%fZ')}-{uuid.uuid4().hex[:12]}" + revision_dir = self._contained_child(revisions_dir, revision_id, label="revision id") + revision_state_path = revision_dir / "run_state.json" + retain_revision = False + try: + durable_mkdir(revision_dir) + for artifact_name, text in artifacts.items(): + safe_name = require_path_component(artifact_name, label="artifact name") + self._write_text(revision_dir / safe_name, text) + try: + self.run_index.record( + state_path=revision_state_path, + run_id=state.run_id, + mode=state.mode, + project_names=project_names, + supersede_run_ids=supersede_run_ids, + ) + except BaseException: + indexed = self._revision_index_commit_state_unlocked( + state=state, + revision_state_path=revision_state_path, + ) + # An unreadable registry has an uncertain commit outcome. Keep + # the immutable revision so a visible primary/backup can never + # be left pointing at artifacts this writer deleted. + retain_revision = indexed is not False + if indexed and on_commit is not None: + on_commit() + raise + retain_revision = True + if on_commit is not None: + on_commit() + finally: + if not retain_revision: + shutil.rmtree(revision_dir, ignore_errors=True) + + try: + self._publish_revision_aliases( + state=state, + artifacts=artifacts, + ) + self._promote_latest_active_state() + for candidate in superseded_candidates: + self._prune_or_enqueue_revision_cleanup_unlocked(candidate) + except OSError: + # The index is the commit point. Aliases are repaired on the next repository read. + for candidate in superseded_candidates: + self._enqueue_revision_cleanup_if_needed_unlocked(candidate) + except BaseException: + for candidate in superseded_candidates: + self._enqueue_revision_cleanup_if_needed_unlocked(candidate) + raise + return revision_state_path + + def _revision_index_commit_state_unlocked( + self, + *, + state: RunState, + revision_state_path: Path, + ) -> bool | None: + try: + expected_state_path = revision_state_path.resolve() + expected_state_fingerprint = file_fingerprint(expected_state_path) + runtime_map_path = expected_state_path.parent / "runtime_map.json" + expected_runtime_map_fingerprint = file_fingerprint(runtime_map_path) + candidates = self.run_index.candidates( + StateSelector(mode=None, project_names=()) + ) + except BaseException: # noqa: BLE001 - unreadable authority is uncertain, never negative + return None + expected_mode = str(state.mode or "").strip().casefold() + return any( + candidate.run_id == state.run_id + and candidate.mode == expected_mode + and candidate.state_path == expected_state_path + and candidate.state_fingerprint == expected_state_fingerprint + and candidate.runtime_map_fingerprint == expected_runtime_map_fingerprint + for candidate in candidates + ) + + def _publish_revision_aliases( + self, + *, + state: RunState, + artifacts: Mapping[str, str], + ) -> None: + run_dir = self.run_dir_path(state.run_id) + for artifact_name in self._RUN_ALIAS_ARTIFACT_NAMES: + alias = run_dir / artifact_name + text = artifacts.get(artifact_name) + if text is None: + alias.unlink(missing_ok=True) + else: + self._write_text(alias, text) + candidates = self.run_index.candidates(StateSelector(mode=state.mode, project_names=())) + committed = next( + (candidate for candidate in candidates if candidate.run_id == state.run_id), + None, + ) + if committed is not None: + self._write_run_alias_manifest(committed) + + def _run_alias_manifest_path(self, run_id: str) -> Path: + return self.run_dir_path(run_id) / self._RUN_ALIAS_MANIFEST_NAME + + def _write_run_alias_manifest(self, candidate: RunIndexCandidate) -> None: + source_dir = candidate.state_path.parent + run_dir = self.run_dir_path(candidate.run_id) + artifacts = { + artifact_name: { + "source": self._path_lstat_signature(source_dir / artifact_name), + "alias": self._path_lstat_signature(run_dir / artifact_name), + } + for artifact_name in self._RUN_ALIAS_ARTIFACT_NAMES + } + self._write_text( + self._run_alias_manifest_path(candidate.run_id), + self._json_text( + { + "version": 1, + "source_dir": str(source_dir), + "artifacts": artifacts, + } + ), + ) + + def _run_aliases_match_manifest(self, candidate: RunIndexCandidate) -> bool: + manifest_path = self._run_alias_manifest_path(candidate.run_id) + if manifest_path.is_symlink() or not manifest_path.is_file(): + return False + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return False + if not isinstance(manifest, Mapping): + return False + if manifest.get("version") != 1 or manifest.get("source_dir") != str(candidate.state_path.parent): + return False + expected_artifacts = manifest.get("artifacts") + if not isinstance(expected_artifacts, Mapping): + return False + source_dir = candidate.state_path.parent + run_dir = self.run_dir_path(candidate.run_id) + actual_artifacts = { + artifact_name: { + "source": self._path_lstat_signature(source_dir / artifact_name), + "alias": self._path_lstat_signature(run_dir / artifact_name), + } + for artifact_name in self._RUN_ALIAS_ARTIFACT_NAMES + } + return expected_artifacts == actual_artifacts + + @staticmethod + def _path_lstat_signature(path: Path) -> dict[str, object]: + try: + metadata = path.lstat() + except FileNotFoundError: + return {"kind": "missing"} + if not stat.S_ISREG(metadata.st_mode): + return { + "kind": "unsafe", + "mode": metadata.st_mode, + "inode": metadata.st_ino, + "size": metadata.st_size, + "mtime_ns": metadata.st_mtime_ns, + "ctime_ns": metadata.st_ctime_ns, + } + return { + "kind": "file", + "inode": metadata.st_ino, + "size": metadata.st_size, + "mtime_ns": metadata.st_mtime_ns, + "ctime_ns": metadata.st_ctime_ns, + } + + def _prune_revisions_for_candidates_unlocked( + self, + indexed_states: Sequence[_IndexedState], + ) -> None: + seen_run_ids: set[str] = set() + for indexed in indexed_states: + candidate = indexed.candidate + if candidate.run_id in seen_run_ids: + continue + seen_run_ids.add(candidate.run_id) + try: + self._prune_run_revisions_unlocked(candidate) + except OSError: + # Active candidates are retried by the next reconciliation. + continue + self._retry_pending_revision_cleanups_unlocked() + + def _prune_run_revisions_unlocked(self, candidate: RunIndexCandidate) -> None: + run_dir = self.run_dir_path(candidate.run_id) + revisions_dir = self._contained_child( + run_dir, + "revisions", + label="revisions directory", + ) + current_revision = candidate.state_path.parent + if ( + revisions_dir.is_symlink() + or not revisions_dir.is_dir() + or current_revision.is_symlink() + or not current_revision.is_dir() + or current_revision.resolve().parent != revisions_dir.resolve() + ): + return + siblings: list[tuple[int, str, Path]] = [] + for path in revisions_dir.iterdir(): + if path == current_revision or path.is_symlink() or not path.is_dir(): + continue + try: + if path.resolve().parent != revisions_dir.resolve(): + continue + metadata = path.stat() + except OSError: + continue + siblings.append((metadata.st_mtime_ns, path.name, path)) + siblings.sort(reverse=True) + removed = False + try: + for _, _, stale_revision in siblings[1:]: + shutil.rmtree(stale_revision) + removed = True + finally: + if removed: + fsync_directory(revisions_dir) + + def _enqueue_revision_cleanup_unlocked(self, candidate: RunIndexCandidate) -> None: + marker_path = self._revision_cleanup_marker_path(candidate.run_id) + run_dir = self.run_dir_path(candidate.run_id) + revisions_dir = self._contained_child( + run_dir, + "revisions", + label="revisions directory", + ) + state_path = candidate.state_path.resolve() + if state_path.name != "run_state.json" or state_path.parent.parent != revisions_dir.resolve(): + raise ValueError("revision cleanup state path is outside the run revisions directory") + cleanup_dir = marker_path.parent + durable_mkdir(cleanup_dir) + self._write_text( + marker_path, + self._json_text( + { + "version": 1, + "runtime_scope_id": self.runtime_scope_id, + "run_id": candidate.run_id, + "mode": candidate.mode, + "state_path": str(state_path), + } + ), + ) + + def _prune_or_enqueue_revision_cleanup_unlocked( + self, + candidate: RunIndexCandidate, + ) -> None: + try: + self._prune_run_revisions_unlocked(candidate) + except OSError: + self._enqueue_revision_cleanup_if_needed_unlocked(candidate) + + def _enqueue_revision_cleanup_if_needed_unlocked( + self, + candidate: RunIndexCandidate, + ) -> None: + if self._run_revision_count(candidate.run_id) <= 2: + return + try: + self._enqueue_revision_cleanup_unlocked(candidate) + except (OSError, RuntimeError, ValueError): + return + + def _retry_pending_revision_cleanups_unlocked(self) -> None: + cleanup_dir = self.runtime_root / self._REVISION_CLEANUP_DIR_NAME + if cleanup_dir.is_symlink() or not cleanup_dir.is_dir(): + return + try: + marker_paths = list(cleanup_dir.iterdir()) + except OSError: + return + for marker_path in marker_paths: + if marker_path.is_symlink() or not marker_path.is_file(): + continue + candidate = self._revision_cleanup_candidate(marker_path) + if candidate is None: + self._remove_revision_cleanup_marker(marker_path) + continue + try: + self._prune_run_revisions_unlocked(candidate) + except OSError: + continue + if self._run_revision_count(candidate.run_id) <= 2: + self._remove_revision_cleanup_marker(marker_path) + try: + cleanup_dir.rmdir() + except OSError: + return + try: + fsync_directory(self.runtime_root) + except OSError: + pass + + def _revision_cleanup_candidate( + self, + marker_path: Path, + ) -> RunIndexCandidate | None: + try: + payload = json.loads(marker_path.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + return None + if payload.get("version") != 1 or payload.get("runtime_scope_id") != self.runtime_scope_id: + return None + raw_run_id = payload.get("run_id") + raw_mode = payload.get("mode") + raw_state_path = payload.get("state_path") + if not all(isinstance(value, str) and value for value in (raw_run_id, raw_mode, raw_state_path)): + return None + run_id = require_path_component(raw_run_id, label="run_id") + if marker_path != self._revision_cleanup_marker_path(run_id): + return None + run_dir = self.run_dir_path(run_id) + revisions_dir = self._contained_child( + run_dir, + "revisions", + label="revisions directory", + ) + state_path = Path(raw_state_path).expanduser().resolve() + if state_path.name != "run_state.json" or state_path.parent.parent != revisions_dir.resolve(): + return None + state = self._load_state_file_candidate( + state_path, + allowed_root=str(self.runtime_dir), + ) + if state is None or state.run_id != run_id: + return None + if state.metadata.get("repo_scope_id") != self.runtime_scope_id: + return None + if state.mode.casefold() != raw_mode.casefold(): + return None + return RunIndexCandidate( + state_path=state_path, + run_id=run_id, mode=state.mode, - project_names=project_names, + project_names=(), + sequence=0, ) + except (OSError, RuntimeError, TypeError, ValueError, json.JSONDecodeError): + return None - return runtime_map + def _revision_cleanup_marker_path(self, run_id: str) -> Path: + normalized_run_id = require_path_component(run_id, label="run_id") + cleanup_dir = self._contained_child( + self.runtime_root, + self._REVISION_CLEANUP_DIR_NAME, + label="revision cleanup directory", + ) + digest = text_fingerprint(normalized_run_id) + marker_name = f"{digest}.json" + return self._contained_child( + cleanup_dir, + marker_name, + label="revision cleanup marker", + ) + + def _remove_revision_cleanup_marker(self, marker_path: Path) -> None: + try: + marker_path.unlink(missing_ok=True) + fsync_directory(marker_path.parent) + except OSError: + return + + def _run_revision_count(self, run_id: str) -> int: + revisions_dir = self.run_dir_path(run_id) / "revisions" + if revisions_dir.is_symlink() or not revisions_dir.is_dir(): + return 0 + try: + return sum(1 for path in revisions_dir.iterdir() if not path.is_symlink() and path.is_dir()) + except OSError: + return 3 - def load_by_pointer(self, pointer_path: str) -> RunState: - return load_state_from_pointer(pointer_path, allowed_root=str(self.runtime_dir)) + def _emit_saved_state( + self, + emit: Callable[..., None], + *, + state: RunState, + state_path: Path, + runtime_map_text: str, + ) -> None: + self._safe_emit(emit, "state.save", run_id=state.run_id, path=str(state_path)) + self._safe_emit( + emit, + "state.fingerprint.after_save", + run_id=state.run_id, + state_fingerprint=state_fingerprint(state), + ) + self._safe_emit(emit, "runtime_map.write", path=str(state_path.parent / "runtime_map.json")) + self._safe_emit( + emit, + "runtime_map.fingerprint", + run_id=state.run_id, + runtime_map_fingerprint=text_fingerprint(runtime_map_text), + ) + + @staticmethod + def _safe_emit(emit: Callable[..., None], event: str, **payload: object) -> None: + try: + emit(event, **payload) + except Exception: + return def _load_state_file_candidate(self, path: Path, *, allowed_root: str) -> RunState | None: if not path.is_file(): @@ -293,68 +1104,499 @@ def _load_state_file_candidate(self, path: Path, *, allowed_root: str) -> RunSta except Exception: return None - def _load_legacy_shell_candidate(self, path: Path, *, allowed_root: str) -> RunState | None: - if not path.is_file(): + def _load_index_candidates( + self, + candidates: Sequence[RunIndexCandidate], + *, + allowed_root: str, + selected_projects: Sequence[str], + ) -> RunState | None: + indexed_states = self._load_valid_index_states(candidates, allowed_root=allowed_root) + if not indexed_states: return None + chosen_states = select_indexed_owners( + indexed_states, + selected_projects=selected_projects, + service_project_name=self._service_project_name, + ) + return state_from_indexed_owners( + chosen_states, + project_names_from_state=self._project_names_from_state, + source_run_ids=self._source_run_ids, + ) + + def _ensure_active_index_unlocked(self) -> None: + if not self.run_index.needs_rebuild(): + self.run_index.repair_copies() + if not self._registry_marker_path.is_file(): + self._write_text( + self._registry_marker_path, + self._json_text({"version": 1, "runtime_scope_id": self.runtime_scope_id}), + ) + return + if self._registry_marker_path.is_file(): + raise RuntimeError( + "Active run registry is unavailable because both the primary and backup indexes are invalid." + ) + + recovered: list[tuple[int, Path, RunState, list[str]]] = [] + runs_root = self.run_dir_path(None) + if runs_root.is_dir(): + for run_dir in runs_root.iterdir(): + if run_dir.is_symlink() or not run_dir.is_dir(): + continue + state_path = run_dir / "run_state.json" + if state_path.is_symlink() or not state_path.is_file(): + continue + try: + run_id = require_path_component(run_dir.name, label="run_id") + if run_dir.resolve().parent != runs_root.resolve(): + continue + state = self._load_state_file_candidate(state_path, allowed_root=str(self.runtime_dir)) + if state is None or state.run_id != run_id: + continue + scope = state.metadata.get("repo_scope_id") + if isinstance(scope, str) and scope and scope != self.runtime_scope_id: + continue + project_names = self._project_names_from_state(state) + if not project_names: + project_names = self._project_names_from_ports_manifest( + state_path.parent / "ports_manifest.json" + ) + if not project_names: + project_names = self._project_names_from_pointers( + self.runtime_root, + state_path, + ) + modified_at = state_path.stat().st_mtime_ns + state.metadata["repo_scope_id"] = self.runtime_scope_id + state.metadata["project_names"] = sorted(set(project_names), key=str.casefold) + recovered.append((modified_at, state_path, state, project_names)) + except (OSError, RuntimeError, ValueError): + continue + recovered_run_ids = {state.run_id for _, _, state, _ in recovered} + for root in self._legacy_migration_roots(): + legacy_candidates = [ + (root / "run_state.json", self._load_state_file_candidate), + (root / "run_state.state", self._load_legacy_shell_candidate), + ] + legacy_candidates.extend( + (pointer, self._load_pointer_candidate) for pointer in self._legacy_pointer_paths(root) + ) + for candidate_path, legacy_loader in legacy_candidates: + if candidate_path.is_symlink() or not candidate_path.is_file(): + continue + try: + state = legacy_loader(candidate_path, allowed_root=str(self.runtime_dir)) + if state is None or state.run_id in recovered_run_ids: + continue + existing_scope = state.metadata.get("repo_scope_id") + if isinstance(existing_scope, str) and existing_scope and existing_scope != self.runtime_scope_id: + continue + project_names = self._project_names_from_state(state) + if not project_names: + project_names = self._project_names_from_ports_manifest(root / "ports_manifest.json") + if not project_names: + project_names = self._project_names_from_pointers(root, candidate_path) + if not project_names and candidate_path.name.startswith(".last_state.trees."): + project_name = candidate_path.name.removeprefix(".last_state.trees.").strip() + if project_name: + project_names = [project_name] + state.metadata["repo_scope_id"] = self.runtime_scope_id + state.metadata["project_names"] = sorted(set(project_names), key=str.casefold) + recovered.append((candidate_path.stat().st_mtime_ns, candidate_path, state, project_names)) + recovered_run_ids.add(state.run_id) + except (OSError, RuntimeError, ValueError): + continue + + indexed_entries: list[RunIndexCandidate] = [] + registry_committed = False try: - return load_legacy_shell_state(str(path), allowed_root=allowed_root) + for sequence, (_, source_path, state, project_names) in enumerate( + sorted(recovered, key=lambda item: (item[0], str(item[1]))), + start=1, + ): + revision_state_path = self._stage_legacy_revision( + state=state, + source_path=source_path, + ) + indexed_entries.append( + RunIndexCandidate( + state_path=revision_state_path, + run_id=state.run_id, + mode=state.mode, + project_names=tuple(sorted(normalized_project_names(project_names))), + sequence=sequence, + ) + ) + self.run_index.replace_all(indexed_entries) + registry_committed = True + finally: + if not registry_committed: + for entry in indexed_entries: + shutil.rmtree(entry.state_path.parent, ignore_errors=True) + self._write_text( + self._registry_marker_path, + self._json_text({"version": 1, "runtime_scope_id": self.runtime_scope_id}), + ) + + def _legacy_migration_roots(self) -> list[Path]: + roots = [self.runtime_root] + if self.compat_mode != self.SCOPED_ONLY and self.runtime_legacy_root != self.runtime_root: + roots.append(self.runtime_legacy_root) + return roots + + @staticmethod + def _legacy_pointer_paths(root: Path) -> list[Path]: + candidates = [ + root / ".last_state.main", + *sorted(root.glob(".last_state.trees.*")), + root / ".last_state", + ] + return [pointer for pointer in candidates if not pointer.is_symlink() and pointer.is_file()] + + @staticmethod + def _load_pointer_candidate(path: Path, *, allowed_root: str) -> RunState | None: + try: + return load_state_from_pointer(str(path), allowed_root=allowed_root) except Exception: return None - def _load_pointer_candidates( + def _stage_legacy_revision(self, *, state: RunState, source_path: Path) -> Path: + run_dir = self.run_dir_path(state.run_id) + revisions_dir = self._contained_child(run_dir, "revisions", label="revisions directory") + revision_id = f"legacy-{uuid.uuid4().hex}" + revision_dir = self._contained_child(revisions_dir, revision_id, label="revision id") + revision_state_path = revision_dir / "run_state.json" + staged = False + try: + durable_mkdir(revision_dir) + self._write_text(revision_state_path, self._json_text(state_to_dict(state))) + for artifact_name in ( + "runtime_map.json", + *self._REVISION_ANCILLARY_ARTIFACT_NAMES, + ): + source_artifact = source_path.parent / artifact_name + if source_artifact.is_symlink() or not source_artifact.is_file(): + continue + try: + self._write_text( + revision_dir / artifact_name, + source_artifact.read_text(encoding="utf-8"), + ) + except (OSError, UnicodeError): + continue + staged = True + finally: + if not staged: + shutil.rmtree(revision_dir, ignore_errors=True) + return revision_state_path + + @staticmethod + def _project_names_from_ports_manifest(path: Path) -> list[str]: + if not path.is_file(): + return [] + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return [] + raw_projects = payload.get("projects") if isinstance(payload, Mapping) else None + if not isinstance(raw_projects, list): + return [] + return sorted( + { + str(project.get("project", "")).strip() + for project in raw_projects + if isinstance(project, Mapping) and str(project.get("project", "")).strip() + }, + key=str.casefold, + ) + + @staticmethod + def _project_names_from_pointers(root: Path, state_path: Path) -> list[str]: + expected = state_path.resolve() + project_names: set[str] = set() + for pointer in root.glob(".last_state.trees.*"): + if pointer.is_symlink() or not pointer.is_file(): + continue + try: + raw_target = pointer.read_text(encoding="utf-8").splitlines()[0].strip() + target = Path(raw_target).expanduser().resolve() + except (IndexError, OSError, RuntimeError, ValueError): + continue + if target == expected: + project_names.add(pointer.name.removeprefix(".last_state.trees.")) + return sorted(project_names, key=str.casefold) + + def _load_valid_index_states( self, - pointers: list[Path], + candidates: Sequence[RunIndexCandidate], *, allowed_root: str, - predicate: Callable[[RunState], bool], - ) -> RunState | None: - for pointer in pointers: + ) -> list[_IndexedState]: + loaded: list[_IndexedState] = [] + for indexed in candidates: try: - candidate = load_state_from_pointer(str(pointer), allowed_root=allowed_root) - except Exception: + if file_fingerprint(indexed.state_path) != indexed.state_fingerprint: + continue + if indexed.runtime_map_fingerprint is not None: + runtime_map_path = indexed.state_path.parent / "runtime_map.json" + if ( + runtime_map_path.is_symlink() + or not runtime_map_path.is_file() + or file_fingerprint(runtime_map_path) != indexed.runtime_map_fingerprint + ): + continue + except OSError: continue - if predicate(candidate): - return candidate - return None + state = self._load_state_file_candidate(indexed.state_path, allowed_root=allowed_root) + if state is None: + continue + if state.run_id != indexed.run_id or state.mode.casefold() != indexed.mode: + continue + if state.metadata.get("repo_scope_id") != self.runtime_scope_id: + continue + state_projects = normalized_project_names(self._project_names_from_state(state)) + indexed_projects = frozenset(indexed.project_names) + if indexed_projects and not indexed_projects.issubset(state_projects): + continue + loaded.append(_IndexedState(candidate=indexed, state=state)) + return loaded - def _write_run_state( + def _promote_latest_active_state(self) -> None: + candidates = self.run_index.candidates(StateSelector(mode=None, project_names=())) + if not candidates: + self._remove_current_aliases() + return + valid_candidates = self._load_valid_index_states( + candidates, + allowed_root=str(self.runtime_dir), + ) + if not valid_candidates: + self._remove_current_aliases() + return + latest_candidate = max( + valid_candidates, + key=lambda indexed: indexed.candidate.activation_sequence, + ).candidate + mode_candidates = [indexed for indexed in valid_candidates if indexed.candidate.mode == latest_candidate.mode] + chosen_states = select_indexed_owners( + mode_candidates, + selected_projects=(), + service_project_name=self._service_project_name, + ) + latest_state = state_from_indexed_owners( + chosen_states, + project_names_from_state=self._project_names_from_state, + source_run_ids=self._source_run_ids, + ) + if latest_state is None: + self._remove_current_aliases() + return + selected_candidates = [indexed.candidate for indexed in chosen_states] + latest_selected = max( + selected_candidates, + key=lambda candidate: ( + candidate.activation_sequence, + candidate.sequence, + str(candidate.state_path), + ), + ) + source_dir = latest_selected.state_path.parent + current_state_text = self._json_text(state_to_dict(latest_state)) + current_runtime_map_text = self._json_text(build_runtime_map(latest_state)) + aggregate_artifacts = { + "ports_manifest.json": self._merged_ports_manifest_text( + latest_state, + selected_candidates, + ), + "error_report.json": self._merged_error_report_text( + latest_state, + selected_candidates, + ), + } + self._write_text(self.run_state_path(), current_state_text) + self._write_text(self.runtime_map_path(), current_runtime_map_text) + for artifact_name in self._REVISION_ANCILLARY_ARTIFACT_NAMES: + current = self.runtime_root / artifact_name + aggregate_text = aggregate_artifacts.get(artifact_name) + if aggregate_text is not None: + self._write_text(current, aggregate_text) + else: + source = source_dir / artifact_name + if source.is_file() and not source.is_symlink(): + self._write_text(current, source.read_text(encoding="utf-8")) + else: + current.unlink(missing_ok=True) + self._rebuild_mode_pointers(self.runtime_root, valid_candidates) + if self.compat_mode == self.COMPAT_READ_WRITE: + self._write_text(self.runtime_legacy_root / "run_state.json", current_state_text) + self._write_text(self.runtime_legacy_root / "runtime_map.json", current_runtime_map_text) + for artifact_name in self._REVISION_ANCILLARY_ARTIFACT_NAMES: + current = self.runtime_legacy_root / artifact_name + aggregate_text = aggregate_artifacts.get(artifact_name) + if aggregate_text is not None: + self._write_text(current, aggregate_text) + else: + source = source_dir / artifact_name + if source.is_file() and not source.is_symlink(): + self._write_text(current, source.read_text(encoding="utf-8")) + else: + current.unlink(missing_ok=True) + self._rebuild_mode_pointers(self.runtime_legacy_root, valid_candidates) + self._write_text( + self._alias_manifest_path, + self._json_text( + { + "version": 2, + "sources": { + indexed.candidate.run_id: str(indexed.candidate.state_path) for indexed in valid_candidates + }, + "source_fingerprints": { + indexed.candidate.run_id: state_fingerprint(indexed.state) for indexed in valid_candidates + }, + "state_fingerprint": text_fingerprint(current_state_text), + "runtime_map_fingerprint": text_fingerprint(current_runtime_map_text), + "source_artifact_signatures": self._revision_artifact_signatures(valid_candidates), + "scoped_alias_signatures": self._artifact_signatures(self.runtime_root), + "legacy_alias_signatures": ( + self._artifact_signatures(self.runtime_legacy_root) + if self.compat_mode == self.COMPAT_READ_WRITE + else None + ), + } + ), + ) + self._prune_revisions_for_candidates_unlocked(valid_candidates) + + def _reconcile_current_aliases_unlocked(self) -> None: + candidates = self.run_index.candidates(StateSelector(mode=None, project_names=())) + if not candidates: + if self.run_state_path().exists() or (self.runtime_root / ".last_state").exists(): + self._remove_current_aliases() + return + valid_candidates = self._load_valid_index_states( + candidates, + allowed_root=str(self.runtime_dir), + ) + expected_sources = {indexed.candidate.run_id: str(indexed.candidate.state_path) for indexed in valid_candidates} + expected_source_fingerprints = { + indexed.candidate.run_id: state_fingerprint(indexed.state) for indexed in valid_candidates + } + expected_source_artifact_signatures = self._revision_artifact_signatures(valid_candidates) + try: + self._reconcile_run_aliases_unlocked(valid_candidates) + except OSError: + pass + try: + manifest = json.loads(self._alias_manifest_path.read_text(encoding="utf-8")) + state_text = self.run_state_path().read_text(encoding="utf-8") + runtime_map_text = self.runtime_map_path().read_text(encoding="utf-8") + if ( + isinstance(manifest, Mapping) + and manifest.get("version") == 2 + and manifest.get("sources") == expected_sources + and manifest.get("source_fingerprints") == expected_source_fingerprints + and manifest.get("source_artifact_signatures") == expected_source_artifact_signatures + and manifest.get("state_fingerprint") == text_fingerprint(state_text) + and manifest.get("runtime_map_fingerprint") == text_fingerprint(runtime_map_text) + and manifest.get("scoped_alias_signatures") == self._artifact_signatures(self.runtime_root) + and manifest.get("legacy_alias_signatures") + == ( + self._artifact_signatures(self.runtime_legacy_root) + if self.compat_mode == self.COMPAT_READ_WRITE + else None + ) + and self._mode_pointers_match(self.runtime_root, valid_candidates) + and ( + self.compat_mode != self.COMPAT_READ_WRITE + or self._mode_pointers_match(self.runtime_legacy_root, valid_candidates) + ) + ): + self._prune_revisions_for_candidates_unlocked(valid_candidates) + return + except (OSError, TypeError, ValueError, json.JSONDecodeError): + pass + try: + self._promote_latest_active_state() + except OSError: + return + + def _revision_artifact_signatures( self, - *, - state: RunState, - run_state_path: Path, - emit: Callable[..., None], - ) -> str: - run_state_path.parent.mkdir(parents=True, exist_ok=True) - dump_state(state, str(run_state_path)) - run_state_text = run_state_path.read_text(encoding="utf-8") - self.run_state_path().parent.mkdir(parents=True, exist_ok=True) - self._write_text(self.run_state_path(), run_state_text) - emit("state.save", run_id=state.run_id, path=str(run_state_path)) - emit("state.fingerprint.after_save", run_id=state.run_id, state_fingerprint=self._state_fingerprint(state)) - return run_state_text - - def _write_runtime_map_text( + indexed_states: Sequence[_IndexedState], + ) -> dict[str, dict[str, dict[str, object]]]: + return { + indexed.candidate.run_id: self._artifact_signatures(indexed.candidate.state_path.parent) + for indexed in indexed_states + } + + def _artifact_signatures(self, root: Path) -> dict[str, dict[str, object]]: + return { + artifact_name: self._path_lstat_signature(root / artifact_name) + for artifact_name in self._RUN_ALIAS_ARTIFACT_NAMES + } + + def _reconcile_run_aliases_unlocked( self, - *, - state: RunState, - runtime_map: dict[str, object], - runtime_map_path: Path, - emit_path: Path, - emit: Callable[..., None], - ) -> str: - runtime_map_text = self._json_text(runtime_map) - self._write_run_and_current_artifact( - run_path=runtime_map_path, - current_path=self.runtime_map_path(), - text=runtime_map_text, - ) - emit("runtime_map.write", path=str(emit_path)) - emit( - "runtime_map.fingerprint", - run_id=state.run_id, - runtime_map_fingerprint=self._text_fingerprint(runtime_map_text), - ) - return runtime_map_text + indexed_states: Sequence[_IndexedState], + ) -> None: + for indexed in indexed_states: + candidate = indexed.candidate + if self._run_aliases_match_manifest(candidate): + continue + source_dir = candidate.state_path.parent + run_dir = self.run_dir_path(candidate.run_id) + for artifact_name in self._RUN_ALIAS_ARTIFACT_NAMES: + source = source_dir / artifact_name + alias = run_dir / artifact_name + if not source.is_file() or source.is_symlink(): + alias.unlink(missing_ok=True) + continue + self._write_text(alias, source.read_text(encoding="utf-8")) + self._write_run_alias_manifest(candidate) + + def _remove_current_aliases(self) -> None: + roots = [self.runtime_root] + if self.compat_mode == self.COMPAT_READ_WRITE and self.runtime_legacy_root != self.runtime_root: + roots.append(self.runtime_legacy_root) + for root in roots: + for name in ( + "run_state.json", + "runtime_map.json", + "ports_manifest.json", + "error_report.json", + "events.jsonl", + "runtime_readiness_report.json", + "current_alias.json", + ): + (root / name).unlink(missing_ok=True) + self._remove_owned_mode_pointers(root) + + @staticmethod + def _remove_pointers_to_state(root: Path, state_path: Path) -> None: + expected = str(state_path.resolve()) + for pointer in root.glob(".last_state*"): + if not pointer.is_file(): + continue + try: + target = pointer.read_text(encoding="utf-8").splitlines()[0].strip() + except (IndexError, OSError, UnicodeError): + continue + try: + resolved_target = str(Path(target).expanduser().resolve()) + except (OSError, RuntimeError, ValueError): + continue + if resolved_target == expected: + pointer.unlink(missing_ok=True) + + def _load_legacy_shell_candidate(self, path: Path, *, allowed_root: str) -> RunState | None: + if not path.is_file(): + return None + try: + return load_legacy_shell_state(str(path), allowed_root=allowed_root) + except Exception: + return None @staticmethod def _json_text(payload: Mapping[str, object]) -> str: @@ -365,17 +1607,75 @@ def _events_text(events: list[dict[str, object]]) -> str: return "".join(json.dumps(event, sort_keys=True) + "\n" for event in events) @staticmethod - def _write_run_and_current_artifact(*, run_path: Path, current_path: Path, text: str) -> None: - run_path.parent.mkdir(parents=True, exist_ok=True) - current_path.parent.mkdir(parents=True, exist_ok=True) - RuntimeStateRepository._write_text(run_path, text) - RuntimeStateRepository._write_text(current_path, text) + def _write_text(path: Path, text: str) -> None: + scavenge_atomic_write_temps(path) + atomic_write_text(path, text) + + @contextmanager + def _repository_lock(self, *, exclusive: bool) -> Iterator[None]: + self._validate_runtime_roots() + with advisory_file_lock(self._lock_path, exclusive=exclusive): + self._validate_runtime_roots() + yield + + def _validate_runtime_roots(self) -> None: + self._runtime_root_identity = self._validate_directory_identity( + self.runtime_root, + expected=self._runtime_root_identity, + create=True, + label="runtime_root", + ) + if self.compat_mode == self.SCOPED_ONLY: + return + self._legacy_root_identity = self._validate_directory_identity( + self.runtime_legacy_root, + expected=self._legacy_root_identity, + create=self.compat_mode == self.COMPAT_READ_WRITE, + label="runtime_legacy_root", + ) + + @staticmethod + def _existing_directory_identity(path: Path) -> tuple[int, int] | None: + if not path.exists(): + return None + if path.is_symlink() or not path.is_dir(): + raise ValueError(f"runtime root is not a real directory: {path}") + metadata = path.stat() + return metadata.st_dev, metadata.st_ino @staticmethod - def _write_text(path: Path, text: str) -> None: - _ = path.write_text(text, encoding="utf-8") + def _validate_directory_identity( + path: Path, + *, + expected: tuple[int, int] | None, + create: bool, + label: str, + ) -> tuple[int, int] | None: + if path.is_symlink() or path.resolve(strict=False) != path: + raise RuntimeError(f"{label} changed identity or became a symlink: {path}") + if create: + durable_mkdir(path) + if not path.exists(): + return expected + if not path.is_dir(): + raise RuntimeError(f"{label} is not a directory: {path}") + metadata = path.stat() + current = metadata.st_dev, metadata.st_ino + if expected is not None and current != expected: + raise RuntimeError(f"{label} changed identity: {path}") + return current def purge(self, *, aggressive: bool = False) -> None: + with self._repository_lock(exclusive=True): + self._purge_unlocked(aggressive=aggressive) + + def _purge_unlocked(self, *, aggressive: bool) -> None: + self._retry_pending_revision_cleanups_unlocked() + self.run_index.purge() + self._write_text( + self._registry_marker_path, + self._json_text({"version": 1, "runtime_scope_id": self.runtime_scope_id}), + ) scoped_paths = ( self.run_state_path(), self.runtime_map_path(), @@ -383,15 +1683,18 @@ def purge(self, *, aggressive: bool = False) -> None: self.error_report_path(), self.runtime_root / "events.jsonl", self.runtime_root / "runtime_readiness_report.json", - self.runtime_root / ".last_state", - self.runtime_root / ".last_state.main", + self._alias_manifest_path, ) for path in scoped_paths: path.unlink(missing_ok=True) - for pointer in self.runtime_root.glob(".last_state.trees.*"): - pointer.unlink(missing_ok=True) + diagnostics_dir = self.runtime_root / "diagnostics" + if diagnostics_dir.is_symlink(): + diagnostics_dir.unlink(missing_ok=True) + elif diagnostics_dir.is_dir(): + shutil.rmtree(diagnostics_dir) + self._remove_owned_mode_pointers(self.runtime_root) - if self.compat_mode != self.SCOPED_ONLY: + if self.compat_mode == self.COMPAT_READ_WRITE: legacy_paths = ( self.runtime_legacy_root / "run_state.json", self.runtime_legacy_root / "runtime_map.json", @@ -399,44 +1702,35 @@ def purge(self, *, aggressive: bool = False) -> None: self.runtime_legacy_root / "error_report.json", self.runtime_legacy_root / "events.jsonl", self.runtime_legacy_root / "runtime_readiness_report.json", - self.runtime_legacy_root / ".last_state", - self.runtime_legacy_root / ".last_state.main", ) for path in legacy_paths: path.unlink(missing_ok=True) - for pointer in self.runtime_legacy_root.glob(".last_state.trees.*"): - pointer.unlink(missing_ok=True) + self._remove_owned_mode_pointers(self.runtime_legacy_root) if aggressive: - for runs_dir in (self.runtime_root / "runs", self.runtime_legacy_root / "runs"): + runs_dirs = [self.runtime_root / "runs"] + if self.compat_mode == self.COMPAT_READ_WRITE and self.runtime_legacy_root != self.runtime_root: + runs_dirs.append(self.runtime_legacy_root / "runs") + for runs_dir in runs_dirs: if runs_dir.is_dir(): - shutil.rmtree(runs_dir, ignore_errors=True) - - def _ordered_scoped_pointers(self, *, mode: str | None) -> list[Path]: - main_pointer = self.runtime_root / ".last_state.main" - tree_pointers = sorted(self.runtime_root.glob(".last_state.trees.*")) - generic_pointer = self.runtime_root / ".last_state" - if mode == "trees": - return [*tree_pointers, generic_pointer, main_pointer] - if mode == "main": - return [main_pointer, generic_pointer, *tree_pointers] - return [main_pointer, *tree_pointers, generic_pointer] - - def _ordered_legacy_pointers(self, *, mode: str | None) -> list[Path]: - main_pointer = self.runtime_legacy_root / ".last_state.main" - tree_pointers = sorted(self.runtime_legacy_root.glob(".last_state.trees.*")) - generic_pointer = self.runtime_legacy_root / ".last_state" - if mode == "trees": - return [*tree_pointers, generic_pointer, main_pointer] - if mode == "main": - return [main_pointer, generic_pointer, *tree_pointers] - return [main_pointer, *tree_pointers, generic_pointer] - - def _state_matches_scope(self, state: RunState) -> bool: - scope = state.metadata.get("repo_scope_id") - if isinstance(scope, str) and scope: - return scope == self.runtime_scope_id - return True + shutil.rmtree(runs_dir) + cleanup_dir = self.runtime_root / self._REVISION_CLEANUP_DIR_NAME + if cleanup_dir.is_symlink(): + cleanup_dir.unlink(missing_ok=True) + elif cleanup_dir.is_dir(): + shutil.rmtree(cleanup_dir) + self._scavenge_atomic_temps() + + def _scavenge_atomic_temps(self) -> None: + roots = {self.runtime_root} + if self.compat_mode == self.COMPAT_READ_WRITE: + roots.add(self.runtime_legacy_root) + for root in roots: + if not root.is_dir(): + continue + for temporary_path in root.rglob(".*.tmp"): + if temporary_path.is_file() and not temporary_path.is_symlink(): + temporary_path.unlink(missing_ok=True) def _context_ports_payload(self, context: object) -> dict[str, object]: ports = getattr(context, "ports", {}) @@ -457,39 +1751,6 @@ def _context_ports_payload(self, context: object) -> dict[str, object]: "ports": payload_ports, } - def _write_scoped_pointers(self, *, run_state_path: Path, state: RunState, contexts: list[object]) -> None: - self._write_mode_pointers( - root=self.runtime_root, - run_state_path=run_state_path, - mode=state.mode, - project_names=self._context_names(contexts), - ) - - def _write_legacy_compat( - self, - *, - run_state_path: Path, - run_state_text: str, - runtime_map_text: str, - ports_manifest_text: str, - error_report_text: str, - events_text: str, - state: RunState, - contexts: list[object], - ) -> None: - self.runtime_legacy_root.mkdir(parents=True, exist_ok=True) - self._write_text(self.runtime_legacy_root / "run_state.json", run_state_text) - self._write_text(self.runtime_legacy_root / "runtime_map.json", runtime_map_text) - self._write_text(self.runtime_legacy_root / "ports_manifest.json", ports_manifest_text) - self._write_text(self.runtime_legacy_root / "error_report.json", error_report_text) - self._write_text(self.runtime_legacy_root / "events.jsonl", events_text) - self._write_mode_pointers( - root=self.runtime_legacy_root, - run_state_path=run_state_path, - mode=state.mode, - project_names=self._context_names(contexts), - ) - @staticmethod def _context_names(contexts: list[object]) -> list[str]: names: set[str] = set() @@ -499,28 +1760,115 @@ def _context_names(contexts: list[object]) -> list[str]: names.add(name) return sorted(names, key=str.lower) - def _project_names_from_state(self, state: RunState) -> list[str]: + def _prepare_state_for_save( + self, + state: RunState, + *, + project_names: Sequence[str], + contexts: Sequence[object] = (), + authoritative_projects: bool = False, + ) -> None: + _ = self.run_dir_path(state.run_id) + if state.mode not in {"main", "trees"}: + raise ValueError("state mode must be 'main' or 'trees'") + existing_scope = state.metadata.get("repo_scope_id") + if isinstance(existing_scope, str) and existing_scope and existing_scope != self.runtime_scope_id: + raise ValueError("state repo_scope_id does not match this repository") + state.metadata["repo_scope_id"] = self.runtime_scope_id + + canonical_names = sorted( + {str(name).strip() for name in project_names if str(name).strip()}, + key=str.casefold, + ) + if canonical_names or authoritative_projects: + state.metadata["project_names"] = canonical_names + + existing_roots = state.metadata.get("project_roots") + project_roots = dict(existing_roots) if isinstance(existing_roots, Mapping) else {} + for context in contexts: + name = str(getattr(context, "name", "")).strip() + root = getattr(context, "root", None) + if name and root is not None and str(root).strip(): + project_roots[name] = str(Path(str(root)).expanduser().resolve(strict=False)) + if authoritative_projects: + normalized_names = normalized_project_names(canonical_names) + project_roots = { + str(name): root + for name, root in project_roots.items() + if str(name).strip().casefold() in normalized_names + } + if project_roots: + state.metadata["project_roots"] = project_roots + elif authoritative_projects: + state.metadata.pop("project_roots", None) + if canonical_names or authoritative_projects: + state.metadata = filter_project_scoped_metadata( + state.metadata, + canonical_names, + case_sensitive=False, + ) + state.metadata["project_names"] = canonical_names + + def _runtime_project_names_from_state(self, state: RunState) -> list[str]: names: set[str] = set() - for project in state.requirements: - normalized = str(project).strip() + for storage_name, requirements in state.requirements.items(): + normalized = str(getattr(requirements, "project", "") or storage_name).strip() if normalized: names.add(normalized) - for service_name in state.services: - project_name = self._project_name_from_service_name(service_name) + for service_name, service in state.services.items(): + project_name = self._service_project_name(service_name, service) if project_name: names.add(project_name) + non_main_names = {name for name in names if name.casefold() != "main"} + if state.mode == "trees" and non_main_names: + names = non_main_names if state.mode == "main": names.add("Main") - return sorted(names, key=str.lower) + if names: + return sorted(names, key=str.casefold) + if bool(state.metadata.get("dashboard_runs_disabled")) or bool( + state.metadata.get(DASHBOARD_STOPPED_SERVICES_KEY) + ): + return self._project_names_from_state(state) + return [] @staticmethod - def _text_fingerprint(text: str) -> str: - return hashlib.sha256(text.encode("utf-8")).hexdigest() + def _source_run_ids(state: RunState) -> list[str]: + raw = state.metadata.get("state_source_run_ids") + values = raw if isinstance(raw, list) else [] + return [str(run_id).strip() for run_id in values if str(run_id).strip()] - @staticmethod - def _state_fingerprint(state: RunState) -> str: - serialized = json.dumps(state_to_dict(state), sort_keys=True) - return RuntimeStateRepository._text_fingerprint(serialized) + def _project_names_from_state(self, state: RunState) -> list[str]: + names: set[str] = set() + metadata_names = state.metadata.get("project_names") + if isinstance(metadata_names, Sequence) and not isinstance(metadata_names, (str, bytes)): + for project in metadata_names: + normalized = str(project).strip() + if normalized: + names.add(normalized) + metadata_roots = state.metadata.get("project_roots") + if isinstance(metadata_roots, Mapping): + for project in metadata_roots: + normalized = str(project).strip() + if normalized: + names.add(normalized) + if names: + return sorted(names, key=str.casefold) + for storage_name, requirements in state.requirements.items(): + normalized = str(getattr(requirements, "project", "") or storage_name).strip() + if normalized: + names.add(normalized) + for service_name, service in state.services.items(): + project_name = str(getattr(service, "project", "") or "").strip() + if not project_name: + project_name = self._project_name_from_service_name(service_name) or "" + if project_name: + names.add(project_name) + if state.mode == "trees" and len(names) > 1: + names.discard("Main") + if state.mode == "main": + names.add("Main") + return sorted(names, key=str.lower) @staticmethod def _project_name_from_service_name(service_name: str) -> str | None: @@ -536,39 +1884,149 @@ def _project_name_from_service_name(service_name: str) -> str | None: return project_name or None return None - def _runtime_update_run_state_path(self, state: RunState) -> Path: - run_id = str(getattr(state, "run_id", "")).strip() - if not run_id: - return self.run_state_path() - return self.run_dir_path(run_id) / "run_state.json" - - def _write_mode_pointers( + def _service_project_name(self, service_name: str, service: ServiceRecord) -> str: + explicit_project = str(getattr(service, "project", "") or "").strip() + if explicit_project: + return explicit_project + return self._project_name_from_service_name(service_name) or "" + + def _contained_child(self, root: Path, component: str, *, label: str) -> Path: + runtime_root = self.runtime_root.resolve() + if root.is_symlink(): + raise ValueError(f"{label} root must not be a symlink") + resolved_root = root.resolve() + try: + resolved_root.relative_to(runtime_root) + except ValueError as exc: + raise ValueError(f"{label} root escapes runtime_root") from exc + candidate = root / component + if candidate.is_symlink(): + raise ValueError(f"{label} must not be a symlink") + if candidate.resolve().parent != resolved_root: + raise ValueError(f"{label} escapes its expected directory") + return candidate + + def _rebuild_mode_pointers( self, - *, root: Path, - run_state_path: Path, - mode: str, - project_names: list[str], + indexed_states: Sequence[_IndexedState], ) -> None: - root.mkdir(parents=True, exist_ok=True) - pointer_text = str(run_state_path) + "\n" - self._write_text(root / ".last_state", pointer_text) - existing_tree_pointers = sorted(root.glob(".last_state.trees.*")) - if mode == "main": - self._write_text(root / ".last_state.main", pointer_text) - return + desired = self._desired_mode_pointers(indexed_states) + stale_names = self._owned_pointer_names(root).difference(desired) + for name, target in desired.items(): + self._write_text(root / name, target + "\n") + for name in stale_names: + (root / name).unlink(missing_ok=True) + self._write_text( + self._pointer_manifest_path(root), + self._json_text( + { + "version": 1, + "runtime_scope_id": self.runtime_scope_id, + "pointers": desired, + } + ), + ) - expected_tree_pointer_names: set[str] = set() - for project_name in project_names: - pointer_name = self._tree_pointer_name(project_name) - if pointer_name is None: + def _desired_mode_pointers( + self, + indexed_states: Sequence[_IndexedState], + ) -> dict[str, str]: + if not indexed_states: + return {} + latest_active = max( + indexed_states, + key=lambda indexed: indexed.candidate.activation_sequence, + ) + desired = {".last_state": str(latest_active.candidate.state_path)} + owned_tree_projects: set[str] = set() + latest_main = max( + (indexed for indexed in indexed_states if indexed.candidate.mode == "main"), + key=lambda indexed: indexed.candidate.activation_sequence, + default=None, + ) + if latest_main is not None: + desired[".last_state.main"] = str(latest_main.candidate.state_path) + for indexed in indexed_states: + candidate = indexed.candidate + target = str(candidate.state_path) + if candidate.mode != "trees": continue - expected_tree_pointer_names.add(pointer_name) - self._write_text(root / pointer_name, pointer_text) + display_names = {name.casefold(): name for name in self._project_names_from_state(indexed.state)} + for project_key in candidate.project_names: + if project_key in owned_tree_projects: + continue + owned_tree_projects.add(project_key) + display_name = display_names.get(project_key, project_key) + pointer_name = self._tree_pointer_name(display_name) + if pointer_name is None: + continue + if pointer_name in desired and desired[pointer_name] != target: + digest = text_fingerprint(display_name)[:10] + pointer_name = f"{pointer_name}-{digest}" + desired[pointer_name] = target + return desired + + def _mode_pointers_match( + self, + root: Path, + indexed_states: Sequence[_IndexedState], + ) -> bool: + desired = self._desired_mode_pointers(indexed_states) + try: + payload = json.loads(self._pointer_manifest_path(root).read_text(encoding="utf-8")) + if not isinstance(payload, Mapping) or payload.get("pointers") != desired: + return False + if self._owned_pointer_names(root) != set(desired): + return False + return all((root / name).read_text(encoding="utf-8") == target + "\n" for name, target in desired.items()) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return False + + def _remove_owned_mode_pointers(self, root: Path) -> None: + for name in self._owned_pointer_names(root): + (root / name).unlink(missing_ok=True) + self._pointer_manifest_path(root).unlink(missing_ok=True) + + def _owned_pointer_names(self, root: Path) -> set[str]: + names: set[str] = set() + try: + payload = json.loads(self._pointer_manifest_path(root).read_text(encoding="utf-8")) + raw_pointers = payload.get("pointers") if isinstance(payload, Mapping) else None + if isinstance(raw_pointers, Mapping): + for raw_name in raw_pointers: + name = str(raw_name) + if name not in {".last_state", ".last_state.main"} and not name.startswith(".last_state.trees."): + continue + if root == self.runtime_root or self._pointer_targets_runtime_root(root / name): + names.add(name) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + pass + for pointer in root.glob(".last_state*"): + if pointer.name not in {".last_state", ".last_state.main"} and not pointer.name.startswith( + ".last_state.trees." + ): + continue + if root == self.runtime_root or self._pointer_targets_runtime_root(pointer): + names.add(pointer.name) + return names - for pointer in existing_tree_pointers: - if pointer.name not in expected_tree_pointer_names: - pointer.unlink(missing_ok=True) + def _pointer_targets_runtime_root(self, pointer: Path) -> bool: + if pointer.is_symlink() or not pointer.is_file(): + return False + try: + target = Path(pointer.read_text(encoding="utf-8").splitlines()[0].strip()).resolve() + target.relative_to(self.runtime_root) + except (IndexError, OSError, RuntimeError, ValueError): + return False + return True + + def _pointer_manifest_path(self, root: Path) -> Path: + if root == self.runtime_root: + return root / "pointer_manifest.json" + scope_stem = safe_artifact_stem(self.runtime_scope_id, fallback="scope") + scope_hash = text_fingerprint(self.runtime_scope_id)[:10] + return root / f".pointer_manifest.{scope_stem}-{scope_hash}.json" @staticmethod def _tree_pointer_name(project_name: str) -> str | None: @@ -576,4 +2034,7 @@ def _tree_pointer_name(project_name: str) -> str | None: if not normalized_project or normalized_project.lower() == "main": return None pointer_suffix = safe_artifact_stem(normalized_project, fallback="project") + if pointer_suffix != normalized_project: + digest = text_fingerprint(normalized_project)[:10] + pointer_suffix = f"{pointer_suffix}-{digest}" return f".last_state.trees.{pointer_suffix}" diff --git a/python/envctl_engine/state/run_index.py b/python/envctl_engine/state/run_index.py new file mode 100644 index 000000000..b5ae0e3f6 --- /dev/null +++ b/python/envctl_engine/state/run_index.py @@ -0,0 +1,582 @@ +from __future__ import annotations + +import json +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Any + +from envctl_engine.state.fingerprints import file_fingerprint +from envctl_engine.state.persistence import ( + advisory_file_lock, + atomic_write_text, + durable_mkdir, + require_path_component, + scavenge_atomic_write_temps, +) + + +_INDEX_VERSION = 4 + + +def _normalize_mode(mode: str | None) -> str | None: + normalized = str(mode or "").strip().casefold() + return normalized or None + + +def _normalize_project_names(project_names: Sequence[str]) -> tuple[str, ...]: + return tuple(sorted({name.strip().casefold() for name in project_names if name.strip()})) + + +@dataclass(frozen=True, slots=True) +class StateSelector: + mode: str | None + project_names: tuple[str, ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "mode", _normalize_mode(self.mode)) + object.__setattr__(self, "project_names", _normalize_project_names(self.project_names)) + + +@dataclass(frozen=True, slots=True) +class RunIndexCandidate: + state_path: Path + run_id: str + mode: str + project_names: tuple[str, ...] + sequence: int + state_fingerprint: str = "" + runtime_map_fingerprint: str | None = None + activation_sequence: int = 0 + + +@dataclass(frozen=True, slots=True) +class _RegistrySnapshot: + generation: int + entries: tuple[RunIndexCandidate, ...] + retired_run_ids: frozenset[str] + + +class RunIndex: + """Selection-aware registry of active project ownership by run artifact.""" + + def __init__(self, *, runtime_root: Path, runtime_dir: Path, runtime_scope_id: str) -> None: + self._input_runtime_root = runtime_root.absolute() + if self._input_runtime_root.is_symlink(): + raise ValueError("runtime_root must not be a symlink") + self.runtime_dir = runtime_dir.resolve() + self.runtime_root = runtime_root.resolve() + self.runtime_scope_id = runtime_scope_id.strip() + if not self.runtime_scope_id: + raise ValueError("runtime_scope_id must not be empty") + if not self._is_within_runtime_dir(self.runtime_root): + raise ValueError("runtime_root must be contained by runtime_dir") + self.index_path = self.runtime_root / "run_index.json" + self.backup_path = self.runtime_root / "run_index.backup.json" + self._lock_path = self.runtime_root / ".run_index.lock" + self._runtime_root_identity = self._existing_directory_identity() + + def record( + self, + *, + state_path: Path, + run_id: str, + mode: str, + project_names: Sequence[str], + supersede_run_ids: Sequence[str] = (), + ) -> None: + normalized_run_id = require_path_component(run_id, label="run_id") + resolved_state_path = self._validated_revision_state_path(state_path, run_id=normalized_run_id) + normalized_mode = _normalize_mode(mode) + if normalized_mode is None: + raise ValueError("mode must not be empty") + normalized_projects = _normalize_project_names(project_names) + state_fingerprint = file_fingerprint(resolved_state_path) + runtime_map_path = resolved_state_path.parent / "runtime_map.json" + runtime_map_fingerprint = ( + file_fingerprint(runtime_map_path) + if runtime_map_path.is_file() and not runtime_map_path.is_symlink() + else None + ) + superseded = { + require_path_component(candidate_run_id, label="superseded run_id") + for candidate_run_id in supersede_run_ids + } + superseded.discard(normalized_run_id) + + with self._lock(exclusive=True): + snapshot = self._read_registry() + entries = list(snapshot.entries) if snapshot is not None else [] + retired_run_ids = set(snapshot.retired_run_ids) if snapshot is not None else set() + if normalized_run_id in retired_run_ids: + raise RuntimeError(f"run_id is retired and cannot publish new state: {normalized_run_id}") + latest_sequence = max((entry.sequence for entry in entries), default=0) + latest_activation_sequence = max( + (entry.activation_sequence for entry in entries), + default=0, + ) + existing_entry = next((entry for entry in entries if entry.run_id == normalized_run_id), None) + if not normalized_projects and existing_entry is not None: + normalized_projects = existing_entry.project_names + entries = [ + entry for entry in entries if entry.run_id != normalized_run_id and entry.run_id not in superseded + ] + retired_run_ids.update(superseded) + entries = self._subtract_replaced_project_owners( + entries, + mode=normalized_mode, + project_names=normalized_projects, + retired_run_ids=retired_run_ids, + ) + sequence = existing_entry.sequence if existing_entry is not None else latest_sequence + 1 + activates_current_mode = existing_entry is None or self._entry_owns_any_project( + existing_entry, + entries, + normalized_projects, + ) + activation_sequence = ( + latest_activation_sequence + 1 if activates_current_mode else existing_entry.activation_sequence + ) + entries.append( + RunIndexCandidate( + state_path=resolved_state_path, + run_id=normalized_run_id, + mode=normalized_mode, + project_names=normalized_projects, + sequence=sequence, + state_fingerprint=state_fingerprint, + runtime_map_fingerprint=runtime_map_fingerprint, + activation_sequence=activation_sequence, + ) + ) + self._write_entries(entries, retired_run_ids=retired_run_ids) + + @staticmethod + def _subtract_replaced_project_owners( + entries: Sequence[RunIndexCandidate], + *, + mode: str, + project_names: Sequence[str], + retired_run_ids: set[str], + ) -> list[RunIndexCandidate]: + replaced_projects = frozenset(project_names) + if not replaced_projects: + return list(entries) + + retained: list[RunIndexCandidate] = [] + for entry in entries: + if entry.mode != mode or replaced_projects.isdisjoint(entry.project_names): + retained.append(entry) + continue + remaining_projects = tuple(project for project in entry.project_names if project not in replaced_projects) + if remaining_projects: + retained.append(replace(entry, project_names=remaining_projects)) + else: + retired_run_ids.add(entry.run_id) + return retained + + def remove(self, run_id: str) -> bool: + return self.remove_many([run_id]) + + def remove_many(self, run_ids: Sequence[str]) -> bool: + normalized_run_ids = {require_path_component(run_id, label="run_id") for run_id in run_ids} + with self._lock(exclusive=True): + snapshot = self._read_registry() + entries = list(snapshot.entries) if snapshot is not None else [] + retired_run_ids = set(snapshot.retired_run_ids) if snapshot is not None else set() + retained = [entry for entry in entries if entry.run_id not in normalized_run_ids] + removed_active_entry = len(retained) != len(entries) + added_tombstone = not normalized_run_ids.issubset(retired_run_ids) + if not removed_active_entry and not added_tombstone: + return False + retired_run_ids.update(normalized_run_ids) + self._write_entries(retained, retired_run_ids=retired_run_ids) + return removed_active_entry + + def candidates(self, selector: StateSelector) -> list[RunIndexCandidate]: + with self._lock(exclusive=False): + entries = self._read_entries() + if entries is None: + return [] + + selected_projects = frozenset(selector.project_names) + ranked: list[tuple[int, int, str, RunIndexCandidate]] = [] + for entry in entries: + if selector.mode is not None and entry.mode != selector.mode: + continue + relation_rank = self._selection_rank( + indexed=frozenset(entry.project_names), + selected=selected_projects, + ) + if relation_rank is None: + continue + ranked.append((-entry.sequence, relation_rank, str(entry.state_path), entry)) + ranked.sort(key=lambda item: (item[0], item[1], item[2])) + return [entry for _, _, _, entry in ranked] + + def candidate_paths(self, selector: StateSelector) -> list[Path]: + return [candidate.state_path for candidate in self.candidates(selector)] + + def purge(self) -> None: + with self._lock(exclusive=True): + snapshot = self._read_registry() + retired_run_ids = set(snapshot.retired_run_ids) if snapshot is not None else set() + if snapshot is not None: + retired_run_ids.update(entry.run_id for entry in snapshot.entries) + self._write_entries([], retired_run_ids=retired_run_ids) + + def needs_rebuild(self) -> bool: + with self._lock(exclusive=False): + return self._read_snapshot(self.index_path) is None and self._read_snapshot(self.backup_path) is None + + def repair_copies(self) -> None: + with self._lock(exclusive=True): + primary = self._read_snapshot(self.index_path) + backup = self._read_snapshot(self.backup_path) + if primary is not None and backup is not None and primary == backup: + return + snapshot = self._read_registry() + if snapshot is not None: + self._write_entries( + list(snapshot.entries), + retired_run_ids=set(snapshot.retired_run_ids), + ) + + def initialize_empty(self) -> None: + with self._lock(exclusive=True): + self._write_entries([], retired_run_ids=set()) + + def replace_all(self, entries: Sequence[RunIndexCandidate]) -> None: + with self._lock(exclusive=True): + self._write_entries( + [self._candidate_with_fingerprints(entry) for entry in entries], + retired_run_ids=set(), + ) + + @staticmethod + def _selection_rank(*, indexed: frozenset[str], selected: frozenset[str]) -> int | None: + if not selected: + return 0 + if not indexed: + return None + if indexed == selected: + return 0 + if indexed.issuperset(selected): + return 1 + if indexed.issubset(selected): + return 2 + if indexed.intersection(selected): + return 3 + return None + + @staticmethod + def _entry_owns_any_project( + existing_entry: RunIndexCandidate, + other_entries: Sequence[RunIndexCandidate], + updated_projects: Sequence[str], + ) -> bool: + if not updated_projects: + return True + for project in updated_projects: + if not any( + other.mode == existing_entry.mode + and other.sequence > existing_entry.sequence + and project in other.project_names + for other in other_entries + ): + return True + return False + + @contextmanager + def _lock(self, *, exclusive: bool) -> Iterator[None]: + self._validate_runtime_root() + with advisory_file_lock(self._lock_path, exclusive=exclusive): + self._validate_runtime_root() + yield + + def _existing_directory_identity(self) -> tuple[int, int] | None: + if not self.runtime_root.exists(): + return None + if self.runtime_root.is_symlink() or not self.runtime_root.is_dir(): + raise ValueError(f"runtime_root is not a real directory: {self.runtime_root}") + metadata = self.runtime_root.stat() + return metadata.st_dev, metadata.st_ino + + def _validate_runtime_root(self) -> None: + if self.runtime_root.is_symlink() or self.runtime_root.resolve(strict=False) != self.runtime_root: + raise RuntimeError(f"runtime_root changed identity or became a symlink: {self.runtime_root}") + durable_mkdir(self.runtime_root) + metadata = self.runtime_root.stat() + current = metadata.st_dev, metadata.st_ino + if self._runtime_root_identity is not None and current != self._runtime_root_identity: + raise RuntimeError(f"runtime_root changed identity: {self.runtime_root}") + self._runtime_root_identity = current + + def _read_entries(self) -> list[RunIndexCandidate] | None: + snapshot = self._read_registry() + return list(snapshot.entries) if snapshot is not None else None + + def _read_registry(self) -> _RegistrySnapshot | None: + primary = self._read_snapshot(self.index_path) + backup = self._read_snapshot(self.backup_path) + self._reject_equal_generation_divergence(primary, backup) + snapshots = [snapshot for snapshot in (primary, backup) if snapshot is not None] + if not snapshots: + return None + return max(snapshots, key=lambda snapshot: snapshot.generation) + + @staticmethod + def _reject_equal_generation_divergence( + primary: _RegistrySnapshot | None, + backup: _RegistrySnapshot | None, + ) -> None: + if primary is not None and backup is not None and primary.generation == backup.generation and primary != backup: + raise RuntimeError( + "Run registry primary and backup diverged at generation " + f"{primary.generation}; refusing to choose one copy" + ) + + def _read_snapshot(self, path: Path) -> _RegistrySnapshot | None: + if not path.is_file(): + return None + try: + payload = json.loads(path.read_text(encoding="utf-8")) + generation = payload.get("generation") if isinstance(payload, dict) else None + if not isinstance(generation, int) or isinstance(generation, bool) or generation < 1: + return None + entries = self._parse_payload(payload) + raw_retired = payload.get("retired_run_ids") + if not isinstance(raw_retired, list) or not all(isinstance(run_id, str) for run_id in raw_retired): + return None + retired = frozenset(require_path_component(run_id, label="retired run_id") for run_id in raw_retired) + if retired.intersection(entry.run_id for entry in entries): + return None + return _RegistrySnapshot(generation, tuple(entries), retired) + except (OSError, RuntimeError, TypeError, ValueError, json.JSONDecodeError): + return None + + def _parse_payload(self, payload: Any) -> list[RunIndexCandidate]: + if not isinstance(payload, dict): + raise ValueError("run index must be a JSON object") + version = payload.get("version") + if version not in {2, 3, _INDEX_VERSION}: + raise ValueError("unsupported run index version") + if payload.get("runtime_scope_id") != self.runtime_scope_id: + raise ValueError("run index scope does not match this repository") + raw_entries = payload.get("entries") + if not isinstance(raw_entries, list): + raise ValueError("run index entries must be a list") + + entries: list[RunIndexCandidate] = [] + seen_run_ids: set[str] = set() + for raw_entry in raw_entries: + if not isinstance(raw_entry, dict) or raw_entry.get("runtime_scope_id") != self.runtime_scope_id: + raise ValueError("invalid run index entry scope") + run_id = raw_entry.get("run_id") + mode = _normalize_mode(raw_entry.get("mode") if isinstance(raw_entry.get("mode"), str) else None) + raw_projects = raw_entry.get("project_names") + sequence = raw_entry.get("sequence") + raw_state_path = raw_entry.get("state_path") + raw_state_fingerprint = raw_entry.get("state_fingerprint") + raw_runtime_map_fingerprint = raw_entry.get("runtime_map_fingerprint") + raw_activation_sequence = raw_entry.get("activation_sequence") + if ( + not isinstance(run_id, str) + or not run_id.strip() + or mode is None + or not isinstance(raw_projects, list) + or not all(isinstance(project, str) for project in raw_projects) + or not isinstance(sequence, int) + or isinstance(sequence, bool) + or sequence < 1 + or not isinstance(raw_state_path, str) + or not raw_state_path.strip() + ): + raise ValueError("invalid run index entry") + normalized_run_id = require_path_component(run_id, label="indexed run_id") + if normalized_run_id in seen_run_ids: + raise ValueError("duplicate run id in run index") + state_path = self._validated_revision_state_path(Path(raw_state_path), run_id=normalized_run_id) + if version == 2: + state_fingerprint = file_fingerprint(state_path) + runtime_map_path = state_path.parent / "runtime_map.json" + runtime_map_fingerprint = ( + file_fingerprint(runtime_map_path) + if runtime_map_path.is_file() and not runtime_map_path.is_symlink() + else None + ) + else: + if not self._valid_fingerprint(raw_state_fingerprint) or ( + raw_runtime_map_fingerprint is not None and not self._valid_fingerprint(raw_runtime_map_fingerprint) + ): + raise ValueError("invalid run index artifact fingerprint") + state_fingerprint = str(raw_state_fingerprint) + runtime_map_fingerprint = ( + str(raw_runtime_map_fingerprint) if raw_runtime_map_fingerprint is not None else None + ) + if version in {2, 3}: + activation_sequence = sequence + elif ( + not isinstance(raw_activation_sequence, int) + or isinstance(raw_activation_sequence, bool) + or raw_activation_sequence < 1 + ): + raise ValueError("invalid run index activation sequence") + else: + activation_sequence = raw_activation_sequence + seen_run_ids.add(normalized_run_id) + entries.append( + RunIndexCandidate( + state_path=state_path, + run_id=normalized_run_id, + mode=mode, + project_names=_normalize_project_names(raw_projects), + sequence=sequence, + state_fingerprint=state_fingerprint, + runtime_map_fingerprint=runtime_map_fingerprint, + activation_sequence=activation_sequence, + ) + ) + return entries + + def _write_entries( + self, + entries: list[RunIndexCandidate], + *, + retired_run_ids: set[str], + ) -> None: + primary = self._read_snapshot(self.index_path) + backup = self._read_snapshot(self.backup_path) + self._reject_equal_generation_divergence(primary, backup) + existing_generations = [snapshot.generation for snapshot in (primary, backup) if snapshot is not None] + payload = { + "version": _INDEX_VERSION, + "generation": max(existing_generations, default=0) + 1, + "runtime_scope_id": self.runtime_scope_id, + "retired_run_ids": sorted(retired_run_ids), + "entries": [ + { + "state_path": str(entry.state_path), + "run_id": entry.run_id, + "mode": entry.mode, + "project_names": list(entry.project_names), + "sequence": entry.sequence, + "state_fingerprint": entry.state_fingerprint, + "runtime_map_fingerprint": entry.runtime_map_fingerprint, + "activation_sequence": entry.activation_sequence, + "runtime_scope_id": self.runtime_scope_id, + } + for entry in entries + ], + } + text = json.dumps(payload, indent=2, sort_keys=True) + "\n" + errors: list[OSError] = [] + successful_writes = 0 + for path in (self.backup_path, self.index_path): + try: + scavenge_atomic_write_temps(path) + atomic_write_text(path, text) + successful_writes += 1 + except OSError as exc: + errors.append(exc) + expected = _RegistrySnapshot( + generation=payload["generation"], + entries=tuple(entries), + retired_run_ids=frozenset(retired_run_ids), + ) + if any(self._read_snapshot(path) == expected for path in (self.index_path, self.backup_path)): + return + if errors: + raise errors[0] + if successful_writes: + raise RuntimeError("run registry write returned without publishing the expected generation") + + def _validated_revision_state_path(self, state_path: Path, *, run_id: str) -> Path: + if not state_path.is_absolute(): + raise ValueError("indexed state_path must be absolute") + resolved = state_path.resolve() + try: + relative = resolved.relative_to(self.runtime_root) + except ValueError as exc: + raise ValueError("indexed state_path escapes runtime_root") from exc + parts = relative.parts + if len(parts) != 5 or parts[0] != "runs" or parts[1] != run_id or parts[2] != "revisions": + raise ValueError("indexed state_path must identify a run revision") + _ = require_path_component(parts[3], label="revision id") + if parts[4] != "run_state.json": + raise ValueError("indexed state_path must identify run_state.json") + if ".." in state_path.parts or state_path.is_symlink(): + raise ValueError("indexed state_path must be canonical and must not traverse symlinks") + lexical_relative: Path | None = None + lexical_root: Path | None = None + for candidate_root in (self._input_runtime_root, self.runtime_root): + try: + lexical_relative = state_path.relative_to(candidate_root) + lexical_root = candidate_root + break + except ValueError: + continue + if lexical_relative is None or lexical_root is None: + raise ValueError("indexed state_path must be canonical and must not traverse symlinks") + lexical_parent = lexical_root + for component in lexical_relative.parts[:-1]: + lexical_parent /= component + if lexical_parent.is_symlink(): + raise ValueError("indexed state_path must be canonical and must not traverse symlinks") + for parent in ( + self.runtime_root / "runs", + self.runtime_root / "runs" / run_id, + self.runtime_root / "runs" / run_id / "revisions", + resolved.parent, + ): + if parent.is_symlink(): + raise ValueError("indexed state_path must not traverse symlinks") + if not self._is_within_runtime_root(resolved): + raise ValueError("indexed state_path must be canonical inside runtime_root") + return resolved + + def _candidate_with_fingerprints(self, entry: RunIndexCandidate) -> RunIndexCandidate: + state_path = self._validated_revision_state_path(entry.state_path, run_id=entry.run_id) + runtime_map_path = state_path.parent / "runtime_map.json" + return RunIndexCandidate( + state_path=state_path, + run_id=entry.run_id, + mode=entry.mode, + project_names=entry.project_names, + sequence=entry.sequence, + state_fingerprint=entry.state_fingerprint or file_fingerprint(state_path), + runtime_map_fingerprint=( + entry.runtime_map_fingerprint + if entry.runtime_map_fingerprint is not None + else ( + file_fingerprint(runtime_map_path) + if runtime_map_path.is_file() and not runtime_map_path.is_symlink() + else None + ) + ), + activation_sequence=entry.activation_sequence or entry.sequence, + ) + + @staticmethod + def _valid_fingerprint(value: object) -> bool: + return ( + isinstance(value, str) and len(value) == 64 and all(character in "0123456789abcdef" for character in value) + ) + + def _is_within_runtime_dir(self, path: Path) -> bool: + try: + path.relative_to(self.runtime_dir) + except ValueError: + return False + return True + + def _is_within_runtime_root(self, path: Path) -> bool: + try: + path.relative_to(self.runtime_root) + except ValueError: + return False + return True + + +__all__ = ["RunIndex", "RunIndexCandidate", "StateSelector"] diff --git a/python/envctl_engine/state/runtime_map.py b/python/envctl_engine/state/runtime_map.py index a3eac6dad..41316ff3b 100644 --- a/python/envctl_engine/state/runtime_map.py +++ b/python/envctl_engine/state/runtime_map.py @@ -1,11 +1,16 @@ from __future__ import annotations import json +from collections.abc import Mapping from pathlib import Path -from typing import cast -from envctl_engine.shared.services import project_name_from_service_name, service_display_name -from envctl_engine.state.models import RunState +from envctl_engine.shared.services import ( + project_name_from_service_name, + service_display_name, + service_project_name, + service_slug_from_record, +) +from envctl_engine.state.models import RunState, ServiceRecord def build_runtime_map(state: RunState, *, host: str = "localhost") -> dict[str, object]: @@ -24,29 +29,22 @@ def write_runtime_map(path: str, state: RunState) -> None: def build_runtime_projection(state: RunState, *, host: str = "localhost") -> dict[str, dict[str, object]]: runtime_map = build_runtime_map_without_projection(state, host=host) projection: dict[str, dict[str, object]] = {} - projects = cast(dict[str, dict[str, object]], runtime_map["projects"]) - service_records = state.services - for project, ports in projects.items(): - backend_port = ports.get("backend_port") - frontend_port = ports.get("frontend_port") - backend_service = service_records.get(f"{project} Backend") - frontend_service = service_records.get(f"{project} Frontend") - backend_ready = _service_projection_ready(backend_service) - frontend_ready = _service_projection_ready(frontend_service) + projects_raw = runtime_map["projects"] + projects = projects_raw if isinstance(projects_raw, Mapping) else {} + for project, ports_raw in projects.items(): + ports = ports_raw if isinstance(ports_raw, Mapping) else {} + services_raw = ports.get("services") + services = services_raw if isinstance(services_raw, Mapping) else {} + backend_port, backend_url, backend_status = _projected_service_fields(services.get("backend")) + frontend_port, frontend_url, frontend_status = _projected_service_fields(services.get("frontend")) projection[project] = { "backend_port": backend_port, "frontend_port": frontend_port, - "backend_url": (f"http://{host}:{backend_port}" if backend_port is not None and backend_ready else None), - "frontend_url": ( - f"http://{host}:{frontend_port}" if frontend_port is not None and frontend_ready else None - ), - "backend_status": getattr(backend_service, "status", "unknown") - if backend_service is not None - else "unknown", - "frontend_status": getattr(frontend_service, "status", "unknown") - if frontend_service is not None - else "unknown", - "services": ports.get("services", {}), + "backend_url": backend_url, + "frontend_url": frontend_url, + "backend_status": backend_status, + "frontend_status": frontend_status, + "services": services, } return projection @@ -58,48 +56,63 @@ def build_runtime_map_without_projection(state: RunState, *, host: str = "localh service_to_url: dict[str, str] = {} service_to_public_url: dict[str, str] = {} service_to_health_url: dict[str, str] = {} - for service in state.services.values(): - project = str(getattr(service, "project", "") or "").strip() or _project_name_from_service_record(service) + + grouped = _services_by_project_and_slug(state) + reserved_service_keys_by_project: dict[str, set[str]] = {} + for project, service_slug in grouped: + reserved_service_keys_by_project.setdefault(project, set()).add(service_slug) + + for (project, service_slug), instances in sorted( + grouped.items(), + key=lambda item: (item[0][0].casefold(), item[0][1].casefold()), + ): project_entry = projects.setdefault(project, {"backend_port": None, "frontend_port": None, "services": {}}) - port = service.actual_port if service.actual_port is not None else service.requested_port - service_ready = _service_projection_ready(service) - url = f"http://{host}:{port}" if port is not None and service_ready else None - public_url = str(getattr(service, "public_url", "") or "").strip() or url - health_url = str(getattr(service, "health_url", "") or "").strip() or None - service_slug = str(getattr(service, "service_slug", "") or "").strip() or service.type services_entry = project_entry.setdefault("services", {}) - if isinstance(services_entry, dict): - services_entry[service_slug] = { - "name": service.name, - "type": service.type, - "service_slug": service_slug, - "port": port, - "url": url, - "public_url": public_url, - "health_url": health_url, - "status": service.status, - "cwd": service.cwd, - "listener_expected": service.listener_expected, - "requested_port": service.requested_port, - "actual_port": service.actual_port, - "log_path": service.log_path, - "failure_detail": getattr(service, "failure_detail", None), - "critical": getattr(service, "critical", True), - "degraded": getattr(service, "degraded", False), - } - if service.type == "backend": - project_entry["backend_port"] = port - elif service.type == "frontend": - project_entry["frontend_port"] = port - if port is not None: - port_to_service[port] = service.name - service_to_actual_port[service.name] = port + if not isinstance(services_entry, dict): + continue + + ordered_instances = sorted( + instances, + key=lambda item: _service_authority_sort_key(project=project, service_slug=service_slug, item=item), + ) + used_service_keys = set(services_entry) + reserved_service_keys = reserved_service_keys_by_project.get(project, set()) + for index, (state_name, service) in enumerate(ordered_instances): + projection_key = ( + service_slug + if index == 0 + else _duplicate_service_projection_key( + service_slug=service_slug, + state_name=state_name, + used_keys=used_service_keys, + reserved_keys=reserved_service_keys, + ) + ) + used_service_keys.add(projection_key) + payload = _runtime_service_payload( + service=service, + service_slug=service_slug, + host=host, + ) + services_entry[projection_key] = payload + if index == 0 and service_slug in {"backend", "frontend"}: + project_entry[f"{service_slug}_port"] = payload["port"] + + port = payload["port"] + url = payload["url"] + public_url = payload["public_url"] + health_url = payload["health_url"] + if not isinstance(port, int): + continue + service_name = str(payload["name"] or "").strip() or state_name + port_to_service.setdefault(port, service_name) + service_to_actual_port[service_name] = port if url is not None: - service_to_url[service.name] = url + service_to_url[service_name] = str(url) if public_url: - service_to_public_url[service.name] = public_url + service_to_public_url[service_name] = str(public_url) if health_url: - service_to_health_url[service.name] = health_url + service_to_health_url[service_name] = str(health_url) return { "schema_version": "1.0", "backend_mode": "python", @@ -114,6 +127,125 @@ def build_runtime_map_without_projection(state: RunState, *, host: str = "localh } +def _services_by_project_and_slug( + state: RunState, +) -> dict[tuple[str, str], list[tuple[str, ServiceRecord]]]: + grouped: dict[tuple[str, str], list[tuple[str, ServiceRecord]]] = {} + for state_name, service in state.services.items(): + project = str(getattr(service, "project", "") or "").strip() or _project_name_from_service_record(service) + service_slug = service_slug_from_record(service) or "service" + grouped.setdefault((project, service_slug), []).append((state_name, service)) + return grouped + + +def _service_authority_sort_key( + *, + project: str, + service_slug: str, + item: tuple[str, ServiceRecord], +) -> tuple[object, ...]: + state_name, service = item + status = str(getattr(service, "status", "unknown") or "unknown").strip().lower() + status_priority = { + "healthy": 0, + "running": 1, + "starting": 2, + "unknown": 3, + "stale": 4, + "degraded": 5, + "stopped": 6, + "termination_failed": 7, + }.get(status, 3) + explicit_metadata_count = sum( + bool(str(value or "").strip()) + for value in (getattr(service, "project", ""), getattr(service, "service_slug", "")) + ) + canonical_name = f"{project} {service_display_name(service_slug)}".casefold() + service_name = str(getattr(service, "name", "") or "").strip() + actual_port = getattr(service, "actual_port", None) + return ( + status == "termination_failed", + not _service_projection_ready(service), + status_priority, + -explicit_metadata_count, + bool(getattr(service, "degraded", False)), + service_name.casefold() != canonical_name, + not isinstance(actual_port, int) or actual_port <= 0, + service_name.casefold(), + service_name, + state_name.casefold(), + state_name, + int(getattr(service, "pid", 0) or 0), + ) + + +def _duplicate_service_projection_key( + *, + service_slug: str, + state_name: str, + used_keys: set[str], + reserved_keys: set[str], +) -> str: + base = f"{service_slug}@{state_name}" + candidate = base + index = 2 + while candidate in used_keys or candidate in reserved_keys: + candidate = f"{base}#{index}" + index += 1 + return candidate + + +def _runtime_service_payload( + *, + service: ServiceRecord, + service_slug: str, + host: str, +) -> dict[str, object]: + port = service.actual_port if service.actual_port is not None else service.requested_port + url = f"http://{host}:{port}" if port is not None and _service_projection_ready(service) else None + public_url = str(getattr(service, "public_url", "") or "").strip() or url + health_url = str(getattr(service, "health_url", "") or "").strip() or None + return { + "name": service.name, + "type": service.type, + "service_slug": service_slug, + "port": port, + "url": url, + "public_url": public_url, + "health_url": health_url, + "status": service.status, + "cwd": service.cwd, + "listener_expected": service.listener_expected, + "requested_port": service.requested_port, + "actual_port": service.actual_port, + "log_path": service.log_path, + "failure_detail": getattr(service, "failure_detail", None), + "critical": getattr(service, "critical", True), + "degraded": getattr(service, "degraded", False), + "runtime_kind": getattr(service, "runtime_kind", "process"), + "container_id": getattr(service, "container_id", None), + "container_name": getattr(service, "container_name", None), + "container_image": getattr(service, "container_image", None), + "container_launch_token": getattr(service, "container_launch_token", None), + "container_cleanup_pending_since": getattr( + service, + "container_cleanup_pending_since", + None, + ), + } + + +def _projected_service_fields(raw: object) -> tuple[int | None, str | None, str]: + if not isinstance(raw, Mapping): + return None, None, "unknown" + port_raw = raw.get("port") + port = port_raw if isinstance(port_raw, int) else None + url_raw = raw.get("url") + url = str(url_raw) if isinstance(url_raw, str) and url_raw else None + status = str(raw.get("status", "unknown") or "unknown") + return port, url, status + + def _service_projection_ready(service: object | None) -> bool: if service is None: return False @@ -122,6 +254,9 @@ def _service_projection_ready(service: object | None) -> bool: def _project_name_from_service_record(service: object) -> str: + resolved_project = service_project_name(service) + if resolved_project: + return resolved_project name = str(getattr(service, "name", "") or "").strip() service_type = str(getattr(service, "type", "") or "").strip().lower() display_suffix = service_display_name(service_type) diff --git a/python/envctl_engine/state/state_aggregation.py b/python/envctl_engine/state/state_aggregation.py new file mode 100644 index 000000000..d0242985f --- /dev/null +++ b/python/envctl_engine/state/state_aggregation.py @@ -0,0 +1,510 @@ +from __future__ import annotations + +import json +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, replace +from pathlib import Path + +from envctl_engine.dashboard_metadata import DASHBOARD_STOPPED_SERVICES_KEY +from envctl_engine.state.fingerprints import text_fingerprint +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord +from envctl_engine.state.project_runtime import ( + PROJECT_SCOPED_METADATA_MAP_KEYS, + filter_project_scoped_metadata, +) +from envctl_engine.state.run_index import RunIndexCandidate + + +ServiceProjectResolver = Callable[[str, ServiceRecord], str] +ProjectNamesResolver = Callable[[RunState], Sequence[str]] +SourceRunIdsResolver = Callable[[RunState], Sequence[str]] + + +@dataclass(frozen=True, slots=True) +class IndexedState: + candidate: RunIndexCandidate + state: RunState + + +def normalized_project_names(project_names: Sequence[str]) -> frozenset[str]: + return frozenset(str(name).strip().casefold() for name in project_names if str(name).strip()) + + +def select_indexed_owners( + indexed_states: Sequence[IndexedState], + *, + selected_projects: Sequence[str], + service_project_name: ServiceProjectResolver, +) -> list[IndexedState]: + if not indexed_states: + return [] + owner_by_project: dict[str, str] = {} + for indexed_state in indexed_states: + for project_name in indexed_state.candidate.project_names: + owner_by_project.setdefault(project_name, indexed_state.candidate.run_id) + if not owner_by_project and not normalized_project_names(selected_projects): + return [indexed_states[0]] + + requested = normalized_project_names(selected_projects) or frozenset(owner_by_project) + chosen_run_ids = {owner_by_project[project_name] for project_name in requested if project_name in owner_by_project} + if not chosen_run_ids: + return [] + + chosen_states: list[IndexedState] = [] + for indexed_state in indexed_states: + run_id = indexed_state.candidate.run_id + if run_id not in chosen_run_ids: + continue + owned_projects = frozenset( + project_name for project_name, owner_run_id in owner_by_project.items() if owner_run_id == run_id + ) + candidate_projects = normalized_project_names(indexed_state.candidate.project_names) + filtered = filter_state_to_owned_projects( + indexed_state.state, + owned_projects, + service_project_name=service_project_name, + fallback_project=( + next(iter(owned_projects)) + if len(owned_projects) == 1 and len(candidate_projects) == 1 + else None + ), + ) + chosen_states.append(IndexedState(candidate=indexed_state.candidate, state=filtered)) + return chosen_states + + +def state_from_indexed_owners( + chosen_states: Sequence[IndexedState], + *, + project_names_from_state: ProjectNamesResolver, + source_run_ids: SourceRunIdsResolver, +) -> RunState | None: + if not chosen_states: + return None + if len(chosen_states) == 1: + chosen = chosen_states[0] + if normalized_project_names(chosen.candidate.project_names) == normalized_project_names( + project_names_from_state(chosen.state) + ): + return chosen.state + return merge_indexed_states( + chosen_states, + project_names_from_state=project_names_from_state, + source_run_ids=source_run_ids, + ) + + +def filter_state_to_owned_projects( + state: RunState, + indexed_projects: frozenset[str], + *, + service_project_name: ServiceProjectResolver, + fallback_project: str | None = None, +) -> RunState: + if not indexed_projects: + return state + + fallback_key = str(fallback_project or "").strip().casefold() + fallback_display = _owned_project_display_name(state, fallback_key) if fallback_key else "" + known_project_keys = _known_state_project_keys(state).union(indexed_projects) + services: dict[str, ServiceRecord] = {} + ambiguous_services: list[str] = [] + for name, service in state.services.items(): + resolved_project = service_project_name(name, service).strip() + if resolved_project and resolved_project.casefold() not in known_project_keys: + resolved_project = "" + if not resolved_project: + resolved_project = _service_project_from_cwd_roots( + state, + service, + ) + if not resolved_project and fallback_key: + resolved_project = fallback_display or fallback_key + service = replace(service, project=resolved_project) + elif resolved_project and not str(getattr(service, "project", "") or "").strip(): + service = replace(service, project=resolved_project) + if not resolved_project: + ambiguous_services.append(name) + continue + if resolved_project.casefold() in indexed_projects: + services[name] = service + if ambiguous_services: + raise RuntimeError( + "Cannot safely determine project ownership for service(s): " + + ", ".join(sorted(ambiguous_services)) + + "; candidate projects: " + + ", ".join(sorted(indexed_projects)) + ) + requirements = { + name: requirement + for name, requirement in state.requirements.items() + if str(getattr(requirement, "project", "") or name).strip().casefold() in indexed_projects + or ( + state.mode == "trees" + and str(getattr(requirement, "project", "") or name).strip().casefold() == "main" + and bool(state.metadata.get("shared_dependencies")) + ) + } + metadata = filter_project_scoped_metadata( + state.metadata, + indexed_projects, + case_sensitive=False, + ) + metadata["project_names"] = _display_project_names( + state, + indexed_projects=indexed_projects, + service_project_name=service_project_name, + ) + filter_startup_identity_projects(metadata, indexed_projects) + return RunState( + run_id=state.run_id, + mode=state.mode, + schema_version=state.schema_version, + backend_mode=state.backend_mode, + services=services, + requirements=requirements, + pointers=dict(state.pointers), + metadata=metadata, + ) + + +def _owned_project_display_name(state: RunState, project_key: str) -> str: + if not project_key: + return "" + metadata_names = state.metadata.get("project_names") + if isinstance(metadata_names, Sequence) and not isinstance(metadata_names, (str, bytes)): + for name in metadata_names: + rendered = str(name).strip() + if rendered.casefold() == project_key: + return rendered + for storage_key, requirements in state.requirements.items(): + rendered = str(getattr(requirements, "project", "") or storage_key).strip() + if rendered.casefold() == project_key: + return rendered + return project_key + + +def _known_state_project_keys(state: RunState) -> set[str]: + projects: set[str] = set() + metadata_names = state.metadata.get("project_names") + if isinstance(metadata_names, Sequence) and not isinstance(metadata_names, (str, bytes)): + projects.update(str(name).strip().casefold() for name in metadata_names if str(name).strip()) + metadata_roots = state.metadata.get("project_roots") + if isinstance(metadata_roots, Mapping): + projects.update(str(name).strip().casefold() for name in metadata_roots if str(name).strip()) + projects.update( + str(getattr(requirements, "project", "") or storage_key).strip().casefold() + for storage_key, requirements in state.requirements.items() + if str(getattr(requirements, "project", "") or storage_key).strip() + ) + return projects + + +def _service_project_from_cwd_roots( + state: RunState, + service: ServiceRecord, +) -> str: + cwd = str(getattr(service, "cwd", "") or "").strip() + roots = state.metadata.get("project_roots") + if not cwd or not isinstance(roots, Mapping): + return "" + try: + cwd_path = Path(cwd).expanduser().resolve(strict=False) + except (OSError, RuntimeError): + return "" + matches: list[str] = [] + for project, root in roots.items(): + project_name = str(project).strip() + if not project_name: + continue + try: + root_path = Path(str(root)).expanduser().resolve(strict=False) + cwd_path.relative_to(root_path) + except (OSError, RuntimeError, ValueError): + continue + matches.append(project_name) + return matches[0] if len(matches) == 1 else "" + + +def _display_project_names( + state: RunState, + *, + indexed_projects: frozenset[str], + service_project_name: ServiceProjectResolver, +) -> list[str]: + display_by_key: dict[str, str] = {} + + def add(raw_name: object) -> None: + name = str(raw_name or "").strip() + key = name.casefold() + if name and key in indexed_projects: + display_by_key.setdefault(key, name) + + metadata_names = state.metadata.get("project_names") + if isinstance(metadata_names, Sequence) and not isinstance(metadata_names, (str, bytes)): + for project in metadata_names: + add(project) + metadata_roots = state.metadata.get("project_roots") + if isinstance(metadata_roots, Mapping): + for project in metadata_roots: + add(project) + for project, requirements in state.requirements.items(): + add(getattr(requirements, "project", "") or project) + for service_name, service in state.services.items(): + add(service_project_name(service_name, service)) + return [display_by_key.get(key, key) for key in sorted(indexed_projects)] + + +def merge_indexed_states( + indexed_states: Sequence[IndexedState], + *, + project_names_from_state: ProjectNamesResolver, + source_run_ids: SourceRunIdsResolver, +) -> RunState: + ordered = sorted(indexed_states, key=lambda item: item.candidate.sequence) + latest = max( + indexed_states, + key=lambda item: ( + item.candidate.activation_sequence, + item.candidate.sequence, + str(item.candidate.state_path), + ), + ).state + services: dict[str, ServiceRecord] = {} + requirements: dict[str, RequirementsResult] = {} + pointers: dict[str, str] = {} + project_roots: dict[str, object] = {} + project_names: dict[str, str] = {} + for indexed_state in ordered: + state = indexed_state.state + state_project_names = [str(name).strip() for name in project_names_from_state(state) if str(name).strip()] + if not state_project_names: + state_project_names = [ + str(name).strip() + for name in indexed_state.candidate.project_names + if str(name).strip() + ] + fallback_project = state_project_names[0] if len(state_project_names) == 1 else "" + incoming_services = { + name: ( + service + if str(getattr(service, "project", "") or "").strip() or not fallback_project + else replace(service, project=fallback_project) + ) + for name, service in state.services.items() + } + _merge_indexed_services( + services, + incoming_services, + service_project_name=_service_project_from_record, + ) + _merge_indexed_requirements(requirements, state.requirements) + pointers.update(state.pointers) + for name in state_project_names: + project_names.setdefault(name.casefold(), name) + roots = state.metadata.get("project_roots") + if isinstance(roots, Mapping): + project_roots.update({str(name): root for name, root in roots.items()}) + + selected_project_names = [project_names[key] for key in sorted(project_names)] + metadata = merge_project_scoped_metadata( + latest.metadata, + ordered, + project_names=selected_project_names, + ) + metadata["project_names"] = selected_project_names + all_source_run_ids = { + source_run_id + for indexed_state in ordered + for source_run_id in ( + indexed_state.candidate.run_id, + *source_run_ids(indexed_state.state), + ) + } + metadata["state_source_run_ids"] = sorted(all_source_run_ids) + if project_roots: + metadata["project_roots"] = project_roots + merge_startup_identities(metadata, ordered, project_roots=project_roots) + return RunState( + run_id=latest.run_id, + mode=latest.mode, + schema_version=latest.schema_version, + backend_mode=latest.backend_mode, + services=services, + requirements=requirements, + pointers=pointers, + metadata=metadata, + ) + + +def _service_project_from_record(name: str, service: ServiceRecord) -> str: + project = str(getattr(service, "project", "") or "").strip() + if project: + return project + rendered = str(name).strip() + service_type = str(getattr(service, "type", "") or "").strip() + suffix = f" {service_type}".casefold() + return rendered[: -len(suffix)].strip() if suffix and rendered.casefold().endswith(suffix) else "" + + +def _merge_indexed_services( + merged: dict[str, ServiceRecord], + incoming: Mapping[str, ServiceRecord], + *, + service_project_name: ServiceProjectResolver, +) -> None: + for storage_name, service in incoming.items(): + existing = merged.get(storage_name) + if existing is None: + merged[storage_name] = service + continue + existing_project = service_project_name(storage_name, existing).strip().casefold() + incoming_project = service_project_name(storage_name, service).strip().casefold() + if not existing_project or not incoming_project or existing_project == incoming_project: + merged[storage_name] = service + continue + replacement_name = _unique_aggregate_storage_name( + storage_name, + owner=str(getattr(service, "project", "") or incoming_project), + pid=getattr(service, "pid", None), + occupied=set(merged).union(incoming), + ) + merged[replacement_name] = replace(service, name=replacement_name) + + +def _merge_indexed_requirements( + merged: dict[str, RequirementsResult], + incoming: Mapping[str, RequirementsResult], +) -> None: + for storage_key, requirements in incoming.items(): + existing = merged.get(storage_key) + if existing is None: + merged[storage_key] = requirements + continue + existing_project = str(existing.project or storage_key).strip().casefold() + incoming_project = str(requirements.project or storage_key).strip().casefold() + if not existing_project or not incoming_project or existing_project == incoming_project: + merged[storage_key] = requirements + continue + replacement_key = _unique_aggregate_storage_name( + storage_key, + owner=str(requirements.project or incoming_project), + pid=None, + occupied=set(merged).union(incoming), + ) + merged[replacement_key] = requirements + + +def _unique_aggregate_storage_name( + original: str, + *, + owner: str, + pid: object, + occupied: set[str], +) -> str: + owner_label = str(owner).strip() or "unknown" + pid_label = f" {pid}" if isinstance(pid, int) and pid > 0 else "" + base = f"{original} State Collision {owner_label}{pid_label}" + candidate = base + index = 2 + while candidate in occupied: + candidate = f"{base} {index}" + index += 1 + return candidate + + +def merge_project_scoped_metadata( + latest_metadata: Mapping[str, object], + indexed_states: Sequence[IndexedState], + *, + project_names: Sequence[str], +) -> dict[str, object]: + metadata = dict(latest_metadata) + for key in PROJECT_SCOPED_METADATA_MAP_KEYS: + merged: dict[str, object] = {} + for indexed_state in indexed_states: + raw = indexed_state.state.metadata.get(key) + if isinstance(raw, Mapping): + merged.update({str(project): value for project, value in raw.items()}) + if merged: + metadata[key] = merged + else: + metadata.pop(key, None) + + stopped_by_identity: dict[tuple[str, str, str], dict[str, object]] = {} + for indexed_state in indexed_states: + raw_stopped = indexed_state.state.metadata.get(DASHBOARD_STOPPED_SERVICES_KEY) + if not isinstance(raw_stopped, list): + continue + for item in raw_stopped: + if not isinstance(item, Mapping): + continue + project = str(item.get("project", "")).strip() + name = str(item.get("name", "")).strip() + service_type = str(item.get("type", "")).strip().lower() + if not project or not name or not service_type: + continue + stopped_by_identity[(project.casefold(), name.casefold(), service_type)] = dict(item) + if stopped_by_identity: + metadata[DASHBOARD_STOPPED_SERVICES_KEY] = [stopped_by_identity[key] for key in sorted(stopped_by_identity)] + else: + metadata.pop(DASHBOARD_STOPPED_SERVICES_KEY, None) + + if len(indexed_states) > 1: + metadata.pop("project_test_results_root", None) + metadata.pop("project_test_results_updated_at", None) + return filter_project_scoped_metadata( + metadata, + list(project_names), + case_sensitive=False, + ) + + +def filter_startup_identity_projects( + metadata: dict[str, object], + project_names: frozenset[str], +) -> None: + raw_identity = metadata.get("startup_identity") + if not isinstance(raw_identity, Mapping): + return + identity = dict(raw_identity) + raw_projects = identity.get("projects") + projects = raw_projects if isinstance(raw_projects, list) else [] + identity["projects"] = [ + project + for project in projects + if isinstance(project, Mapping) and str(project.get("name", "")).strip().casefold() in project_names + ] + identity.pop("fingerprint", None) + identity["fingerprint"] = text_fingerprint(json.dumps(identity, sort_keys=True)) + metadata["startup_identity"] = identity + + +def merge_startup_identities( + metadata: dict[str, object], + indexed_states: Sequence[IndexedState], + *, + project_roots: Mapping[str, object], +) -> None: + identities = [ + dict(identity) + for indexed_state in indexed_states + if isinstance((identity := indexed_state.state.metadata.get("startup_identity")), Mapping) + ] + if not identities: + metadata.pop("startup_identity", None) + return + comparable = [ + {key: value for key, value in identity.items() if key not in {"fingerprint", "projects"}} + for identity in identities + ] + if any(payload != comparable[0] for payload in comparable[1:]): + metadata.pop("startup_identity", None) + return + identity = dict(comparable[0]) + identity["projects"] = [ + {"name": name, "root": str(root)} + for name, root in sorted(project_roots.items(), key=lambda item: item[0].casefold()) + ] + identity["fingerprint"] = text_fingerprint(json.dumps(identity, sort_keys=True)) + metadata["startup_identity"] = identity diff --git a/python/envctl_engine/ui/dashboard/ai_session_rendering.py b/python/envctl_engine/ui/dashboard/ai_session_rendering.py index f937e4df0..7f058366e 100644 --- a/python/envctl_engine/ui/dashboard/ai_session_rendering.py +++ b/python/envctl_engine/ui/dashboard/ai_session_rendering.py @@ -22,7 +22,9 @@ def print_dashboard_ai_session_row( current_tmux_target_fn: Callable[..., tuple[str, str]], ) -> None: import subprocess # noqa: PLC0415 - from envctl_engine.planning.plan_agent.workflow import resolve_plan_agent_launch_command # noqa: PLC0415 + from envctl_engine.planning.plan_agent.workflow_review_support import ( # noqa: PLC0415 + resolve_plan_agent_launch_command, + ) from envctl_engine.runtime.session_management import list_tmux_sessions # noqa: PLC0415 project_root = project_root_fn(self, state=state, project=project) diff --git a/python/envctl_engine/ui/dashboard/dependency_rendering.py b/python/envctl_engine/ui/dashboard/dependency_rendering.py index 645128101..77444cd21 100644 --- a/python/envctl_engine/ui/dashboard/dependency_rendering.py +++ b/python/envctl_engine/ui/dashboard/dependency_rendering.py @@ -25,19 +25,23 @@ def print_dashboard_dependency_rows( severity_color_fn: SeverityColorFn, ) -> None: _ = label_color - requirements = state.requirements.get(project) - if requirements is None: - return - print_dashboard_requirement_component_rows( - self, - requirements=requirements, - ok_color=ok_color, - warn_color=warn_color, - bad_color=bad_color, - reset=reset, - visual_url_fn=visual_url_fn, - severity_color_fn=severity_color_fn, - ) + project_key = str(project).strip().casefold() + matching_requirements = [ + requirements + for storage_key, requirements in state.requirements.items() + if str(getattr(requirements, "project", "") or storage_key).strip().casefold() == project_key + ] + for requirements in matching_requirements: + print_dashboard_requirement_component_rows( + self, + requirements=requirements, + ok_color=ok_color, + warn_color=warn_color, + bad_color=bad_color, + reset=reset, + visual_url_fn=visual_url_fn, + severity_color_fn=severity_color_fn, + ) def print_dashboard_shared_dependency_rows( diff --git a/python/envctl_engine/ui/dashboard/pr_link_rendering.py b/python/envctl_engine/ui/dashboard/pr_link_rendering.py index ee66d0253..1e5e1271a 100644 --- a/python/envctl_engine/ui/dashboard/pr_link_rendering.py +++ b/python/envctl_engine/ui/dashboard/pr_link_rendering.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Any +from envctl_engine.shared.services import service_for_project_type from envctl_engine.state.models import RunState @@ -72,8 +73,8 @@ def dashboard_project_root(self: Any, *, state: RunState, project: str) -> Path if isinstance(root_raw, str) and root_raw.strip(): return Path(root_raw).expanduser() - for service_suffix in (" Backend", " Frontend"): - service = state.services.get(f"{project}{service_suffix}") + for service_type in ("backend", "frontend"): + service = service_for_project_type(state, project=project, service_type=service_type) cwd_raw = getattr(service, "cwd", None) if not isinstance(cwd_raw, str) or not cwd_raw.strip(): continue diff --git a/python/envctl_engine/ui/dashboard/pr_selection_support.py b/python/envctl_engine/ui/dashboard/pr_selection_support.py index cc6fa4c45..85cfad303 100644 --- a/python/envctl_engine/ui/dashboard/pr_selection_support.py +++ b/python/envctl_engine/ui/dashboard/pr_selection_support.py @@ -4,7 +4,7 @@ import subprocess from typing import Any, Callable, cast -from envctl_engine.actions.project_action_domain import detect_default_branch +from envctl_engine.actions.action_git_state_support import detect_pr_base_branch from envctl_engine.runtime.command_router import Route from envctl_engine.state.models import RunState from envctl_engine.ui.dashboard.pr_flow import run_pr_flow @@ -12,6 +12,20 @@ from envctl_engine.ui.selector_model import SelectorItem +def _dashboard_git_output(git_root: Path, args: list[str]) -> str: + result = subprocess.run( + ["git", "-C", str(git_root), *args], + text=True, + capture_output=True, + check=False, + ) + return result.stdout if result.returncode == 0 else "" + + +def _detect_pr_base_branch(git_root: Path) -> str: + return detect_pr_base_branch(git_root, git_output=_dashboard_git_output) + + def apply_pr_selection(owner: Any, route: Route, state: RunState, rt: object) -> Route | None: runtime_any = cast(Any, rt) default_branch = owner._default_pr_base_branch(runtime_any) @@ -83,7 +97,7 @@ def default_pr_base_branch( owner: Any, runtime: Any, *, - detect_default_branch_fn: Callable[[Path], str] = detect_default_branch, + detect_default_branch_fn: Callable[[Path], str] = _detect_pr_base_branch, pr_git_root_fn: Callable[[Any, Any], Path] = pr_git_root, ) -> str: git_root = pr_git_root_fn(owner, runtime) diff --git a/python/envctl_engine/ui/dashboard/restart_selection_support.py b/python/envctl_engine/ui/dashboard/restart_selection_support.py index d047092f2..53ce54d91 100644 --- a/python/envctl_engine/ui/dashboard/restart_selection_support.py +++ b/python/envctl_engine/ui/dashboard/restart_selection_support.py @@ -300,8 +300,6 @@ def restart_services_by_project(state: RunState, runtime: Any) -> dict[str, list project_name = service_project_name(service) if not project_name: project_name = str(runtime._project_name_from_service(service_name) or "").strip() - if not project_name: - project_name = str(service_project_name(service_name)).strip() if callable(service_project_name) else "" if not project_name: project_name = "Services" service_type = stop_service_type(service_name, service) @@ -344,6 +342,8 @@ def restart_services_by_project(state: RunState, runtime: Any) -> dict[str, list def dashboard_stopped_services_by_project(state: RunState) -> dict[str, dict[str, str]]: + from envctl_engine.dashboard_metadata import normalize_dashboard_service_types + metadata = state.metadata if isinstance(state.metadata, dict) else {} raw_stopped = metadata.get("dashboard_stopped_services") stopped: dict[str, dict[str, str]] = {} @@ -353,11 +353,12 @@ def dashboard_stopped_services_by_project(state: RunState) -> dict[str, dict[str if not isinstance(item, dict): continue project = str(item.get("project", "") or "").strip() - service_type = str(item.get("type", "") or "").strip().lower() + normalized_types = normalize_dashboard_service_types([item.get("type", "")]) + service_type = normalized_types[0] if normalized_types else "" name = str(item.get("name", "") or "").strip() - if not project or service_type not in {"backend", "frontend"}: + if not project or not service_type: continue - stopped.setdefault(project, {})[service_type] = name or f"{project} {service_type.title()}" + stopped.setdefault(project, {})[service_type] = name or f"{project} {service_display_name(service_type)}" return stopped diff --git a/python/envctl_engine/ui/dashboard/snapshot_rendering.py b/python/envctl_engine/ui/dashboard/snapshot_rendering.py index 5f3cab187..31c008e2a 100644 --- a/python/envctl_engine/ui/dashboard/snapshot_rendering.py +++ b/python/envctl_engine/ui/dashboard/snapshot_rendering.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from typing import Any +from envctl_engine.shared.services import service_for_project_type from envctl_engine.state.models import RunState from envctl_engine.ui.dashboard.snapshot_support import build_dashboard_snapshot_model @@ -137,8 +138,8 @@ def _print_project( ) -> None: backend_url = project_item.get("backend_url") frontend_url = project_item.get("frontend_url") - backend_service = state.services.get(f"{project} Backend") - frontend_service = state.services.get(f"{project} Frontend") + backend_service = service_for_project_type(state, project=project, service_type="backend") + frontend_service = service_for_project_type(state, project=project, service_type="frontend") stopped_for_project = snapshot.stopped_services.get(project, {}) configured_missing_for_project = snapshot.configured_missing_services.get(project, set()) backend_stopped = backend_service is None and ( diff --git a/python/envctl_engine/ui/dashboard/stop_scope_support.py b/python/envctl_engine/ui/dashboard/stop_scope_support.py index b90935b90..c78ce7786 100644 --- a/python/envctl_engine/ui/dashboard/stop_scope_support.py +++ b/python/envctl_engine/ui/dashboard/stop_scope_support.py @@ -212,8 +212,8 @@ def stop_project_order(state: RunState, runtime: Any, *, project_names_from_stat if name and name.casefold() not in seen: seen.add(name.casefold()) names.append(name) - for project_name in state.requirements: - name = str(project_name).strip() + for storage_name, requirements in state.requirements.items(): + name = str(getattr(requirements, "project", "") or storage_name).strip() if name and name.casefold() not in seen: seen.add(name.casefold()) names.append(name) @@ -242,8 +242,8 @@ def stop_services_by_project(state: RunState, runtime: Any) -> dict[str, list[tu def stop_dependencies_by_project(state: RunState) -> dict[str, list[tuple[str, str]]]: labels = {definition.id: definition.display_name for definition in dependency_definitions()} dependencies_by_project: dict[str, list[tuple[str, str]]] = {} - for project_name, requirements in state.requirements.items(): - project = str(project_name).strip() + for storage_name, requirements in state.requirements.items(): + project = str(getattr(requirements, "project", "") or storage_name).strip() if not project: continue for definition in dependency_definitions(): diff --git a/python/envctl_engine/ui/selection_support.py b/python/envctl_engine/ui/selection_support.py index 99ab5f157..e9042dcb0 100644 --- a/python/envctl_engine/ui/selection_support.py +++ b/python/envctl_engine/ui/selection_support.py @@ -4,7 +4,7 @@ from typing import Any, Sequence from envctl_engine.runtime.command_router import Route -from envctl_engine.shared.services import project_name_from_service_name +from envctl_engine.shared.services import resolve_service_project_name from envctl_engine.state.models import RunState from envctl_engine.ui.dashboard.terminal_ui import RuntimeTerminalUI from envctl_engine.ui.selection_types import TargetSelection @@ -36,8 +36,8 @@ def interactive_selection_allowed( def project_names_from_state(runtime: Any, state: RunState) -> list[object]: names: list[str] = [] seen: set[str] = set() - for project in state.requirements: - project_name = str(project).strip() + for storage_name, requirements in state.requirements.items(): + project_name = str(getattr(requirements, "project", "") or storage_name).strip() if not project_name: continue lowered = project_name.lower() @@ -45,10 +45,12 @@ def project_names_from_state(runtime: Any, state: RunState) -> list[object]: continue seen.add(lowered) names.append(project_name) - for name in state.services: - project = runtime._project_name_from_service(name) # type: ignore[attr-defined] - if not project: - project = project_name_from_service_name(str(name)) + for name, service in state.services.items(): + project = resolve_service_project_name( + name, + service, + project_name_from_service=runtime._project_name_from_service, # type: ignore[attr-defined] + ) if not project: continue lowered = project.lower() @@ -77,10 +79,12 @@ def services_from_selection(runtime: Any, selection: TargetSelection, state: Run return {name for name in state.services if name in selection.service_names} if selection.project_names: selected: set[str] = set() - for name in state.services: - project = runtime._project_name_from_service(name) # type: ignore[attr-defined] - if not project: - project = project_name_from_service_name(str(name)) + for name, service in state.services.items(): + project = resolve_service_project_name( + name, + service, + project_name_from_service=runtime._project_name_from_service, # type: ignore[attr-defined] + ) if not project: continue for wanted in selection.project_names: @@ -110,9 +114,11 @@ def no_target_selected_message( return f"No {label} target selected." mode = str(route.mode).strip().lower() discovery = "envctl --list-trees --json" if mode == "trees" else "envctl --list-targets --main --json" + all_selector = "--all --yes" if command in {"commit", "migrate", "pr", "ship"} else "--all" return ( f"No {label} target selected. " - f"In headless mode, run '{discovery}' and retry with '--project ', '--service ', or '--all'." + f"In headless mode, run '{discovery}' and retry with '--project ', '--service ', " + f"or '{all_selector}'." ) diff --git a/tests/python/actions/commit_action_test_support.py b/tests/python/actions/commit_action_test_support.py index 2efcadbf4..24229bd34 100644 --- a/tests/python/actions/commit_action_test_support.py +++ b/tests/python/actions/commit_action_test_support.py @@ -57,7 +57,7 @@ def run_git(self, _git_root: Path, args: list[str]) -> subprocess.CompletedProce return subprocess.CompletedProcess(args=args, returncode=0, stdout=self._next_pre_stage_status(), stderr="") if args[:3] == ["reset", "-q", "--"]: return subprocess.CompletedProcess(args=args, returncode=self.reset_returncode, stdout="", stderr=self.reset_stderr) - if args[:2] == ["add", "--"]: + if args[:3] == ["add", "--all", "--"]: return subprocess.CompletedProcess(args=args, returncode=0, stdout="", stderr="") if args == ["status", "--porcelain"]: return subprocess.CompletedProcess(args=args, returncode=0, stdout=self.staged_status, stderr="") diff --git a/tests/python/actions/ship_workflow_test_support.py b/tests/python/actions/ship_workflow_test_support.py index 16dca6348..4d7243175 100644 --- a/tests/python/actions/ship_workflow_test_support.py +++ b/tests/python/actions/ship_workflow_test_support.py @@ -11,6 +11,7 @@ from types import SimpleNamespace from typing import Any +from envctl_engine.actions.action_git_state_support import ExistingPullRequest from envctl_engine.actions.action_ship_support import run_ship_workflow @@ -44,6 +45,7 @@ def run( add_ship_pr_label: Callable[[object, Path, str], int] | None = None, probe_dirty_worktree: Callable[..., object] | None = None, existing_pr_url: Callable[[Path, str], str] | None = None, + existing_pull_request: Callable[[Path, str], ExistingPullRequest | None] | None = None, partition_envctl_protected_paths: Callable[[str], object] | None = None, ordered_unique_paths: Callable[..., list[str]] | None = None, github_pr_checks: Callable[..., dict[str, object]] | None = None, @@ -64,6 +66,7 @@ def run( add_ship_pr_label=add_ship_pr_label or self.add_ship_pr_label, probe_dirty_worktree=probe_dirty_worktree or self.probe_dirty_worktree, existing_pr_url=existing_pr_url or self.existing_pr_url, + existing_pull_request=existing_pull_request, partition_envctl_protected_paths=( partition_envctl_protected_paths or self.partition_envctl_protected_paths ), diff --git a/tests/python/actions/test_action_command_execution_support.py b/tests/python/actions/test_action_command_execution_support.py index b982bfa98..bb8213ad8 100644 --- a/tests/python/actions/test_action_command_execution_support.py +++ b/tests/python/actions/test_action_command_execution_support.py @@ -44,6 +44,7 @@ def __init__(self) -> None: self.statuses: list[str] = [] self.restored = False self.spinner = _Spinner() + self.ship_calls = 0 def resolve_targets(self, _route, *, trees_only: bool) -> tuple[list[object], str | None]: return [SimpleNamespace(name="Main")], None @@ -66,6 +67,10 @@ def restore() -> None: def run_test_action(self, _route, _targets: list[object]) -> int: return 0 + def run_ship_action(self, _route, _targets: list[object]) -> int: + self.ship_calls += 1 + return 0 + class _DeferredOutputOrchestrator(_Orchestrator): def __init__(self) -> None: @@ -136,6 +141,55 @@ def test_execute_action_command_returns_target_resolution_error(self) -> None: orchestrator.runtime.events, ) + def test_execute_action_command_requires_confirmation_for_bulk_mutation(self) -> None: + orchestrator = _Orchestrator() + orchestrator.resolve_targets = lambda _route, trees_only: ( # type: ignore[assignment] + [SimpleNamespace(name="one"), SimpleNamespace(name="two")], + None, + ) + route = SimpleNamespace(command="ship", mode="trees", flags={"all": True}) + + code = execute_action_command( + orchestrator, + route, + spinner_factory=lambda *_args, **_kwargs: _policy_context(orchestrator.spinner), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + use_spinner_policy_fn=_policy_context, + ) + + self.assertEqual(code, 1) + self.assertEqual(orchestrator.ship_calls, 0) + self.assertIn( + { + "event": "action.command.finish", + "command": "ship", + "code": 1, + "error": "ship --all targets 2 projects and requires --yes or --force", + }, + orchestrator.runtime.events, + ) + + def test_execute_action_command_allows_confirmed_bulk_mutation(self) -> None: + orchestrator = _Orchestrator() + orchestrator.resolve_targets = lambda _route, trees_only: ( # type: ignore[assignment] + [SimpleNamespace(name="one"), SimpleNamespace(name="two")], + None, + ) + route = SimpleNamespace(command="ship", mode="trees", flags={"all": True, "yes": True}) + + code = execute_action_command( + orchestrator, + route, + spinner_factory=lambda *_args, **_kwargs: _policy_context(orchestrator.spinner), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + use_spinner_policy_fn=_policy_context, + ) + + self.assertEqual(code, 0) + self.assertEqual(orchestrator.ship_calls, 1) + def test_execute_action_command_flushes_deferred_output_before_finish(self) -> None: orchestrator = _DeferredOutputOrchestrator() route = SimpleNamespace(command="test", mode="main", flags={}) diff --git a/tests/python/actions/test_action_command_orchestrator_targets.py b/tests/python/actions/test_action_command_orchestrator_targets.py index 0eef8bb8e..658d29272 100644 --- a/tests/python/actions/test_action_command_orchestrator_targets.py +++ b/tests/python/actions/test_action_command_orchestrator_targets.py @@ -10,6 +10,7 @@ REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" from envctl_engine.actions.action_command_orchestrator import ActionCommandOrchestrator +from envctl_engine.actions.action_test_execution_support import build_test_action_execution_plan import envctl_engine.actions.action_test_summary_support as action_test_summary_support_module from envctl_engine.actions.action_test_summary_support import ( write_failed_tests_summary_for_orchestrator, @@ -183,6 +184,64 @@ def test_resolve_targets_does_not_flush_for_non_interactive_command(self) -> Non self.assertEqual(len(targets), 2) flush_pending.assert_not_called() + def test_untested_scope_reaches_legacy_matrix_without_main_project_filter(self) -> None: + runtime = _RuntimeStub() + runtime.config = SimpleNamespace( + base_dir=Path("/tmp/repo"), + raw={"ENVCTL_ACTION_TEST_CMD": "bash scripts/test-all-trees.sh"}, + frontend_test_path="", + ) + runtime._discover_projects = lambda mode: [ + SimpleNamespace(name="feature-a-1", root=Path("/tmp/repo/trees/feature-a/1")) + ] + runtime.split_command = lambda raw, *, replacements: raw.split() + orchestrator = ActionCommandOrchestrator(runtime) + route = Route(command="test", mode="trees", flags={"untested": True}) + prereq_roots: list[Path] = [] + + targets, error = orchestrator.resolve_targets(route, trees_only=False) + with patch( + "envctl_engine.actions.action_test_execution_support.ensure_repo_local_test_prereqs", + side_effect=lambda root, emit_status: prereq_roots.append(Path(root)), + ): + plan = build_test_action_execution_plan(orchestrator, route, targets) + + self.assertIsNone(error) + self.assertEqual(targets, []) + self.assertEqual(plan.target_contexts, []) + self.assertEqual(prereq_roots, [Path("/tmp/repo").resolve()]) + self.assertEqual(len(plan.execution_specs), 1) + self.assertEqual(plan.execution_specs[0].project_name, "all-targets") + self.assertEqual(plan.execution_specs[0].args, ["untested=true"]) + self.assertEqual(plan.execution_specs[0].spec.cwd, Path("/tmp/repo")) + + def test_untested_scope_does_not_run_non_matrix_command_against_main(self) -> None: + runtime = _RuntimeStub() + runtime.config = SimpleNamespace( + base_dir=Path("/tmp/repo"), + raw={"ENVCTL_ACTION_TEST_CMD": "python -m pytest"}, + frontend_test_path="", + ) + runtime._discover_projects = lambda mode: [ + SimpleNamespace(name="feature-a-1", root=Path("/tmp/repo/trees/feature-a/1")) + ] + runtime.split_command = lambda raw, *, replacements: raw.split() + orchestrator = ActionCommandOrchestrator(runtime) + route = Route(command="test", mode="trees", flags={"untested": True}) + + targets, error = orchestrator.resolve_targets(route, trees_only=False) + plan = build_test_action_execution_plan(orchestrator, route, targets) + + self.assertIsNone(error) + self.assertEqual(targets, []) + self.assertEqual(plan.target_contexts, []) + self.assertEqual(plan.execution_specs, []) + self.assertEqual( + plan.missing_execution_specs_message, + "No test command supports the all-untested scope. " + "Configure a test-all-trees.sh action command or select projects explicitly.", + ) + def test_projects_for_services_matches_additional_service_slug_from_state(self) -> None: runtime = _RuntimeStub() runtime._latest_state = RunState( @@ -206,6 +265,47 @@ def test_projects_for_services_matches_additional_service_slug_from_state(self) self.assertEqual(projects, ["Main"]) + def test_builtin_test_service_scopes_keep_current_worktree_when_state_resolves_service(self) -> None: + current = SimpleNamespace(name="feature-a-1", root=Path("/tmp/external-feature-a")) + for service_name in ("backend", "frontend"): + with self.subTest(service=service_name): + runtime = _RuntimeStub() + runtime.config = SimpleNamespace( + base_dir=Path("/tmp/repo"), + execution_root=current.root, + raw={}, + ) + runtime._latest_state = RunState( + run_id="run-1", + mode="trees", + services={ + f"feature-a-1 {service_name.title()}": ServiceRecord( + name=f"feature-a-1 {service_name.title()}", + type=service_name, + cwd=str(current.root), + requested_port=8000, + actual_port=8000, + status="running", + ) + }, + ) + runtime._project_name_from_service = lambda name: ( + "feature-a-1" if name.startswith("feature-a-1 ") else "" + ) + runtime._discover_projects = lambda mode: ( + [SimpleNamespace(name="Main", root=Path("/tmp/repo"))] if mode == "main" else [] + ) + orchestrator = ActionCommandOrchestrator(runtime) + orchestrator._resolve_current_worktree_target = lambda **_kwargs: current + route = Route(command="test", mode="main", flags={"services": [service_name]}) + + targets, error = orchestrator.resolve_targets(route, trees_only=False) + selected_services = orchestrator._test_service_selection(route, None, None) + + self.assertIsNone(error) + self.assertEqual(targets, [current]) + self.assertEqual(selected_services, (service_name == "backend", service_name == "frontend")) + def test_test_plan_action_delegates_to_owner(self) -> None: runtime = _RuntimeStub() runtime.config = SimpleNamespace(base_dir=Path("/tmp/repo"), raw={}) @@ -333,7 +433,7 @@ def test_test_summary_artifact_methods_delegate_to_summary_owner(self) -> None: summary_metadata={"Main": {"status": "passed"}}, ) - def test_build_test_execution_specs_uses_additional_service_test_command(self) -> None: + def test_config_only_service_selector_resolves_current_target_and_builds_test_spec(self) -> None: runtime = _RuntimeStub() runtime.config = SimpleNamespace( base_dir=Path("/tmp/repo"), @@ -351,10 +451,13 @@ def test_build_test_execution_specs_uses_additional_service_test_command(self) - orchestrator = ActionCommandOrchestrator(runtime) route = Route(command="test", mode="main", flags={"services": ["voice-runtime"]}) target = SimpleNamespace(name="Main", root=Path("/tmp/repo")) + orchestrator._resolve_current_worktree_target = lambda **_kwargs: target + + targets, error = orchestrator.resolve_targets(route, trees_only=False) specs = orchestrator._build_test_execution_specs( route=route, - targets=[target], + targets=targets, target_contexts=[TargetContext(project_name="Main", project_root=Path("/tmp/repo"), target_obj=target)], include_backend=False, include_frontend=False, @@ -362,6 +465,8 @@ def test_build_test_execution_specs_uses_additional_service_test_command(self) - untested=False, ) + self.assertIsNone(error) + self.assertEqual(targets, [target]) self.assertEqual(len(specs), 1) self.assertEqual(specs[0].spec.command, ["python", "-m", "pytest", "voice-runtime/tests"]) self.assertEqual(specs[0].spec.cwd, Path("/tmp/repo/voice-runtime")) diff --git a/tests/python/actions/test_action_commit_support.py b/tests/python/actions/test_action_commit_support.py index ae63d96f0..d7e483c79 100644 --- a/tests/python/actions/test_action_commit_support.py +++ b/tests/python/actions/test_action_commit_support.py @@ -48,11 +48,99 @@ def _run_git(_git_root: Path, args: list[str]) -> subprocess.CompletedProcess[st ) self.assertEqual(code, 0) - self.assertIn(["add", "--", "app.py"], seen) + self.assertIn(["add", "--all", "--", "app.py"], seen) self.assertTrue(any(args[:2] == ["commit", "-F"] for args in seen)) self.assertIn(["push", "-u", "origin", "feature-branch"], seen) self.assertEqual(ledger.read_text(encoding="utf-8"), "Old summary\n\nNew summary\n\n### Envctl pointer ###\n") + def test_stage_paths_records_unstaged_tracked_deletions(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + git_root = Path(temp_dir) + + def run_git(_git_root: Path, args: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=git_root, + text=True, + capture_output=True, + check=False, + ) + + for args in ( + ["init"], + ["config", "user.email", "envctl-tests@example.invalid"], + ["config", "user.name", "Envctl Tests"], + ): + self.assertEqual(run_git(git_root, args).returncode, 0) + obsolete = git_root / "obsolete.py" + obsolete.write_text("obsolete = True\n", encoding="utf-8") + self.assertEqual(run_git(git_root, ["add", "obsolete.py"]).returncode, 0) + self.assertEqual(run_git(git_root, ["commit", "-m", "initial"]).returncode, 0) + obsolete.unlink() + + runner = action_commit_support.CommitWorkflowRunner( + context=SimpleNamespace(), + dependencies=action_commit_support.CommitWorkflowDependencies( + resolve_git_root=lambda project_root, _repo_root: project_root, + git_available=True, + git_output=lambda _git_root, _args: "", + run_git=run_git, + print_error=lambda _prefix, _result: None, + partition_envctl_protected_paths=partition_envctl_protected_paths, + ordered_unique_paths=lambda *groups: [item for group in groups for item in group], + ), + ) + + self.assertTrue(runner._stage_paths(git_root, ["obsolete.py"])) + staged = run_git(git_root, ["diff", "--cached", "--name-status"]) + self.assertEqual(staged.returncode, 0) + self.assertEqual(staged.stdout.strip(), "D\tobsolete.py") + + def test_stage_candidates_preserves_an_already_staged_deletion(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + git_root = Path(temp_dir) + + def run_git(_git_root: Path, args: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=git_root, + text=True, + capture_output=True, + check=False, + ) + + for args in ( + ["init"], + ["config", "user.email", "envctl-tests@example.invalid"], + ["config", "user.name", "Envctl Tests"], + ): + self.assertEqual(run_git(git_root, args).returncode, 0) + obsolete = git_root / "obsolete.py" + obsolete.write_text("obsolete = True\n", encoding="utf-8") + self.assertEqual(run_git(git_root, ["add", "obsolete.py"]).returncode, 0) + self.assertEqual(run_git(git_root, ["commit", "-m", "initial"]).returncode, 0) + obsolete.unlink() + self.assertEqual(run_git(git_root, ["add", "--all"]).returncode, 0) + + runner = action_commit_support.CommitWorkflowRunner( + context=SimpleNamespace(), + dependencies=action_commit_support.CommitWorkflowDependencies( + resolve_git_root=lambda project_root, _repo_root: project_root, + git_available=True, + git_output=lambda _git_root, _args: "", + run_git=run_git, + print_error=lambda _prefix, _result: None, + partition_envctl_protected_paths=partition_envctl_protected_paths, + ordered_unique_paths=lambda *groups: [item for group in groups for item in group], + ), + ) + + partition = runner._stage_commit_candidates(git_root) + self.assertIsNotNone(partition) + staged = run_git(git_root, ["diff", "--cached", "--name-status"]) + self.assertEqual(staged.returncode, 0) + self.assertEqual(staged.stdout.strip(), "D\tobsolete.py") + def test_read_commit_ledger_segment_reports_empty_pointer(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: ledger = Path(temp_dir) / ".envctl-commit-message.md" diff --git a/tests/python/actions/test_action_git_state_support.py b/tests/python/actions/test_action_git_state_support.py index 30acf4c4b..91cc6a6d4 100644 --- a/tests/python/actions/test_action_git_state_support.py +++ b/tests/python/actions/test_action_git_state_support.py @@ -25,23 +25,120 @@ def fake_git_output(_git_root: Path, args: list[str]) -> str: self.assertTrue(report.untracked) self.assertTrue(report.dirty) - def test_detect_default_branch_prefers_origin_head_then_local_candidates(self) -> None: + def test_detect_default_branch_prefers_dev_then_main_then_master(self) -> None: calls: list[list[str]] = [] - def origin_head(_git_root: Path, args: list[str]) -> str: + def dev_exists(_git_root: Path, args: list[str]) -> str: calls.append(args) - return "origin/develop\n" + return "abc123\n" if args == ["rev-parse", "--verify", "refs/remotes/origin/dev"] else "" - self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=origin_head), "develop") - self.assertEqual(calls, [["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]]) + self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=dev_exists), "dev") + self.assertEqual(calls[0], ["rev-parse", "--verify", "refs/remotes/origin/dev"]) + self.assertTrue(any(args[:3] == ["ls-remote", "--symref", "origin"] for args in calls)) def local_main(_git_root: Path, args: list[str]) -> str: - if args == ["rev-parse", "--verify", "main"]: + if args == ["rev-parse", "--verify", "refs/heads/main"]: return "abc123\n" return "" self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=local_main), "main") + def remote_master(_git_root: Path, args: list[str]) -> str: + if args == ["rev-parse", "--verify", "refs/remotes/origin/master"]: + return "abc123\n" + return "" + + self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=remote_master), "master") + + def test_detect_default_branch_uses_remote_head_only_after_preferred_branches_are_absent(self) -> None: + def remote_head(_git_root: Path, args: list[str]) -> str: + if args == ["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]: + return "origin/develop\n" + return "" + + self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=remote_head), "develop") + + def test_detect_default_branch_discovers_dev_omitted_by_local_fetch_refspec(self) -> None: + def single_branch_clone(_git_root: Path, args: list[str]) -> str: + if args == ["rev-parse", "--verify", "refs/remotes/origin/main"]: + return "abc123\n" + if args[:3] == ["ls-remote", "--symref", "origin"]: + return "ref: refs/heads/main\tHEAD\nmainsha\tHEAD\ndevsha\trefs/heads/dev\nmainsha\trefs/heads/main\n" + return "" + + self.assertEqual( + git_state.detect_default_branch(Path("/repo"), git_output=single_branch_clone), + "dev", + ) + + def test_live_remote_inventory_overrides_stale_deleted_dev_tracking_ref(self) -> None: + def stale_dev(_git_root: Path, args: list[str]) -> str: + if args == ["rev-parse", "--verify", "refs/remotes/origin/dev"]: + return "stale-dev-sha\n" + if args == ["rev-parse", "--verify", "refs/remotes/origin/main"]: + return "main-sha\n" + if args[:3] == ["ls-remote", "--symref", "origin"]: + return ( + "ref: refs/heads/main\tHEAD\n" + "main-sha\tHEAD\n" + "main-sha\trefs/heads/main\n" + "master-sha\trefs/heads/master\n" + ) + return "" + + self.assertEqual( + git_state.detect_default_branch(Path("/repo"), git_output=stale_dev), + "main", + ) + + def test_live_remote_default_overrides_stale_dev_when_no_preferred_head_exists(self) -> None: + def no_preferred_remote(_git_root: Path, args: list[str]) -> str: + if args == ["rev-parse", "--verify", "refs/remotes/origin/dev"]: + return "stale-dev-sha\n" + if args[:3] == ["ls-remote", "--symref", "origin"]: + return "ref: refs/heads/feature/alpha\tHEAD\nfeature-sha\tHEAD\n" + return "" + + self.assertEqual( + git_state.detect_default_branch(Path("/repo"), git_output=no_preferred_remote), + "feature/alpha", + ) + + def test_detect_pr_base_branch_uses_preferred_default_branch_contract(self) -> None: + def remote_trunk(_git_root: Path, args: list[str]) -> str: + if args[:3] == ["ls-remote", "--symref", "origin"]: + return "ref: refs/heads/trunk\tHEAD\ntrunk-sha\tHEAD\n" + return "" + + self.assertEqual( + git_state.detect_pr_base_branch(Path("/repo"), git_output=remote_trunk), + "trunk", + ) + + def test_detect_pr_base_branch_does_not_probe_origin_when_local_candidate_exists(self) -> None: + calls: list[list[str]] = [] + + def local_main(_git_root: Path, args: list[str]) -> str: + calls.append(args) + if args == ["rev-parse", "--verify", "refs/remotes/origin/main"]: + return "abc123\n" + if args[0] == "ls-remote": + raise AssertionError("remote must not be probed when a local base ref is available") + return "" + + self.assertEqual( + git_state.detect_pr_base_branch(Path("/repo"), git_output=local_main), + "main", + ) + self.assertEqual( + calls, + [ + ["rev-parse", "--verify", "refs/remotes/origin/dev"], + ["rev-parse", "--verify", "refs/heads/dev"], + ["rev-parse", "--verify", "refs/remotes/origin/main"], + ], + ) + def test_existing_pr_url_uses_gh_pr_list_for_open_branch(self) -> None: calls: list[tuple[list[str], str]] = [] @@ -57,7 +154,12 @@ def fake_run( self.assertTrue(capture_output) self.assertFalse(check) calls.append((args, cwd)) - return subprocess.CompletedProcess(args, 0, stdout="https://github.com/acme/repo/pull/9\n", stderr="") + return subprocess.CompletedProcess( + args, + 0, + stdout='[{"url":"https://github.com/acme/repo/pull/9","baseRefName":"dev"}]\n', + stderr="", + ) url = git_state.existing_pr_url(Path("/repo"), "feature", gh_path="/usr/bin/gh", run_process=fake_run) @@ -66,17 +168,95 @@ def fake_run( self.assertEqual(calls[0][0][:4], ["/usr/bin/gh", "pr", "list", "--head"]) self.assertIn("feature", calls[0][0]) + def test_existing_pull_request_preserves_base_metadata(self) -> None: + def fake_run(args: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + args, + 0, + stdout='[{"url":"https://github.com/acme/repo/pull/9","baseRefName":"release"}]', + stderr="", + ) + + existing = git_state.existing_pull_request( + Path("/repo"), + "feature", + gh_path="/usr/bin/gh", + run_process=fake_run, + ) + + self.assertEqual( + existing, + git_state.ExistingPullRequest( + url="https://github.com/acme/repo/pull/9", + base_branch="release", + ), + ) + + def test_update_pull_request_base_uses_pr_url_and_reports_missing_gh(self) -> None: + existing = git_state.ExistingPullRequest(url="https://github.com/acme/repo/pull/9", base_branch="main") + calls: list[list[str]] = [] + + def fake_run(args: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + calls.append(args) + return subprocess.CompletedProcess(args, 0, stdout="updated\n", stderr="") + + updated = git_state.update_pull_request_base( + Path("/repo"), + existing, + "dev", + gh_path="/usr/bin/gh", + run_process=fake_run, + ) + missing = git_state.update_pull_request_base( + Path("/repo"), + existing, + "dev", + gh_path=None, + run_process=lambda *_args, **_kwargs: self.fail("process should not run"), + ) + + self.assertEqual(updated.returncode, 0) + self.assertEqual( + calls, + [["/usr/bin/gh", "pr", "edit", "https://github.com/acme/repo/pull/9", "--base", "dev"]], + ) + self.assertEqual(missing.returncode, 127) + self.assertIn("gh is required", missing.stderr) + def test_existing_pr_url_skips_invalid_inputs_and_failed_gh(self) -> None: def unexpected_run(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: raise AssertionError("run_process should not be called") - self.assertEqual(git_state.existing_pr_url(Path("/repo"), "HEAD", gh_path="/usr/bin/gh", run_process=unexpected_run), "") - self.assertEqual(git_state.existing_pr_url(Path("/repo"), "feature", gh_path=None, run_process=unexpected_run), "") + self.assertEqual( + git_state.existing_pr_url(Path("/repo"), "HEAD", gh_path="/usr/bin/gh", run_process=unexpected_run), "" + ) + self.assertEqual( + git_state.existing_pr_url(Path("/repo"), "feature", gh_path=None, run_process=unexpected_run), "" + ) + + def test_remote_branch_probe_is_noninteractive_and_bounded(self) -> None: + captured: dict[str, object] = {} + + def timed_out(args: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + captured.update(kwargs) + raise subprocess.TimeoutExpired(args, timeout=float(kwargs["timeout"])) + + result = git_state.run_git( + Path("/repo"), + ["ls-remote", "--heads", "origin", "refs/heads/dev"], + run_process=timed_out, + ) + + self.assertEqual(result.returncode, 124) + self.assertEqual(captured["timeout"], 10) + self.assertEqual(captured["env"]["GIT_TERMINAL_PROMPT"], "0") # type: ignore[index] def failed_run(*args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: return subprocess.CompletedProcess(args, 1, stdout="", stderr="not found") - self.assertEqual(git_state.existing_pr_url(Path("/repo"), "feature", gh_path="/usr/bin/gh", run_process=failed_run), "") + self.assertEqual( + git_state.existing_pr_url(Path("/repo"), "feature", gh_path="/usr/bin/gh", run_process=failed_run), "" + ) if __name__ == "__main__": diff --git a/tests/python/actions/test_action_pr_workflow_support.py b/tests/python/actions/test_action_pr_workflow_support.py index 9ccaf6068..0020ae767 100644 --- a/tests/python/actions/test_action_pr_workflow_support.py +++ b/tests/python/actions/test_action_pr_workflow_support.py @@ -7,9 +7,65 @@ from types import SimpleNamespace from envctl_engine.actions import action_pr_workflow_support as support +from envctl_engine.actions.action_git_state_support import ExistingPullRequest class ActionPrWorkflowSupportTests(unittest.TestCase): + def test_pr_workflow_rejects_default_branch_as_the_head(self) -> None: + context = SimpleNamespace( + repo_root=Path("/repo"), + project_root=Path("/repo"), + project_name="Main", + ) + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "dev\n", + resolve_base_branch_fn=lambda _context, _git_root: "dev", + existing_pull_request_fn=lambda *_args: None, + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), + probe_dirty_worktree_fn=lambda *_args: self.fail("worktree should not be probed"), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: self.fail("title should not be built"), + pr_body_fn=lambda *_args: self.fail("body should not be built"), + write_pr_body_file_fn=lambda _body: self.fail("body file should not be written"), + print_process_output_fn=lambda _result: None, + gh_path="/usr/bin/gh", + ) + + self.assertEqual(code, 1) + + def test_pr_workflow_rediscovers_existing_pr_before_self_base_guard(self) -> None: + context = SimpleNamespace( + repo_root=Path("/repo"), + project_root=Path("/repo"), + project_name="Main", + ) + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "dev\n", + resolve_base_branch_fn=lambda _context, _git_root: "dev", + existing_pull_request_fn=lambda _git_root, _branch: ExistingPullRequest( + url="https://github.com/acme/repo/pull/12", + base_branch="dev", + ), + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), + probe_dirty_worktree_fn=lambda *_args: self.fail("worktree should not be probed"), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: self.fail("title should not be built"), + pr_body_fn=lambda *_args: self.fail("body should not be built"), + write_pr_body_file_fn=lambda _body: self.fail("body file should not be written"), + print_process_output_fn=lambda _result: None, + gh_path="/usr/bin/gh", + ) + + self.assertEqual(code, 0) + def test_pr_workflow_skips_existing_pull_request(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo_root = Path(tmpdir) @@ -22,7 +78,11 @@ def test_pr_workflow_skips_existing_pull_request(self) -> None: git_available=True, git_output_fn=lambda _git_root, _args: "feature/demo\n", resolve_base_branch_fn=lambda _context, _git_root: "main", - existing_pr_url_fn=lambda _git_root, _branch: "https://github.com/acme/repo/pull/12", + existing_pull_request_fn=lambda _git_root, _branch: ExistingPullRequest( + url="https://github.com/acme/repo/pull/12", + base_branch="main", + ), + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), probe_dirty_worktree_fn=lambda *_args: self.fail("dirty worktree should not be probed"), run_commit_action_fn=lambda _context: self.fail("commit should not run"), pr_title_fn=lambda *_args: self.fail("title should not be built"), @@ -46,7 +106,9 @@ def test_pr_workflow_creates_with_gh_and_removes_body_file(self) -> None: def run_process(args: list[str], **_kwargs): # noqa: ANN001 calls.append([str(token) for token in args]) self.assertTrue(body_file.is_file()) - return subprocess.CompletedProcess(args=args, returncode=0, stdout="https://example.test/pr/1\n", stderr="") + return subprocess.CompletedProcess( + args=args, returncode=0, stdout="https://example.test/pr/1\n", stderr="" + ) def write_body(_body: str) -> Path: body_file.write_text("Body", encoding="utf-8") @@ -58,7 +120,8 @@ def write_body(_body: str) -> Path: git_available=True, git_output_fn=lambda _git_root, _args: "feature/demo\n", resolve_base_branch_fn=lambda _context, _git_root: "main", - existing_pr_url_fn=lambda _git_root, _branch: "", + existing_pull_request_fn=lambda _git_root, _branch: None, + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), probe_dirty_worktree_fn=lambda *_args: SimpleNamespace(dirty=False), run_commit_action_fn=lambda _context: self.fail("commit should not run"), pr_title_fn=lambda _context, _git_root, _head_branch: "Demo PR", @@ -75,6 +138,157 @@ def write_body(_body: str) -> Path: self.assertIn("--base", calls[0]) self.assertIn("main", calls[0]) + def test_explicit_base_updates_existing_pr_and_verifies_result(self) -> None: + context = SimpleNamespace( + repo_root=Path("/repo"), + project_root=Path("/repo"), + project_name="Main", + env={"ENVCTL_PR_BASE": "release"}, + ) + existing_values = iter( + [ + ExistingPullRequest("https://github.com/acme/repo/pull/12", "main"), + ExistingPullRequest("https://github.com/acme/repo/pull/12", "release"), + ] + ) + update_calls: list[tuple[ExistingPullRequest, str]] = [] + + def update_base( + _git_root: Path, + existing: ExistingPullRequest, + base_branch: str, + ) -> subprocess.CompletedProcess[str]: + update_calls.append((existing, base_branch)) + return subprocess.CompletedProcess([], 0, stdout="updated\n", stderr="") + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "feature/demo\n", + resolve_base_branch_fn=lambda *_args: self.fail("implicit resolver should not run"), + existing_pull_request_fn=lambda *_args: next(existing_values), + update_pull_request_base_fn=update_base, + probe_dirty_worktree_fn=lambda *_args: self.fail("worktree should not be probed"), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: self.fail("title should not be built"), + pr_body_fn=lambda *_args: self.fail("body should not be built"), + write_pr_body_file_fn=lambda _body: self.fail("body file should not be written"), + print_process_output_fn=lambda _result: None, + gh_path="/usr/bin/gh", + ) + + self.assertEqual(code, 0) + self.assertEqual(len(update_calls), 1) + self.assertEqual(update_calls[0][1], "release") + + def test_implicit_default_preserves_existing_pr_base(self) -> None: + context = SimpleNamespace( + repo_root=Path("/repo"), + project_root=Path("/repo"), + project_name="Main", + env={}, + ) + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "feature/demo\n", + resolve_base_branch_fn=lambda *_args: self.fail("implicit resolver should not run for an existing PR"), + existing_pull_request_fn=lambda *_args: ExistingPullRequest( + "https://github.com/acme/repo/pull/12", + "main", + ), + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), + probe_dirty_worktree_fn=lambda *_args: self.fail("worktree should not be probed"), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: self.fail("title should not be built"), + pr_body_fn=lambda *_args: self.fail("body should not be built"), + write_pr_body_file_fn=lambda _body: self.fail("body file should not be written"), + print_process_output_fn=lambda _result: None, + gh_path="/usr/bin/gh", + ) + + self.assertEqual(code, 0) + + def test_failed_gh_create_recovers_concurrent_matching_pr(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + repo_root = Path(tmpdir) + context = SimpleNamespace(repo_root=repo_root, project_root=repo_root, project_name="Main", env={}) + existing_values = iter( + [ + None, + ExistingPullRequest("https://github.com/acme/repo/pull/13", "dev"), + ] + ) + body_file = repo_root / "body.md" + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "feature/race\n", + resolve_base_branch_fn=lambda *_args: "dev", + existing_pull_request_fn=lambda *_args: next(existing_values), + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), + probe_dirty_worktree_fn=lambda *_args: SimpleNamespace(dirty=False), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: "Race", + pr_body_fn=lambda *_args: "Body", + write_pr_body_file_fn=lambda _body: body_file, + print_process_output_fn=lambda _result: self.fail("recovered create error should stay quiet"), + gh_path="/usr/bin/gh", + run_process_fn=lambda args, **_kwargs: subprocess.CompletedProcess( + args, + 1, + stdout="", + stderr="a pull request already exists", + ), + ) + + self.assertEqual(code, 0) + + def test_failed_helper_create_recovers_concurrent_matching_pr(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + repo_root = Path(tmpdir) + helper = repo_root / "utils" / "create-pr.sh" + helper.parent.mkdir(parents=True) + helper.write_text("#!/bin/sh\n", encoding="utf-8") + helper.chmod(0o755) + context = SimpleNamespace(repo_root=repo_root, project_root=repo_root, project_name="Main", env={}) + existing_values = iter( + [ + None, + ExistingPullRequest("https://github.com/acme/repo/pull/14", "dev"), + ] + ) + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "feature/race\n", + resolve_base_branch_fn=lambda *_args: "dev", + existing_pull_request_fn=lambda *_args: next(existing_values), + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), + probe_dirty_worktree_fn=lambda *_args: SimpleNamespace(dirty=False), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: self.fail("title should not be built"), + pr_body_fn=lambda *_args: self.fail("body should not be built"), + write_pr_body_file_fn=lambda _body: self.fail("body file should not be written"), + print_process_output_fn=lambda _result: self.fail("recovered helper error should stay quiet"), + gh_path=None, + run_process_fn=lambda args, **_kwargs: subprocess.CompletedProcess( + args, + 1, + stdout="", + stderr="a pull request already exists", + ), + ) + + self.assertEqual(code, 0) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/actions/test_action_ship_checks.py b/tests/python/actions/test_action_ship_checks.py index 4b70a2af6..aea5d77a6 100644 --- a/tests/python/actions/test_action_ship_checks.py +++ b/tests/python/actions/test_action_ship_checks.py @@ -77,8 +77,8 @@ def fail_run(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[s "duration_seconds": 0.0, } -def test_github_pr_checks_default_timeout_is_two_minute_wait_window() -> None: - assert action_ship_checks.DEFAULT_CHECK_TIMEOUT_SECONDS == 120.0 +def test_github_pr_checks_default_timeout_is_three_minute_wait_window() -> None: + assert action_ship_checks.DEFAULT_CHECK_TIMEOUT_SECONDS == 180.0 def test_github_pr_checks_default_no_checks_grace_is_fifteen_seconds() -> None: assert action_ship_checks.DEFAULT_NO_CHECKS_GRACE_SECONDS == 15.0 @@ -299,6 +299,46 @@ def fake_sleep(seconds: float) -> None: assert checks["duration_seconds"] == 2.0 assert updates == [] + +def test_github_pr_checks_ignores_transient_empty_rollup_after_checks_start(tmp_path: Path) -> None: + clock = {"now": 0.0} + outputs = [ + { + "headRefOid": "newsha", + "statusCheckRollup": [{"name": "pytest", "workflowName": "Tests", "status": "IN_PROGRESS"}], + }, + {"headRefOid": "newsha", "statusCheckRollup": []}, + { + "headRefOid": "newsha", + "statusCheckRollup": [ + {"name": "pytest", "workflowName": "Tests", "status": "COMPLETED", "conclusion": "SUCCESS"} + ], + }, + ] + + def fake_run(args: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + if args[:2] == ["/usr/bin/gh", "api"]: + return subprocess.CompletedProcess(args=args, returncode=0, stdout="[]", stderr="") + return subprocess.CompletedProcess(args=args, returncode=0, stdout=json.dumps(outputs.pop(0)), stderr="") + + def fake_sleep(seconds: float) -> None: + clock["now"] += seconds + + checks = github_pr_checks( + tmp_path, + expected_head_sha="newsha", + timeout_seconds=30.0, + poll_interval_seconds=1.0, + no_checks_grace_seconds=0.0, + run_command=fake_run, + monotonic=lambda: clock["now"], + sleep=fake_sleep, + ) + + assert checks["state"] == "checks_passed" + assert checks["duration_seconds"] == 2.0 + assert outputs == [] + def test_github_pr_checks_waits_for_expected_head_sha_before_accepting_rollup(tmp_path: Path) -> None: calls: list[list[str]] = [] outputs = [ diff --git a/tests/python/actions/test_action_ship_support.py b/tests/python/actions/test_action_ship_support.py index df0ece678..36da1e92b 100644 --- a/tests/python/actions/test_action_ship_support.py +++ b/tests/python/actions/test_action_ship_support.py @@ -20,7 +20,7 @@ def test_public_run_ship_workflow_is_thin_compatibility_wrapper() -> None: source = inspect.getsource(run_ship_workflow) assert "ShipWorkflowRunner" in source - assert len(source.splitlines()) <= 35 + assert len(source.splitlines()) <= 40 def test_ship_workflow_runner_exposes_named_phases() -> None: @@ -58,6 +58,7 @@ def test_ship_workflow_dependencies_group_injected_collaborators() -> None: "partition_envctl_protected_paths", "ordered_unique_paths", "github_pr_checks", + "existing_pull_request", } runner_fields = set(getattr(ship_support.ShipWorkflowRunner, "__dataclass_fields__", {})) diff --git a/tests/python/actions/test_action_ship_workflow_lifecycle.py b/tests/python/actions/test_action_ship_workflow_lifecycle.py index ca8a86615..c53019a9a 100644 --- a/tests/python/actions/test_action_ship_workflow_lifecycle.py +++ b/tests/python/actions/test_action_ship_workflow_lifecycle.py @@ -3,6 +3,7 @@ from pathlib import Path from types import SimpleNamespace +from envctl_engine.actions.action_git_state_support import ExistingPullRequest from tests.python.actions.ship_workflow_test_support import ship_workflow_fixture @@ -55,6 +56,82 @@ def github_pr_checks( assert result.payload["checks_expected_head_sha"] == "abc123" +def test_run_ship_workflow_reconciles_explicit_base_before_reusing_existing_pr() -> None: + with ship_workflow_fixture() as fixture: + fixture.context.env["ENVCTL_PR_BASE"] = "dev" + pr_calls: list[object] = [] + resolved_bases: list[str] = [] + + def run_pr_action(context: object) -> int: + pr_calls.append(context) + return 0 + + def resolve_base_ref(_git_root: Path, base_branch: str) -> str: + resolved_bases.append(base_branch) + return f"origin/{base_branch}" + + result = fixture.run(run_pr_action=run_pr_action, resolve_base_ref=resolve_base_ref) + + assert result.code == 0 + assert pr_calls == [fixture.context] + assert resolved_bases == ["dev"] + assert result.payload["pr_created"] is False + assert result.payload["pr_url"] == "https://github.com/acme/repo/pull/7" + assert result.payload["step_statuses"] == ["clean_no_changes", "pr_exists", "checks_passed"] + + +def test_run_ship_workflow_predicts_conflicts_against_implicit_existing_pr_base() -> None: + with ship_workflow_fixture() as fixture: + resolved_bases: list[str] = [] + + def resolve_base_branch(_context: object, _git_root: Path) -> str: + raise AssertionError("default resolver must not replace an existing PR base") + + def resolve_base_ref(_git_root: Path, base_branch: str) -> str: + resolved_bases.append(base_branch) + return f"origin/{base_branch}" + + result = fixture.run( + existing_pull_request=lambda _git_root, _branch: ExistingPullRequest( + url="https://github.com/acme/repo/pull/7", + base_branch="main", + ), + resolve_base_branch=resolve_base_branch, + resolve_base_ref=resolve_base_ref, + ) + + assert result.code == 0 + assert resolved_bases == ["main"] + assert result.payload["merge_conflicts"]["base_branch"] == "main" + + +def test_run_ship_workflow_tracks_concurrent_pr_winner_after_pr_action_recovers() -> None: + with ship_workflow_fixture() as fixture: + existing_calls = 0 + pr_calls = 0 + + def existing_pr_url(_git_root: Path, _branch: str) -> str: + nonlocal existing_calls + existing_calls += 1 + return "" if existing_calls == 1 else "https://github.com/acme/repo/pull/9" + + def run_pr_action(_context: object) -> int: + nonlocal pr_calls + pr_calls += 1 + return 0 + + result = fixture.run( + existing_pr_url=existing_pr_url, + run_pr_action=run_pr_action, + ) + + assert result.code == 0 + assert pr_calls == 1 + assert existing_calls >= 2 + assert result.payload["pr_url"] == "https://github.com/acme/repo/pull/9" + assert result.payload["step_statuses"] == ["clean_no_changes", "pr_created", "checks_passed"] + + def test_run_ship_workflow_pushes_clean_local_head_when_existing_pr_branch_is_stale() -> None: with ship_workflow_fixture() as fixture: run_git_calls: list[list[str]] = [] @@ -357,6 +434,4 @@ def git_output(_git_root: Path, args: list[str]) -> str: assert result.payload["committed"] is False assert result.payload["pushed"] is False assert result.payload["protected_local_artifacts_skipped"] == [".envctl-state/code-intelligence.json"] - assert result.stderr.splitlines() == [ - "ship: PR already exists for Main: https://github.com/acme/repo/pull/7" - ] + assert result.stderr.splitlines() == ["ship: PR already exists for Main: https://github.com/acme/repo/pull/7"] diff --git a/tests/python/actions/test_action_target_support.py b/tests/python/actions/test_action_target_support.py index 40a8b4463..6e60643ba 100644 --- a/tests/python/actions/test_action_target_support.py +++ b/tests/python/actions/test_action_target_support.py @@ -14,6 +14,7 @@ action_target_identities, action_target_names, action_target_names_with_roots, + build_action_target_contexts, execute_targeted_action, emit_action_output, projects_for_services, @@ -50,6 +51,17 @@ def test_action_target_identities_normalize_name_root_and_skip_invalid_targets(s self.assertEqual(action_target_names(targets), ["feature-a-1", "missing-root"]) self.assertEqual(action_target_names_with_roots(targets), ["feature-a-1"]) + def test_build_action_target_contexts_reports_progress_over_valid_targets_only(self) -> None: + contexts = build_action_target_contexts( + [ + _Target(name="feature-a-1", root="/repo/trees/feature-a/1"), + _Target(name="", root="/repo/trees/invalid/1"), + _Target(name="feature-b-1", root="/repo/trees/feature-b/1"), + ] + ) + + self.assertEqual([(context.index, context.total) for context in contexts], [(1, 2), (2, 2)]) + def test_projects_for_services_resolves_from_state_and_deduplicates(self) -> None: state = SimpleNamespace( services={ @@ -67,6 +79,50 @@ def test_projects_for_services_resolves_from_state_and_deduplicates(self) -> Non self.assertEqual(resolved, ["feature-a-1", "feature-b-1"]) + def test_projects_for_services_resolves_all_projects_for_shared_service_type_once(self) -> None: + state = SimpleNamespace( + services={ + "feature-a-1 Worker": SimpleNamespace(type="worker"), + "feature-b-1 Worker": SimpleNamespace(type="worker"), + } + ) + mapped: list[str] = [] + + def project_name_from_service(name: str) -> str: + mapped.append(name) + return name.split()[0] if name.startswith("feature-") else "" + + runtime = SimpleNamespace( + load_existing_state=lambda mode: state if mode == "trees" else None, + project_name_from_service=project_name_from_service, + ) + + resolved = projects_for_services(runtime, ["worker", "service:worker"]) + + self.assertEqual(resolved, ["feature-a-1", "feature-b-1"]) + self.assertEqual(mapped, ["feature-a-1 Worker", "feature-b-1 Worker"]) + + def test_projects_for_services_prefers_additional_service_project_metadata(self) -> None: + state = SimpleNamespace( + services={ + "My Project Voice Runtime": SimpleNamespace( + name="My Project Voice Runtime", + type="voice-runtime", + project="My Project", + ) + } + ) + mapped: list[str] = [] + runtime = SimpleNamespace( + load_existing_state=lambda mode: state if mode == "trees" else None, + project_name_from_service=lambda name: mapped.append(name) or "", + ) + + resolved = projects_for_services(runtime, ["voice-runtime"]) + + self.assertEqual(resolved, ["My Project"]) + self.assertEqual(mapped, []) + def test_resolve_action_targets_uses_single_candidate_without_prompt(self) -> None: target = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") runtime = SimpleNamespace( @@ -110,6 +166,166 @@ def test_resolve_action_targets_prefers_current_worktree_for_tree_mode_action_co self.assertIsNone(error) self.assertEqual(calls, [{"require_configured_main_root": False, "require_configured_root_match": True}]) + def test_unresolved_external_checkout_never_falls_back_to_main_candidate(self) -> None: + main = _Target(name="Main", root="/tmp/repo") + runtime = SimpleNamespace( + config=SimpleNamespace( + base_dir=Path("/tmp/repo"), + execution_root=Path("/tmp/envctl-external-worktree"), + ), + discover_projects=lambda mode: [main] if mode == "main" else [], + selectors_from_passthrough=lambda _args: [], + ) + route = parse_route(["ship"], env={"ENVCTL_DEFAULT_MODE": "main"}) + + selected, error = resolve_action_targets( + runtime=runtime, + route=route, + trees_only=False, + resolve_current_worktree_target=lambda **_kwargs: None, + interactive_selection_allowed=lambda _route: False, + no_target_selected_message=lambda _route: "no ship target", + ) + + self.assertEqual(selected, []) + self.assertEqual(error, "no ship target") + + def test_all_current_checkout_actions_prefer_external_linked_target_in_both_modes(self) -> None: + external = _Target(name="feature/external", root="/tmp/envctl-external-worktree") + main = _Target(name="Main", root="/tmp/repo") + runtime = SimpleNamespace( + config=SimpleNamespace( + base_dir=Path("/tmp/repo"), + execution_root=Path(external.root), + ), + discover_projects=lambda mode: [main] if mode == "main" else [], + selectors_from_passthrough=lambda _args: [], + ) + + for mode in ("trees", "main"): + for command in ("ship", "pr", "commit", "test", "test-focused"): + with self.subTest(mode=mode, command=command): + calls: list[dict[str, object]] = [] + route = parse_route([command], env={"ENVCTL_DEFAULT_MODE": mode}) + + selected, error = resolve_action_targets( + runtime=runtime, + route=route, + trees_only=False, + resolve_current_worktree_target=lambda **kwargs: calls.append(kwargs) or external, + interactive_selection_allowed=lambda _route: False, + no_target_selected_message=lambda _route: "no target", + ) + + self.assertEqual(selected, [external]) + self.assertIsNone(error) + self.assertEqual( + calls, + [ + { + "require_configured_main_root": mode == "main", + "require_configured_root_match": True, + } + ], + ) + + def test_resolve_action_targets_explicit_project_overrides_current_worktree(self) -> None: + current = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") + requested = _Target(name="feature-b-1", root="/tmp/repo/trees/feature-b/1") + runtime = SimpleNamespace( + discover_projects=lambda mode: [current, requested] if mode == "trees" else [], + selectors_from_passthrough=lambda _args: [], + ) + route = parse_route( + ["test", "--project", "feature-b-1"], + env={"ENVCTL_DEFAULT_MODE": "trees"}, + ) + + selected, error = resolve_action_targets( + runtime=runtime, + route=route, + trees_only=False, + resolve_current_worktree_target=lambda **_kwargs: current, + interactive_selection_allowed=lambda _route: False, + no_target_selected_message=lambda _route: "no target", + ) + + self.assertEqual(selected, [requested]) + self.assertIsNone(error) + + def test_resolve_action_targets_unknown_service_does_not_fall_through_to_single_project(self) -> None: + target = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") + runtime = SimpleNamespace( + discover_projects=lambda mode: [target] if mode == "trees" else [], + selectors_from_passthrough=lambda _args: [], + load_existing_state=lambda mode: None, + project_name_from_service=lambda _name: "", + ) + route = parse_route( + ["test", "--service", "missing-worker"], + env={"ENVCTL_DEFAULT_MODE": "trees"}, + ) + + selected, error = resolve_action_targets( + runtime=runtime, + route=route, + trees_only=False, + resolve_current_worktree_target=lambda **_kwargs: target, + interactive_selection_allowed=lambda _route: False, + no_target_selected_message=lambda _route: "no target", + ) + + self.assertEqual(selected, []) + self.assertEqual(error, "No matching targets found for: missing-worker") + + def test_resolve_action_targets_rejects_partial_service_match(self) -> None: + target = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") + state = SimpleNamespace( + services={"feature-a-1 Worker": SimpleNamespace(type="worker")} + ) + runtime = SimpleNamespace( + discover_projects=lambda mode: [target] if mode == "trees" else [], + selectors_from_passthrough=lambda _args: [], + load_existing_state=lambda mode: state if mode == "trees" else None, + project_name_from_service=lambda name: name.split()[0] if name.startswith("feature-") else "", + ) + route = parse_route( + ["test", "--service", "worker", "--service", "misspelled"], + env={"ENVCTL_DEFAULT_MODE": "trees"}, + ) + + selected, error = resolve_action_targets( + runtime=runtime, + route=route, + trees_only=False, + resolve_current_worktree_target=lambda **_kwargs: target, + interactive_selection_allowed=lambda _route: False, + no_target_selected_message=lambda _route: "no target", + ) + + self.assertEqual(selected, []) + self.assertEqual(error, "No matching targets found for: misspelled") + + def test_resolve_action_targets_untested_does_not_auto_select_single_project(self) -> None: + target = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") + runtime = SimpleNamespace( + discover_projects=lambda mode: [target] if mode == "trees" else [], + selectors_from_passthrough=lambda _args: [], + ) + route = parse_route(["test", "--untested"], env={"ENVCTL_DEFAULT_MODE": "trees"}) + + selected, error = resolve_action_targets( + runtime=runtime, + route=route, + trees_only=False, + resolve_current_worktree_target=lambda **_kwargs: target, + interactive_selection_allowed=lambda _route: False, + no_target_selected_message=lambda _route: "no target", + ) + + self.assertEqual(selected, []) + self.assertIsNone(error) + def test_resolve_action_targets_applies_interactive_selection(self) -> None: target_a = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") target_b = _Target(name="feature-b-1", root="/tmp/repo/trees/feature-b/1") @@ -142,6 +358,29 @@ def apply_to_route(self, route): # noqa: ANN001 self.assertEqual(calls[0]["prompt"], "Select test target") self.assertTrue(calls[0]["allow_untested"]) + def test_resolve_action_targets_rejects_partial_explicit_match(self) -> None: + target = _Target(name="feature-a-1", root="/tmp/repo/trees/feature-a/1") + runtime = SimpleNamespace( + discover_projects=lambda mode: [target] if mode == "trees" else [], + selectors_from_passthrough=lambda _args: [], + ) + route = parse_route( + ["test", "--project", "feature-a-1", "--project", "misspelled"], + env={"ENVCTL_DEFAULT_MODE": "trees"}, + ) + + selected, error = resolve_action_targets( + runtime=runtime, + route=route, + trees_only=False, + resolve_current_worktree_target=lambda **_kwargs: None, + interactive_selection_allowed=lambda _route: False, + no_target_selected_message=lambda _route: "no target", + ) + + self.assertEqual(selected, []) + self.assertEqual(error, "No matching targets found for: misspelled") + def test_emit_action_output_trims_and_emits_status(self) -> None: printed: list[str] = [] emitted: list[str] = [] @@ -363,6 +602,77 @@ def test_execute_targeted_action_can_replace_generic_success_messages(self) -> N self.assertNotIn("ship action succeeded for Main.", printed) self.assertNotIn("ship succeeded for Main", emitted) + def test_execute_targeted_action_counts_invalid_targets_and_keeps_valid_progress_accurate(self) -> None: + printed: list[str] = [] + emitted: list[str] = [] + + code = execute_targeted_action( + targets=[ + _Target(name="", root="/tmp/invalid"), + _Target(name="feature-a-1", root="/tmp/feature-a-1"), + ], + command_name="test", + interactive_command=False, + resolve_command=lambda context: ActionCommandResolution(command=["true"], cwd=context.root), + build_env=lambda _context: {}, + process_run=lambda _command, _cwd, _env: _Completed(returncode=0), + emit_status=emitted.append, + printer=printed.append, + ) + + self.assertEqual(code, 1) + self.assertIn("test action skipped 1 invalid target(s) without both a name and root.", printed) + self.assertIn("Running test for feature-a-1 (1/1)...", emitted) + self.assertIn("test action succeeded for feature-a-1.", printed) + + def test_execute_targeted_action_contains_callback_exception_and_continues_other_targets(self) -> None: + targets = [ + _Target(name="feature-a-1", root="/tmp/feature-a-1"), + _Target(name="feature-b-1", root="/tmp/feature-b-1"), + ] + processed: list[str] = [] + failures: list[tuple[str, str]] = [] + + def build_env(context): # noqa: ANN001,ANN202 + if context.name == "feature-a-1": + raise RuntimeError("broken environment") + return {} + + code = execute_targeted_action( + targets=targets, + command_name="test", + interactive_command=False, + resolve_command=lambda context: ActionCommandResolution(command=["true"], cwd=context.root), + build_env=build_env, + process_run=lambda _command, cwd, _env: processed.append(cwd.name) or _Completed(returncode=0), + emit_status=lambda _message: None, + printer=lambda _message: None, + on_failure=lambda context, error: failures.append((context.name, error)), + ) + + self.assertEqual(code, 1) + self.assertEqual(failures, [("feature-a-1", "RuntimeError: broken environment")]) + self.assertEqual(processed, ["feature-b-1"]) + + def test_execute_targeted_action_rejects_empty_target_list(self) -> None: + printed: list[str] = [] + emitted: list[str] = [] + + code = execute_targeted_action( + targets=[], + command_name="review", + interactive_command=False, + resolve_command=lambda _context: ActionCommandResolution(command=["true"], cwd=Path("/tmp")), + build_env=lambda _context: {}, + process_run=lambda _command, _cwd, _env: _Completed(returncode=0), + emit_status=emitted.append, + printer=printed.append, + ) + + self.assertEqual(code, 1) + self.assertEqual(printed, ["review action failed: no targets were provided."]) + self.assertEqual(emitted, printed) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/actions/test_action_test_command_support.py b/tests/python/actions/test_action_test_command_support.py index f3a73c222..49e0b5b99 100644 --- a/tests/python/actions/test_action_test_command_support.py +++ b/tests/python/actions/test_action_test_command_support.py @@ -10,6 +10,7 @@ from envctl_engine.actions.action_test_command_support import ActionTestExecutionSpecBuilder from envctl_engine.actions.action_test_command_support import SharedTestCommandPlanner from envctl_engine.actions.action_test_command_support import build_test_execution_specs +from envctl_engine.actions.action_test_command_support import build_test_target_contexts from envctl_engine.actions.action_test_command_support import is_legacy_tree_test_script from envctl_engine.actions.action_test_command_support import normalize_backend_python_test_command from envctl_engine.actions.action_test_support_models import TestTargetContext as TargetContext @@ -58,6 +59,16 @@ def test_shared_legacy_tree_command_collapses_to_one_all_targets_spec(self) -> N self.assertEqual(specs[0].spec.cwd, repo) self.assertEqual(specs[0].args, ["projects=feature-a-1", "untested=true"]) + def test_empty_target_contexts_keep_main_fallback_for_non_aggregate_callers(self) -> None: + repo = Path("/repo") + + contexts = build_test_target_contexts([], repo_root=repo) + + self.assertEqual(len(contexts), 1) + self.assertEqual(contexts[0].project_name, "Main") + self.assertEqual(contexts[0].project_root, repo) + self.assertIsNone(contexts[0].target_obj) + def test_legacy_tree_script_detection_accepts_absolute_bash_path(self) -> None: self.assertTrue(is_legacy_tree_test_script(["bash", "scripts/test-all-trees.sh"])) self.assertTrue(is_legacy_tree_test_script(["/bin/bash", "/repo/scripts/test-all-trees.sh"])) diff --git a/tests/python/actions/test_action_test_runner.py b/tests/python/actions/test_action_test_runner.py index 46f901ef1..77ae61bee 100644 --- a/tests/python/actions/test_action_test_runner.py +++ b/tests/python/actions/test_action_test_runner.py @@ -49,6 +49,38 @@ def _run(orchestrator: _Orchestrator, route, targets: list[object]) -> int: # n class ActionTestRunnerShipOnPassTests(unittest.TestCase): + def test_untested_scope_without_matrix_command_fails_without_running_main(self) -> None: + orchestrator = _Orchestrator() + route = parse_route(["test", "--untested"], env={}) + + with ( + patch.object( + action_test_runner, + "build_test_action_execution_plan", + return_value=SimpleNamespace( + execution_specs=[], + interactive_command=False, + missing_execution_specs_message=( + "No test command supports the all-untested scope. " + "Configure a test-all-trees.sh action command or select projects explicitly." + ), + ), + ), + patch("builtins.print") as printer, + patch.object( + action_test_runner, + "execute_test_suites", + side_effect=AssertionError("test suites must not run"), + ), + ): + code = _run(orchestrator, route, []) + + self.assertEqual(code, 1) + printer.assert_called_once_with( + "No test command supports the all-untested scope. " + "Configure a test-all-trees.sh action command or select projects explicitly." + ) + def test_ship_on_pass_rejects_blank_message_before_planning_tests(self) -> None: orchestrator = _Orchestrator() route = parse_route(["test", "--ship-on-pass="], env={}) diff --git a/tests/python/actions/test_action_worktree_runner.py b/tests/python/actions/test_action_worktree_runner.py index 39e25d99c..46ff77784 100644 --- a/tests/python/actions/test_action_worktree_runner.py +++ b/tests/python/actions/test_action_worktree_runner.py @@ -74,9 +74,9 @@ def test_worktree_runner_uses_named_owners_for_target_resolution_and_delete_exec target_resolution_source = ( REPO_ROOT / "python/envctl_engine/actions/action_worktree_target_resolution.py" ).read_text(encoding="utf-8") - self_destruct_source = ( - REPO_ROOT / "python/envctl_engine/actions/action_worktree_self_destruct.py" - ).read_text(encoding="utf-8") + self_destruct_source = (REPO_ROOT / "python/envctl_engine/actions/action_worktree_self_destruct.py").read_text( + encoding="utf-8" + ) self.assertIn("class ActionWorktreeDeleteRunner", source) self.assertIn("return ActionWorktreeDeleteRunner(", source) @@ -106,9 +106,9 @@ def test_resolve_current_worktree_target_discovers_matching_current_tree(self) - runtime=runtime, require_configured_main_root=True, current_cwd=lambda: Path("/should/not/be/used"), - discover_tree_projects_fn=lambda repo_root, trees_dir_name: [("feature-a-1", tree_root)] - if repo_root == repo and trees_dir_name == "trees" - else [], + discover_tree_projects_fn=lambda repo_root, trees_dir_name: ( + [("feature-a-1", tree_root)] if repo_root == repo and trees_dir_name == "trees" else [] + ), main_repo_root_for_linked_worktree_fn=lambda _worktree_root: None, git_main_repo_root_for_worktree_fn=lambda _worktree_root, trees_dir_name=None: repo, ) @@ -117,6 +117,66 @@ def test_resolve_current_worktree_target_discovers_matching_current_tree(self) - self.assertEqual(getattr(target, "name"), "feature-a-1") self.assertEqual(getattr(target, "root"), tree_root) + def test_resolve_current_worktree_target_accepts_external_linked_checkout_in_both_modes(self) -> None: + repo = Path("/tmp/repo").resolve() + linked_root = Path("/tmp/envctl-deep-code-cleanup").resolve() + invocation_cwd = linked_root / "python" / "envctl_engine" + + def run(command, *, cwd, timeout): # noqa: ANN001 + self.assertEqual(timeout, 10.0) + if command == ["git", "rev-parse", "--show-toplevel"]: + return SimpleNamespace(returncode=0, stdout=f"{linked_root}\n") + if command == ["git", "branch", "--show-current"]: + self.assertEqual(Path(cwd), linked_root) + return SimpleNamespace(returncode=0, stdout="agent/deep-code-cleanup\n") + return SimpleNamespace(returncode=1, stdout="") + + runtime = SimpleNamespace( + env={"ENVCTL_INVOCATION_CWD": str(invocation_cwd)}, + raw_runtime=SimpleNamespace( + config=SimpleNamespace(base_dir=repo, trees_dir_name="trees"), + process_runner=SimpleNamespace(run=run), + ), + ) + + for require_configured_main_root in (False, True): + with self.subTest(require_configured_main_root=require_configured_main_root): + target = resolve_current_worktree_target( + runtime=runtime, + require_configured_main_root=require_configured_main_root, + require_configured_root_match=True, + current_cwd=lambda: Path("/should/not/be/used"), + discover_tree_projects_fn=lambda _repo_root, _trees_dir_name: [], + main_repo_root_for_linked_worktree_fn=lambda _worktree_root: None, + git_main_repo_root_for_worktree_fn=lambda **_kwargs: repo, + ) + + self.assertIsNotNone(target) + self.assertEqual(getattr(target, "name"), "agent/deep-code-cleanup") + self.assertEqual(getattr(target, "root"), linked_root) + + def test_resolve_current_worktree_target_does_not_synthesize_main_checkout(self) -> None: + repo = Path("/tmp/repo").resolve() + runtime = SimpleNamespace( + env={"ENVCTL_INVOCATION_CWD": str(repo / "python")}, + raw_runtime=SimpleNamespace( + config=SimpleNamespace(base_dir=repo, trees_dir_name="trees"), + process_runner=SimpleNamespace( + run=lambda *_args, **_kwargs: SimpleNamespace(returncode=0, stdout=f"{repo}\n") + ), + ), + ) + + target = resolve_current_worktree_target( + runtime=runtime, + require_configured_main_root=True, + require_configured_root_match=True, + discover_tree_projects_fn=lambda _repo_root, _trees_dir_name: [], + git_main_repo_root_for_worktree_fn=lambda **_kwargs: repo, + ) + + self.assertIsNone(target) + def test_run_delete_worktree_action_runs_cleanup_and_delete(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo_root = Path(tmpdir) / "repo" @@ -245,15 +305,14 @@ def test_run_self_destruct_worktree_action_uses_explicit_current_worktree_safety calls: list[tuple[str, object]] = [] runtime = SimpleNamespace( - _blast_worktree_before_delete=lambda **kwargs: ( - calls.append(("cleanup", kwargs)) or ["cleanup warning"] - ), + _blast_worktree_before_delete=lambda **kwargs: calls.append(("cleanup", kwargs)) or ["cleanup warning"], _trees_root_for_worktree=lambda value: calls.append(("trees_root", value)) or tree_root.parent, ) orchestrator = SimpleNamespace( runtime=runtime, - resolve_targets=lambda route, *, trees_only: calls.append(("resolve", (route.command, trees_only))) - or ([target], None), + resolve_targets=lambda route, *, trees_only: ( + calls.append(("resolve", (route.command, trees_only))) or ([target], None) + ), _main_repo_root_for_worktree=lambda value: calls.append(("main_repo", value)) or tree_root.parent, _spawn_self_destruct_helper=lambda **kwargs: calls.append(("spawn", kwargs)) or True, ) diff --git a/tests/python/actions/test_actions_cli_commit_protected_artifacts.py b/tests/python/actions/test_actions_cli_commit_protected_artifacts.py index f7367ad18..92e6a2e44 100644 --- a/tests/python/actions/test_actions_cli_commit_protected_artifacts.py +++ b/tests/python/actions/test_actions_cli_commit_protected_artifacts.py @@ -16,7 +16,7 @@ def test_commit_action_skips_envctl_local_artifacts_and_commits_normal_changes(t assert result.code == 0 assert "Skipping envctl-local artifacts: .envctl-commit-message.md, MAIN_TASK.md" in result.output - assert ["add", "--", "app.py"] in harness.seen_git_args + assert ["add", "--all", "--", "app.py"] in harness.seen_git_args assert ["push", "-u", "origin", "feature/demo"] in harness.seen_git_args @@ -57,7 +57,7 @@ def test_commit_action_recovers_when_envctl_local_artifacts_are_already_staged(t assert "Skipping envctl-local artifacts: MAIN_TASK.md, .envctl-commit-message.md" in result.output assert "Refusing to commit" not in result.output assert ["reset", "-q", "--", "MAIN_TASK.md"] in harness.seen_git_args - assert ["add", "--", "app.py"] in harness.seen_git_args + assert ["add", "--all", "--", "app.py"] in harness.seen_git_args assert ["push", "-u", "origin", "feature/demo"] in harness.seen_git_args diff --git a/tests/python/actions/test_actions_cli_pr.py b/tests/python/actions/test_actions_cli_pr.py index e92c9dd72..3ceae7fe1 100644 --- a/tests/python/actions/test_actions_cli_pr.py +++ b/tests/python/actions/test_actions_cli_pr.py @@ -2,6 +2,7 @@ from __future__ import annotations from tests.python.actions.actions_cli_test_support import * # noqa: F403,F405 +from envctl_engine.actions.action_git_state_support import ExistingPullRequest class ActionsCliPrTests(unittest.TestCase): @@ -31,11 +32,14 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch( - "envctl_engine.actions.project_action_domain.existing_pr_url", - return_value="https://github.com/acme/supportopia/pull/42", + "envctl_engine.actions.project_action_domain.existing_pull_request", + return_value=ExistingPullRequest( + url="https://github.com/acme/supportopia/pull/42", + base_branch="main", + ), ), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), @@ -93,9 +97,9 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.probe_dirty_worktree") as dirty_probe, patch("envctl_engine.actions.project_action_domain.run_commit_action", return_value=0) as commit_action, patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -144,9 +148,9 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.probe_dirty_worktree") as dirty_probe, patch("envctl_engine.actions.project_action_domain.run_commit_action", return_value=1) as commit_action, patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -221,9 +225,9 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -300,9 +304,9 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -367,9 +371,9 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -429,9 +433,9 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch.dict(os.environ, {"ENVCTL_PR_BODY": "Typed PR body\n\nWith details."}, clear=False), - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -493,9 +497,9 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 patch.dict(os.environ, {"ENVCTL_ACTION_INTERACTIVE": "1"}, clear=False), patch("sys.stdin.isatty", return_value=True), patch("builtins.input", side_effect=AssertionError("input() should not be called for pr action")), - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -551,9 +555,9 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -600,9 +604,9 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -657,9 +661,9 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 return subprocess.CompletedProcess(args=command, returncode=1, stdout="", stderr="unexpected command") with ( - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): diff --git a/tests/python/actions/test_actions_cli_ship.py b/tests/python/actions/test_actions_cli_ship.py index 569e6da2a..4332ec872 100644 --- a/tests/python/actions/test_actions_cli_ship.py +++ b/tests/python/actions/test_actions_cli_ship.py @@ -1,6 +1,8 @@ # ruff: noqa: F403,F405 from __future__ import annotations +from envctl_engine.actions.action_git_state_support import ExistingPullRequest +from envctl_engine.actions.action_protected_artifacts import unstaged_stageable_paths from tests.python.actions.actions_cli_test_support import * # noqa: F403,F405 @@ -34,6 +36,21 @@ def test_partition_envctl_protected_paths_separates_staged_skipped_and_stageable ) self.assertEqual(partition.stageable_paths, ["app.py", "docs/reference/commands.md", "new_name.py"]) + def test_unstaged_stageable_paths_excludes_staged_only_changes_and_keeps_rename_source(self) -> None: + self.assertEqual( + unstaged_stageable_paths( + "D already-deleted.py\n" + "M already-modified.py\n" + " D deleted.py\n" + " M modified.py\n" + "MM both.py\n" + " R old_name.py -> new_name.py\n" + "?? new.py\n" + " M MAIN_TASK.md\n" + ), + ["deleted.py", "modified.py", "both.py", "old_name.py", "new_name.py", "new.py"], + ) + def test_ship_action_reuses_commit_and_pr_then_reports_passed_checks_json(self) -> None: domain = importlib.import_module("envctl_engine.actions.project_action_domain") with tempfile.TemporaryDirectory() as tmpdir: @@ -80,7 +97,14 @@ def fake_commit_action(_context: object) -> int: patch("envctl_engine.actions.project_action_domain.add_ship_pr_label", return_value=0) as label_pr, patch( "envctl_engine.actions.project_action_domain.existing_pr_url", - side_effect=["", "https://github.com/acme/repo/pull/7"], + return_value="https://github.com/acme/repo/pull/7", + ), + patch( + "envctl_engine.actions.project_action_domain.existing_pull_request", + side_effect=[ + None, + ExistingPullRequest("https://github.com/acme/repo/pull/7", "main"), + ], ), patch( "envctl_engine.actions.project_action_domain._github_pr_checks", @@ -169,6 +193,10 @@ def fake_run_git(_git_root: Path, args: list[str]) -> subprocess.CompletedProces "envctl_engine.actions.project_action_domain.existing_pr_url", return_value="https://github.com/acme/repo/pull/7", ), + patch( + "envctl_engine.actions.project_action_domain.existing_pull_request", + return_value=ExistingPullRequest("https://github.com/acme/repo/pull/7", "main"), + ), patch( "envctl_engine.actions.project_action_domain._github_pr_checks", return_value={ @@ -236,7 +264,7 @@ def fake_run(git_root: Path, args: list[str]): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.shutil.which", return_value="/usr/bin/git"), - patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), + patch("envctl_engine.actions.project_action_domain.git_state_support.detect_pr_base_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch("envctl_engine.actions.project_action_domain._run_git", side_effect=fake_run), patch("envctl_engine.actions.project_action_domain.run_commit_action", return_value=0) as commit_action, @@ -245,6 +273,10 @@ def fake_run(git_root: Path, args: list[str]): # noqa: ANN001 "envctl_engine.actions.project_action_domain.existing_pr_url", return_value="https://github.com/acme/repo/pull/7", ), + patch( + "envctl_engine.actions.project_action_domain.existing_pull_request", + return_value=ExistingPullRequest("https://github.com/acme/repo/pull/7", "main"), + ), redirect_stdout(StringIO()) as stdout, ): code = domain.run_ship_action(context) @@ -284,11 +316,7 @@ def fake_git_output(_git_root: Path, args: list[str]) -> str: if args == ["diff", "--name-only", "--diff-filter=U"]: return "src/service.py\n" if args == ["ls-files", "-u"]: - return ( - "100644 aaa 1\tsrc/service.py\n" - "100644 bbb 2\tsrc/service.py\n" - "100644 ccc 3\tsrc/service.py\n" - ) + return "100644 aaa 1\tsrc/service.py\n100644 bbb 2\tsrc/service.py\n100644 ccc 3\tsrc/service.py\n" return "" with ( diff --git a/tests/python/actions/test_actions_git_parity.py b/tests/python/actions/test_actions_git_parity.py index d4eab762e..9bd724713 100644 --- a/tests/python/actions/test_actions_git_parity.py +++ b/tests/python/actions/test_actions_git_parity.py @@ -1,5 +1,8 @@ from __future__ import annotations +import os +import shlex +import subprocess import sys import tempfile from contextlib import redirect_stdout @@ -19,6 +22,9 @@ ) +REPO_ROOT = Path(__file__).resolve().parents[3] + + class ActionsGitParityTests(_ActionsParityTestCase): def test_headless_review_auto_selects_single_main_target(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -225,6 +231,76 @@ def test_ship_without_project_infers_current_worktree_in_trees_mode(self) -> Non self.assertEqual(fake_runner.run_envs[0]["ENVCTL_ACTION_PROJECT"], "feature-a-1") self.assertEqual(fake_runner.run_envs[0]["ENVCTL_ACTION_PROJECT_ROOT"], str(target.resolve())) + def test_repo_wrapper_ship_targets_external_linked_worktree_in_main_and_trees_modes(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = root / "repo" + linked = root / "external-linked-worktree" + repo.mkdir() + + def git(*args: str, cwd: Path = repo) -> None: + subprocess.run( + ["git", *args], + cwd=cwd, + check=True, + capture_output=True, + text=True, + ) + + git("init", "-q", "-b", "main") + git("config", "user.email", "envctl-test@example.test") + git("config", "user.name", "Envctl Test") + (repo / ".envctl").write_text( + "ENVCTL_DEFAULT_MODE=trees\nMAIN_STARTUP_ENABLE=false\nTREES_STARTUP_ENABLE=false\n", + encoding="utf-8", + ) + (repo / "README.md").write_text("# linked worktree ship test\n", encoding="utf-8") + git("add", "-f", ".envctl", "README.md") + git("commit", "-q", "-m", "initial") + git("worktree", "add", "-q", "-b", "feature/external", str(linked)) + + invocation_cwd = linked / "nested" / "source" + invocation_cwd.mkdir(parents=True) + assertion_script = ( + "import os,sys; from pathlib import Path; " + "expected=Path(sys.argv[1]).resolve(); " + "ok=(Path.cwd().resolve()==expected " + "and os.environ.get('ENVCTL_ACTION_PROJECT')=='feature/external' " + "and Path(os.environ.get('ENVCTL_ACTION_PROJECT_ROOT','')).resolve()==expected); " + "raise SystemExit(0 if ok else 9)" + ) + harmless_ship_command = shlex.join( + [sys.executable, "-c", assertion_script, str(linked.resolve())] + ) + + for mode in ("trees", "main"): + with self.subTest(mode=mode): + env = dict(os.environ) + for key in ("RUN_REPO_ROOT", "ENVCTL_EXECUTION_ROOT", "ENVCTL_INVOCATION_CWD"): + env.pop(key, None) + env.update( + { + "ENVCTL_ACTION_SHIP_CMD": harmless_ship_command, + "ENVCTL_DEFAULT_MODE": mode, + "ENVCTL_UI_BACKEND": "non_interactive", + "ENVCTL_USE_REPO_WRAPPER": "1", + "RUN_SH_RUNTIME_DIR": str(root / f"runtime-{mode}"), + } + ) + completed = subprocess.run( + [str(REPO_ROOT / "bin" / "envctl"), "ship", "--human"], + cwd=invocation_cwd, + env=env, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + self.assertEqual(completed.returncode, 0, msg=completed.stdout + completed.stderr) + self.assertIn("ship handoff status for feature/external: success", completed.stdout) + self.assertNotIn("ship handoff status for Main", completed.stdout) + def test_git_actions_use_python_native_defaults_when_available(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo = Path(tmpdir) / "repo" diff --git a/tests/python/actions/test_actions_migrate_parity.py b/tests/python/actions/test_actions_migrate_parity.py index 84bc451e6..b2217f6b3 100644 --- a/tests/python/actions/test_actions_migrate_parity.py +++ b/tests/python/actions/test_actions_migrate_parity.py @@ -546,7 +546,7 @@ def test_migrate_action_uses_distinct_env_contracts_for_each_target_in_multi_tar fake_runner = _FakeRunner(returncode=0) engine.process_runner = fake_runner # type: ignore[assignment] - route = parse_route(["migrate", "--all"], env={"ENVCTL_DEFAULT_MODE": "trees"}) + route = parse_route(["migrate", "--all", "--yes"], env={"ENVCTL_DEFAULT_MODE": "trees"}) code = engine.dispatch(route) self.assertEqual(code, 0) @@ -567,4 +567,3 @@ def test_migrate_action_uses_distinct_env_contracts_for_each_target_in_multi_tar ) self.assertEqual(first_env.get("SQLALCHEMY_DATABASE_URL"), first_env.get("DATABASE_URL")) self.assertEqual(second_env.get("SQLALCHEMY_DATABASE_URL"), second_env.get("DATABASE_URL")) - diff --git a/tests/python/debug/test_debug_utils_file_lock.py b/tests/python/debug/test_debug_utils_file_lock.py new file mode 100644 index 000000000..46cb5a970 --- /dev/null +++ b/tests/python/debug/test_debug_utils_file_lock.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from pathlib import Path + +from envctl_engine.debug.debug_utils import file_lock + + +def test_file_lock_keeps_stable_inode_after_release(tmp_path: Path) -> None: + lock_path = tmp_path / "stable.lock" + + with file_lock(lock_path, timeout=1.0): + inode = lock_path.stat().st_ino + + assert lock_path.stat().st_ino == inode diff --git a/tests/python/planning/plan_agent_launch_support_test_support.py b/tests/python/planning/plan_agent_launch_support_test_support.py index 260aa7385..2c6110202 100644 --- a/tests/python/planning/plan_agent_launch_support_test_support.py +++ b/tests/python/planning/plan_agent_launch_support_test_support.py @@ -26,25 +26,28 @@ terminal_screen, tmux_session, tmux_transport, - workflow, + workflow_build, + workflow_prompt_support, + workflow_review_support, + workflow_runtime_support, ) -from envctl_engine.planning.plan_agent.models import _WorkspaceLaunchTarget +from envctl_engine.planning.plan_agent.models import _PlanAgentWorkflowStep, _WorkspaceLaunchTarget from envctl_engine.planning.plan_agent_launch_support import CreatedPlanWorktree, launch_plan_agent_terminals from envctl_engine.runtime.command_router import parse_route from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord _screen_looks_ready = terminal_screen._screen_looks_ready _prompt_submit_screen_looks_ready = terminal_screen._prompt_submit_screen_looks_ready -_tab_title_for_worktree = workflow._tab_title_for_worktree -_build_plan_agent_workflow = cast(Any, workflow._build_plan_agent_workflow) -_finalization_instruction_text = cast(Any, workflow._finalization_instruction_text) -_first_cycle_completion_instruction_text = cast(Any, workflow._first_cycle_completion_instruction_text) -_intermediate_cycle_completion_instruction_text = cast(Any, workflow._intermediate_cycle_completion_instruction_text) -_browser_e2e_instruction_text = cast(Any, workflow._browser_e2e_instruction_text) -_pr_review_comments_instruction_text = cast(Any, workflow._pr_review_comments_instruction_text) +_tab_title_for_worktree = workflow_build._tab_title_for_worktree +_build_plan_agent_workflow = cast(Any, workflow_build._build_plan_agent_workflow) +_finalization_instruction_text = cast(Any, workflow_build._finalization_instruction_text) +_first_cycle_completion_instruction_text = cast(Any, workflow_build._first_cycle_completion_instruction_text) +_intermediate_cycle_completion_instruction_text = cast(Any, workflow_build._intermediate_cycle_completion_instruction_text) +_browser_e2e_instruction_text = cast(Any, workflow_build._browser_e2e_instruction_text) +_pr_review_comments_instruction_text = cast(Any, workflow_build._pr_review_comments_instruction_text) _wait_for_codex_queue_ready = cast(Any, cmux_transport._wait_for_codex_queue_ready) _codex_goal_screen_looks_active = cast(Any, terminal_screen._codex_goal_screen_looks_active) -_workflow_step_prompt_text = cast(Any, workflow._workflow_step_prompt_text) +_workflow_step_prompt_text = cast(Any, workflow_prompt_support._workflow_step_prompt_text) _ensure_tmux_window = cast(Any, tmux_transport._ensure_tmux_window) _tmux_session_name_for_worktree = cast(Any, tmux_transport._tmux_session_name_for_worktree) _run_tmux_worktree_bootstrap = cast(Any, tmux_transport._run_tmux_worktree_bootstrap) @@ -124,8 +127,14 @@ class _StateRepositoryHarness: def __init__(self, state: RunState | None) -> None: self.state = state - def load_latest(self, *, mode: str | None = None, strict_mode_match: bool = False) -> RunState | None: - _ = mode, strict_mode_match + def load_latest( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: list[str] | None = None, + ) -> RunState | None: + _ = mode, strict_mode_match, project_names return self.state diff --git a/tests/python/planning/test_plan_agent_launch_cmux_goal.py b/tests/python/planning/test_plan_agent_launch_cmux_goal.py index 15c24f567..e12741c8d 100644 --- a/tests/python/planning/test_plan_agent_launch_cmux_goal.py +++ b/tests/python/planning/test_plan_agent_launch_cmux_goal.py @@ -7,7 +7,7 @@ class PlanAgentLaunchCmuxGoalTests(PlanAgentLaunchSupportTestCase): def test_codex_goal_text_is_compact_and_main_task_first(self) -> None: - goal = workflow._codex_goal_text_for_worktree( + goal = workflow_runtime_support._codex_goal_text_for_worktree( worktree=CreatedPlanWorktree(name="feature-a-1", root=Path("/repo"), plan_file="features/a.md"), preset="implement_task", workflow_mode="codex_cycles", @@ -24,7 +24,7 @@ def test_codex_goal_text_is_compact_and_main_task_first(self) -> None: self.assertLessEqual(len(goal), 180) def test_codex_goal_text_preserves_omx_completion_contract(self) -> None: - goal = workflow._codex_goal_text_for_worktree( + goal = workflow_runtime_support._codex_goal_text_for_worktree( worktree=CreatedPlanWorktree(name="feature-a-1", root=Path("/repo"), plan_file="features/a.md"), preset="implement_task", workflow_mode="single_prompt", diff --git a/tests/python/planning/test_plan_agent_launch_cmux_review.py b/tests/python/planning/test_plan_agent_launch_cmux_review.py index 9c192b976..e3817f49e 100644 --- a/tests/python/planning/test_plan_agent_launch_cmux_review.py +++ b/tests/python/planning/test_plan_agent_launch_cmux_review.py @@ -168,7 +168,7 @@ def test_review_launch_resolves_original_plan_file_from_worktree_provenance(self json.dumps({"schema_version": 1, "plan_file": "implementations/feature-a.md"}) + "\n", encoding="utf-8", ) - original_plan_path = workflow._review_original_plan_path( + original_plan_path = workflow_review_support._review_original_plan_path( "feature-a-1", project_root, repo_root=repo, @@ -182,7 +182,7 @@ def test_review_launch_returns_none_when_original_plan_file_cannot_be_resolved(s project_root = repo / "trees" / "feature-a" / "1" project_root.mkdir(parents=True, exist_ok=True) - original_plan_path = workflow._review_original_plan_path( + original_plan_path = workflow_review_support._review_original_plan_path( "feature-a-1", project_root, repo_root=repo, @@ -204,7 +204,7 @@ def test_review_launch_does_not_infer_when_recorded_plan_file_is_missing(self) - encoding="utf-8", ) - original_plan_path = workflow._review_original_plan_path( + original_plan_path = workflow_review_support._review_original_plan_path( "implementations_task-1", project_root, repo_root=repo, @@ -221,7 +221,7 @@ def test_review_launch_can_infer_original_plan_file_without_provenance(self) -> original_plan.parent.mkdir(parents=True, exist_ok=True) original_plan.write_text("# done plan\n", encoding="utf-8") - original_plan_path = workflow._review_original_plan_path( + original_plan_path = workflow_review_support._review_original_plan_path( "implementations_task-1", project_root, repo_root=repo, @@ -241,7 +241,7 @@ def test_review_launch_prefers_active_plan_over_archived_copy(self) -> None: active_plan.write_text("# active plan\n", encoding="utf-8") archived_plan.write_text("# archived plan\n", encoding="utf-8") - original_plan_path = workflow._review_original_plan_path( + original_plan_path = workflow_review_support._review_original_plan_path( "features_task", project_root, repo_root=repo, diff --git a/tests/python/planning/test_plan_agent_launch_omx_workflow.py b/tests/python/planning/test_plan_agent_launch_omx_workflow.py index 846d34f17..8d8a5476d 100644 --- a/tests/python/planning/test_plan_agent_launch_omx_workflow.py +++ b/tests/python/planning/test_plan_agent_launch_omx_workflow.py @@ -175,7 +175,10 @@ def test_omx_workflow_launch_does_not_double_wrap_existing_keyword(self) -> None with self.subTest(workflow_name=workflow_name): prompt = f"${workflow_name}\n\nIMPLEMENT TASK BODY" self.assertEqual( - workflow._wrap_omx_initial_prompt_for_workflow(prompt, workflow=workflow_name), + workflow_runtime_support._wrap_omx_initial_prompt_for_workflow( + prompt, + workflow=workflow_name, + ), prompt, ) diff --git a/tests/python/planning/test_plan_agent_launch_prompt_options.py b/tests/python/planning/test_plan_agent_launch_prompt_options.py index c0ca018fb..57b3a2037 100644 --- a/tests/python/planning/test_plan_agent_launch_prompt_options.py +++ b/tests/python/planning/test_plan_agent_launch_prompt_options.py @@ -48,7 +48,7 @@ def test_resolve_plan_agent_launch_config_no_ulw_loop_route_disables_default_pre ) rt = self._runtime(repo, runtime, env={"HOME": tmpdir}) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( rt, launch_config=launch_config, cli="opencode", diff --git a/tests/python/planning/test_plan_agent_launch_workflow_prompt.py b/tests/python/planning/test_plan_agent_launch_workflow_prompt.py index 8637a2148..33e26176e 100644 --- a/tests/python/planning/test_plan_agent_launch_workflow_prompt.py +++ b/tests/python/planning/test_plan_agent_launch_workflow_prompt.py @@ -26,7 +26,7 @@ def test_resolve_preset_submission_text_defaults_ulw_loop_for_explicit_opencode( ) rt = self._runtime(repo, runtime, env={"HOME": tmpdir}) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( rt, launch_config=launch_config, cli="opencode", @@ -48,7 +48,7 @@ def test_workflow_step_prompt_text_uses_codex_skill_command_for_direct_steps(sel env={}, process_runner=_RecordingRunner(), ) - step = workflow._PlanAgentWorkflowStep(kind="submit_direct_prompt", text="implement_task") + step = _PlanAgentWorkflowStep(kind="submit_direct_prompt", text="implement_task") prompt_text, error = _workflow_step_prompt_text( runtime, @@ -89,7 +89,7 @@ def test_workflow_step_prompt_text_keeps_codex_skill_command_single_line_with_wo env={}, process_runner=_RecordingRunner(), ) - step = workflow._PlanAgentWorkflowStep(kind="submit_direct_prompt", text="implement_task") + step = _PlanAgentWorkflowStep(kind="submit_direct_prompt", text="implement_task") worktree = CreatedPlanWorktree(name="feature-a-1", root=worktree_root, plan_file="feature-a.md") prompt_text, error = _workflow_step_prompt_text( @@ -116,7 +116,7 @@ def test_codex_preset_submission_text_collapses_arguments_into_skill_command(sel env={}, process_runner=_RecordingRunner(), ) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( runtime, launch_config=_launch_config_for_tests(cli="codex"), cli="codex", @@ -162,7 +162,7 @@ def test_resolve_preset_submission_text_returns_opencode_direct_body_when_enable ulw_suffix=False, ) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( runtime, launch_config=launch_config, cli="opencode", @@ -199,7 +199,7 @@ def test_resolve_preset_submission_text_keeps_slash_mode_for_opencode_by_default ulw_suffix=False, ) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( runtime, launch_config=launch_config, cli="opencode", @@ -240,7 +240,7 @@ def test_resolve_preset_submission_text_prepends_ulw_loop_for_direct_prompts(sel ulw_suffix=False, ) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( runtime, launch_config=launch_config, cli="opencode", @@ -278,7 +278,7 @@ def test_resolve_preset_submission_text_appends_ulw_suffix_in_slash_mode(self) - ulw_suffix=True, ) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( runtime, launch_config=launch_config, cli="opencode", @@ -315,7 +315,7 @@ def test_resolve_preset_submission_text_rejects_ulw_loop_prefix_in_slash_mode(se ulw_suffix=False, ) - prompt_text, error = workflow._resolve_preset_submission_text( + prompt_text, error = workflow_prompt_support._resolve_preset_submission_text( runtime, launch_config=launch_config, cli="opencode", @@ -326,7 +326,7 @@ def test_resolve_preset_submission_text_rejects_ulw_loop_prefix_in_slash_mode(se self.assertEqual(error, "prompt_resolution_failed: ulw_loop_prefix_requires_direct_prompt") def test_shape_prompt_text_rejects_multiple_slash_commands_for_direct_prompts(self) -> None: - shaped, error = workflow._shape_prompt_text( + shaped, error = workflow_prompt_support._shape_prompt_text( "/implement_task", direct_prompt=True, ulw_loop_prefix=True, @@ -337,7 +337,7 @@ def test_shape_prompt_text_rejects_multiple_slash_commands_for_direct_prompts(se self.assertEqual(error, "prompt_resolution_failed: multiple_slash_commands_not_allowed") def test_shape_prompt_text_rejects_embedded_second_slash_command_for_direct_prompts(self) -> None: - shaped, error = workflow._shape_prompt_text( + shaped, error = workflow_prompt_support._shape_prompt_text( "Implement this directly.\n/implement_task", direct_prompt=True, ulw_loop_prefix=True, @@ -348,7 +348,7 @@ def test_shape_prompt_text_rejects_embedded_second_slash_command_for_direct_prom self.assertEqual(error, "prompt_resolution_failed: multiple_slash_commands_not_allowed") def test_shape_prompt_text_preserves_existing_ulw_loop_prefix_and_allows_suffix(self) -> None: - shaped, error = workflow._shape_prompt_text( + shaped, error = workflow_prompt_support._shape_prompt_text( "/ulw-loop Implement this directly.", direct_prompt=True, ulw_loop_prefix=True, @@ -359,7 +359,7 @@ def test_shape_prompt_text_preserves_existing_ulw_loop_prefix_and_allows_suffix( self.assertEqual(shaped, "/ulw-loop Implement this directly. ulw") def test_shape_prompt_text_allows_absolute_path_literals_in_direct_prompts(self) -> None: - shaped, error = workflow._shape_prompt_text( + shaped, error = workflow_prompt_support._shape_prompt_text( 'Review bundle: "/tmp/review.md"\nWorktree directory: "/tmp/tree"', direct_prompt=True, ulw_loop_prefix=True, @@ -371,7 +371,7 @@ def test_shape_prompt_text_allows_absolute_path_literals_in_direct_prompts(self) self.assertIn('Review bundle: "/tmp/review.md"', shaped) def test_review_prompt_arguments_preserve_newlines_for_direct_codex_submission(self) -> None: - rendered = workflow._review_prompt_arguments( + rendered = workflow_review_support._review_prompt_arguments( project_name="feature-a-1", project_root=Path("/tmp/worktree path"), review_bundle_path=Path("/tmp/review bundle.md"), diff --git a/tests/python/planning/test_plan_agent_module_layout.py b/tests/python/planning/test_plan_agent_module_layout.py index d5de7b910..9c4942c18 100644 --- a/tests/python/planning/test_plan_agent_module_layout.py +++ b/tests/python/planning/test_plan_agent_module_layout.py @@ -67,13 +67,13 @@ def test_plan_agent_package_modules_exist(self) -> None: "tmux_worktree_launch_support.py", "tmux_launch_support.py", "tmux_transport.py", - "workflow.py", "workflow_build.py", "workflow_e2e_prompt_context.py", "workflow_prompt_support.py", "workflow_runtime_addresses.py", "workflow_queue_support.py", "workflow_review_support.py", + "workflow_runtime_support.py", } actual = {path.name for path in PLAN_AGENT_ROOT.glob("*.py")} self.assertTrue(expected.issubset(actual)) @@ -198,10 +198,12 @@ def test_launch_module_owns_public_dispatch_functions(self) -> None: def test_core_helpers_live_in_owner_modules(self) -> None: expected = { "config.py": { - "_parse_codex_cycles", "resolve_plan_agent_launch_config", "plan_agent_launch_prereq_commands", }, + "launch_policy.py": { + "parse_codex_cycles", + }, "launch_evaluation.py": { "PlanAgentLaunchEvaluation", "build_plan_agent_launch_evaluation", @@ -233,9 +235,10 @@ def test_core_helpers_live_in_owner_modules(self) -> None: "_state_project_matches_worktree", "_state_service_matches_worktree", }, - "workflow.py": { + "workflow_runtime_support.py": { "_codex_goal_text_for_worktree", "_emit_codex_goal_event", + "_surface_respawn_command", "_wrap_omx_initial_prompt_for_workflow", }, "workflow_queue_support.py": { @@ -464,7 +467,7 @@ def test_no_placeholder_extraction_modules_remain(self) -> None: self.assertNotIn("Constants stay in launch", text) def test_plan_agent_facades_use_explicit_compatibility_imports(self) -> None: - for filename in ("config.py", "launch.py", "workflow.py", "recovery.py"): + for filename in ("config.py", "launch.py", "recovery.py"): path = PLAN_AGENT_ROOT / filename tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) wildcard_imports = [ diff --git a/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py b/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py index 127f9fcd4..003136731 100644 --- a/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py +++ b/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py @@ -24,8 +24,14 @@ class _StateRepositoryHarness: def __init__(self, state: RunState | None) -> None: self.state = state - def load_latest(self, *, mode: str | None = None, strict_mode_match: bool = False) -> RunState | None: - _ = mode, strict_mode_match + def load_latest( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: list[str] | None = None, + ) -> RunState | None: + _ = mode, strict_mode_match, project_names return self.state @@ -37,6 +43,43 @@ def __init__(self, state: RunState | None) -> None: class PlanAgentWorkflowRuntimeAddressTests(unittest.TestCase): + def test_worktree_address_lookup_retries_unfiltered_for_base_project_state(self) -> None: + calls: list[list[str] | None] = [] + state = RunState( + run_id="run-base", + mode="trees", + requirements={ + "feature-a": RequirementsResult( + project="feature-a", + redis={"enabled": True, "final": 6381}, + ) + }, + ) + + class BaseProjectRepository: + def load_latest( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: list[str] | None = None, + ) -> RunState | None: + _ = mode, strict_mode_match + calls.append(project_names) + return state if project_names is None else None + + runtime = SimpleNamespace(state_repository=BaseProjectRepository()) + worktree = CreatedPlanWorktree( + name="feature-a-1", + root=Path("/repo/trees/feature-a/1"), + plan_file="feature-a.md", + ) + + section = _runtime_addresses_prompt_section(runtime, worktree=worktree) + + self.assertEqual(calls, [["feature-a-1"], None]) + self.assertIn("Redis (feature-a): redis://localhost:6381", section) + def test_builder_renders_dependency_and_service_addresses_for_matching_worktree(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: worktree_root = Path(tmpdir) / "repo" / "trees" / "feature-a-1" @@ -120,16 +163,25 @@ def test_matching_and_port_helpers_cover_stale_or_partial_state(self) -> None: cwd="/tmp/repo/trees/feature-a-2/apps/api", requested_port=8000, ) + metadata_service = ServiceRecord( + name="Opaque API Process", + type="backend", + cwd="/tmp/outside-project-root", + requested_port=8100, + project="feature-a", + ) stale_service = ServiceRecord(name="feature-b backend", type="backend", cwd="", actual_port=9000) self.assertTrue(_state_project_matches_worktree("feature-a", worktree)) self.assertFalse(_state_project_matches_worktree("feature-b", worktree)) self.assertTrue(_state_service_matches_worktree(matching_service, worktree)) + self.assertTrue(_state_service_matches_worktree(metadata_service, worktree)) self.assertFalse(_state_service_matches_worktree(stale_service, worktree)) self.assertEqual(_component_port({"requested": "0", "resources": {"primary": "49152"}}), 49152) - self.assertEqual(_service_address_lines(RunState(run_id="run-1", mode="trees", services={"api": matching_service})), [ - "Backend (backend): http://localhost:8000" - ]) + self.assertEqual( + _service_address_lines(RunState(run_id="run-1", mode="trees", services={"api": matching_service})), + ["Backend (backend): http://localhost:8000"], + ) def test_runtime_section_uses_legacy_state_loader_when_repository_is_unavailable(self) -> None: state = RunState( diff --git a/tests/python/planning/test_planning_menu_rendering.py b/tests/python/planning/test_planning_menu_rendering.py index 92e98bef9..db79f3de5 100644 --- a/tests/python/planning/test_planning_menu_rendering.py +++ b/tests/python/planning/test_planning_menu_rendering.py @@ -118,9 +118,11 @@ def test_run_flushes_pending_input_on_exit(self) -> None: patch("sys.stdin.fileno", return_value=7), patch("termios.tcgetattr", return_value=[0, 0, 0, 0, 0, 0]), patch("termios.tcsetattr"), + patch("termios.tcflush"), patch("tty.setraw"), patch("os.read", return_value=b"\n"), patch.object(PlanningSelectionMenu, "flush_pending_input") as flush_mock, + patch.object(PlanningSelectionMenu, "_flush_input_buffer") as cleanup_flush_mock, patch.object(PlanningSelectionMenu, "_prompt_toolkit_enabled", return_value=False), redirect_stdout(StringIO()), ): @@ -132,7 +134,52 @@ def test_run_flushes_pending_input_on_exit(self) -> None: self.assertFalse(result.cancelled) self.assertEqual(result.selected_counts, {"a.md": 1}) - self.assertGreaterEqual(flush_mock.call_count, 2) + flush_mock.assert_called_once_with(fd=7) + self.assertEqual(cleanup_flush_mock.call_count, 2) + + def test_run_preserves_result_when_cleanup_drain_select_or_read_fails(self) -> None: + for key, expected_counts, expected_cancelled in ( + (b"\n", {"a.md": 1}, False), + (b"q", {}, True), + ): + for failing_operation in ("select", "read"): + with self.subTest(key=key, operation=failing_operation): + menu = PlanningSelectionMenu() + tcflush_calls = 0 + + def fail_cleanup_flush(_fd: int, _queue: int) -> None: + nonlocal tcflush_calls + tcflush_calls += 1 + if tcflush_calls > 1: + raise termios.error("stale fd") + + error = OSError("unreadable fd") + read_side_effect = [key, error, error] if failing_operation == "read" else None + select_side_effect = error if failing_operation == "select" else None + with ( + patch("sys.stdin.fileno", return_value=7), + patch("termios.tcgetattr", return_value=[0, 0, 0, 0, 0, 0]), + patch("termios.tcsetattr"), + patch("termios.tcflush", side_effect=fail_cleanup_flush), + patch("tty.setraw"), + patch("os.read", return_value=key, side_effect=read_side_effect), + patch( + "select.select", + return_value=([7], [], []), + side_effect=select_side_effect, + ), + patch.object(PlanningSelectionMenu, "_prompt_toolkit_enabled", return_value=False), + redirect_stdout(StringIO()), + ): + result = menu.run( + planning_files=["a.md"], + selected_counts={"a.md": 1}, + existing_counts={"a.md": 0}, + ) + + self.assertEqual(result.cancelled, expected_cancelled) + self.assertEqual(result.selected_counts, expected_counts) + self.assertGreaterEqual(tcflush_calls, 3) def test_run_submit_preserves_zero_counts_when_existing_worktrees_present(self) -> None: menu = PlanningSelectionMenu() diff --git a/tests/python/planning/test_planning_selection.py b/tests/python/planning/test_planning_selection.py index ed77bcb2a..cb3a7d8f3 100644 --- a/tests/python/planning/test_planning_selection.py +++ b/tests/python/planning/test_planning_selection.py @@ -18,7 +18,6 @@ resolve_planning_files, select_projects_for_plan_files, ) -from envctl_engine.planning.worktree_selection_memory import initial_plan_selected_counts def _init_repo_with_commit(repo: Path) -> None: @@ -257,15 +256,5 @@ def test_planning_existing_counts_ignore_stale_branch_names_without_discovered_w self.assertEqual(counts["backend/task.md"], 0) - def test_initial_plan_selected_counts_do_not_resurrect_deleted_worktrees_from_memory(self) -> None: - selected = initial_plan_selected_counts( - planning_files=["backend/task.md", "frontend/task.md"], - existing_counts={"backend/task.md": 0, "frontend/task.md": 2}, - remembered_counts={"backend/task.md": 3, "frontend/task.md": 1}, - ) - - self.assertEqual(selected, {"backend/task.md": 0, "frontend/task.md": 2}) - - if __name__ == "__main__": unittest.main() diff --git a/tests/python/planning/test_planning_worktree_setup_provenance.py b/tests/python/planning/test_planning_worktree_setup_provenance.py index 8fbd5cc96..c10e2e1f3 100644 --- a/tests/python/planning/test_planning_worktree_setup_provenance.py +++ b/tests/python/planning/test_planning_worktree_setup_provenance.py @@ -1,10 +1,62 @@ # ruff: noqa: F403,F405 from __future__ import annotations +from types import SimpleNamespace + +from envctl_engine.planning.worktree_provenance import detect_default_branch from tests.python.planning.planning_worktree_setup_test_support import * class PlanningWorktreeSetupProvenanceTests(PlanningWorktreeSetupTestCase): + def test_detached_default_branch_uses_live_remote_over_stale_tracking_ref(self) -> None: + calls: list[dict[str, object]] = [] + + def fake_run(cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 + args = [str(token) for token in cmd] + calls.append({"args": args, "cwd": cwd, "env": env, "timeout": timeout}) + git_args = args[3:] + if git_args == ["rev-parse", "--verify", "refs/remotes/origin/dev"]: + return subprocess.CompletedProcess(args, 0, stdout="stale-dev\n", stderr="") + if git_args == ["rev-parse", "--verify", "refs/remotes/origin/main"]: + return subprocess.CompletedProcess(args, 0, stdout="main\n", stderr="") + if git_args[:3] == ["ls-remote", "--symref", "origin"]: + return subprocess.CompletedProcess( + args, + 0, + stdout=("ref: refs/heads/main\tHEAD\nmain-sha\tHEAD\nmain-sha\trefs/heads/main\n"), + stderr="", + ) + return subprocess.CompletedProcess(args, 1, stdout="", stderr="missing") + + runtime = SimpleNamespace( + config=SimpleNamespace(base_dir=Path("/repo")), + process_runner=SimpleNamespace(run=fake_run), + _command_env=lambda *, port: {"BASE": "1", "PORT": str(port)}, + ) + + self.assertEqual(detect_default_branch(runtime), "main") + live_probe = next(call for call in calls if call["args"][3] == "ls-remote") + self.assertEqual(live_probe["timeout"], 10.0) + self.assertEqual(live_probe["env"]["GIT_TERMINAL_PROMPT"], "0") + self.assertEqual(live_probe["env"]["BASE"], "1") + + def test_detached_default_branch_falls_back_to_local_tracking_ref_offline(self) -> None: + def fake_run(cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 + _ = cwd, env, timeout + args = [str(token) for token in cmd] + git_args = args[3:] + if git_args == ["rev-parse", "--verify", "refs/remotes/origin/dev"]: + return subprocess.CompletedProcess(args, 0, stdout="local-dev\n", stderr="") + return subprocess.CompletedProcess(args, 1, stdout="", stderr="offline") + + runtime = SimpleNamespace( + config=SimpleNamespace(base_dir=Path("/repo")), + process_runner=SimpleNamespace(run=fake_run), + _command_env=lambda *, port: {"PORT": str(port)}, + ) + + self.assertEqual(detect_default_branch(runtime), "dev") + def test_setup_worktree_creation_writes_provenance_file(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -57,9 +109,7 @@ def fake_run(cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 'project_name: "repo"\n', ) self.assertEqual( - (repo / "trees" / "feature-a" / "1" / ".serena" / "project.local.yml").read_text( - encoding="utf-8" - ), + (repo / "trees" / "feature-a" / "1" / ".serena" / "project.local.yml").read_text(encoding="utf-8"), 'project_name: "repo-feature-a-1"\n', ) self.assertEqual( @@ -233,7 +283,9 @@ def fake_run(cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 if git_cwd == str(unrelated.resolve()) and command[3:] == ["rev-parse", "--show-toplevel"]: return subprocess.CompletedProcess(args=command, returncode=0, stdout=f"{unrelated}\n", stderr="") if git_cwd == str(repo.resolve()) and command[3:] == ["rev-parse", "--git-common-dir"]: - return subprocess.CompletedProcess(args=command, returncode=0, stdout=f"{repo / '.git'}\n", stderr="") + return subprocess.CompletedProcess( + args=command, returncode=0, stdout=f"{repo / '.git'}\n", stderr="" + ) if git_cwd == str(unrelated.resolve()) and command[3:] == ["rev-parse", "--git-common-dir"]: return subprocess.CompletedProcess( args=command, returncode=0, stdout=f"{unrelated / '.git'}\n", stderr="" diff --git a/tests/python/planning/test_planning_worktree_setup_selection.py b/tests/python/planning/test_planning_worktree_setup_selection.py index b1d4785ba..2751b87db 100644 --- a/tests/python/planning/test_planning_worktree_setup_selection.py +++ b/tests/python/planning/test_planning_worktree_setup_selection.py @@ -73,7 +73,6 @@ def test_interactive_plan_dry_run_previews_without_syncing_worktrees_or_memory(s orchestrator = engine.planning_worktree_orchestrator route = parse_route(["--plan", "--dry-run"], env={}) sync_calls: list[dict[str, int]] = [] - save_calls: list[dict[str, int]] = [] def fake_menu( planning_files: list[str], @@ -88,13 +87,9 @@ def fake_sync(**kwargs: object) -> object: sync_calls.append(dict(kwargs["plan_counts"])) # type: ignore[index,arg-type] return engine._sync_plan_worktrees_from_plan_counts(**kwargs) # type: ignore[arg-type] - def fake_save(chosen: dict[str, int]) -> None: - save_calls.append(dict(chosen)) - with ( patch.object(orchestrator, "_run_planning_selection_menu", side_effect=fake_menu), patch.object(orchestrator, "_sync_plan_worktrees_from_plan_counts", side_effect=fake_sync), - patch.object(orchestrator, "_save_plan_selection_memory", side_effect=fake_save), patch.object(sys.stdin, "isatty", return_value=True), patch.object(sys.stdout, "isatty", return_value=True), ): @@ -102,8 +97,8 @@ def fake_save(chosen: dict[str, int]) -> None: self.assertEqual([ctx.name for ctx in selected], ["feature_task-1"]) self.assertEqual(sync_calls, []) - self.assertEqual(save_calls, []) self.assertFalse((repo / "trees" / "feature_task" / "1").exists()) + self.assertEqual(list(runtime.rglob("planning_selection.json")), []) result = orchestrator.last_plan_selection_result() self.assertEqual([worktree.name for worktree in result.created_worktrees], ["feature_task-1"]) diff --git a/tests/python/planning/test_worktree_prompt_selection.py b/tests/python/planning/test_worktree_prompt_selection.py index 7932205c5..4e2992d04 100644 --- a/tests/python/planning/test_worktree_prompt_selection.py +++ b/tests/python/planning/test_worktree_prompt_selection.py @@ -2,13 +2,13 @@ import unittest from pathlib import Path -from typing import Any +from unittest.mock import patch from envctl_engine.planning.worktree_prompt_selection import prompt_planning_selection class WorktreePromptSelectionTests(unittest.TestCase): - def test_prompt_planning_selection_seeds_menu_from_existing_counts_and_memory(self) -> None: + def test_prompt_planning_selection_seeds_menu_from_current_existing_counts(self) -> None: calls: dict[str, object] = {} def choose(**kwargs: object) -> dict[str, int]: @@ -18,51 +18,30 @@ def choose(**kwargs: object) -> dict[str, int]: chosen = prompt_planning_selection( planning_files=["backend/task.md", "frontend/task.md"], raw_projects=[("backend_task-1", Path("/repo/trees/backend_task/1"))], - initial_plan_selected_counts=lambda **kwargs: { - "backend/task.md": kwargs["existing_counts"]["backend/task.md"], - "frontend/task.md": 2, - }, run_planning_selection_menu=choose, - save_plan_selection_memory=self._unexpected, - persist_memory=False, ) self.assertEqual(chosen, {"backend/task.md": 1, "frontend/task.md": 2}) menu_call = calls["menu"] self.assertEqual(menu_call["existing_counts"], {"backend/task.md": 1, "frontend/task.md": 0}) - self.assertEqual(menu_call["selected_counts"], {"backend/task.md": 1, "frontend/task.md": 2}) + self.assertEqual(menu_call["selected_counts"], {"backend/task.md": 1, "frontend/task.md": 0}) - def test_prompt_planning_selection_persists_chosen_counts_when_enabled(self) -> None: - saved: list[dict[str, int]] = [] - - chosen = prompt_planning_selection( - planning_files=["backend/task.md"], - raw_projects=[], - persist_memory=True, - initial_plan_selected_counts=lambda **_kwargs: {"backend/task.md": 1}, - run_planning_selection_menu=lambda **_kwargs: {"backend/task.md": 3}, - save_plan_selection_memory=saved.append, - ) - - self.assertEqual(chosen, {"backend/task.md": 3}) - self.assertEqual(saved, [{"backend/task.md": 3}]) - - def test_prompt_planning_selection_skips_memory_for_cancel_empty_or_dry_run(self) -> None: - for persist_memory, menu_result in [(True, None), (True, {}), (False, {"backend/task.md": 1})]: - with self.subTest(persist_memory=persist_memory, menu_result=menu_result): - saved: list[dict[str, int]] = [] - - chosen = prompt_planning_selection( - planning_files=["backend/task.md"], - raw_projects=[], - persist_memory=persist_memory, - initial_plan_selected_counts=lambda **_kwargs: {"backend/task.md": 1}, - run_planning_selection_menu=lambda **_kwargs: menu_result, - save_plan_selection_memory=saved.append, - ) - - self.assertEqual(chosen, menu_result) - self.assertEqual(saved, []) + def test_prompt_planning_selection_clamps_invalid_existing_counts(self) -> None: + calls: dict[str, object] = {} - def _unexpected(self, *_args: Any, **_kwargs: Any) -> Any: - raise AssertionError("callback should not be called") + def choose(**kwargs: object) -> None: + calls.update(kwargs) + return None + + with patch( + "envctl_engine.planning.worktree_prompt_selection.planning_existing_counts", + return_value={"backend/task.md": -2, "frontend/task.md": 3}, + ): + chosen = prompt_planning_selection( + planning_files=["backend/task.md", "frontend/task.md"], + raw_projects=[], + run_planning_selection_menu=choose, + ) + + self.assertIsNone(chosen) + self.assertEqual(calls["selected_counts"], {"backend/task.md": 0, "frontend/task.md": 3}) diff --git a/tests/python/planning/test_worktree_selection_memory.py b/tests/python/planning/test_worktree_selection_memory.py deleted file mode 100644 index 05ed4affb..000000000 --- a/tests/python/planning/test_worktree_selection_memory.py +++ /dev/null @@ -1,105 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -import tempfile -import unittest - -from envctl_engine.planning.worktree_selection_memory import ( - initial_plan_selected_counts, - load_plan_selection_memory, - plan_selection_memory_path, - save_plan_selection_memory, -) - - -class WorktreeSelectionMemoryTests(unittest.TestCase): - def test_plan_selection_memory_path_uses_runtime_root(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - runtime_root = Path(tmpdir) / "runtime" - - self.assertEqual( - plan_selection_memory_path(runtime_root=runtime_root), - runtime_root / "planning_selection.json", - ) - - def test_load_plan_selection_memory_prefers_current_file_and_filters_values(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - runtime_root = root / "runtime" - legacy_root = root / "legacy" - runtime_root.mkdir() - legacy_root.mkdir() - (runtime_root / "planning_selection.json").write_text( - json.dumps( - { - "selected_counts": { - "feature/a.md": "2", - "feature/b.md": 0, - "feature/c.md": -1, - "feature/d.md": "invalid", - } - } - ), - encoding="utf-8", - ) - (legacy_root / "planning_selection.json").write_text( - json.dumps({"selected_counts": {"feature/a.md": 9}}), - encoding="utf-8", - ) - - self.assertEqual( - load_plan_selection_memory(runtime_root=runtime_root, runtime_legacy_root=legacy_root), - {"feature/a.md": 2, "feature/b.md": 0}, - ) - - def test_load_plan_selection_memory_falls_back_to_legacy_file(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - runtime_root = root / "runtime" - legacy_root = root / "legacy" - runtime_root.mkdir() - legacy_root.mkdir() - (runtime_root / "planning_selection.json").write_text("not json", encoding="utf-8") - (legacy_root / "planning_selection.json").write_text( - json.dumps({"selected_counts": {"feature/a.md": "3"}}), - encoding="utf-8", - ) - - self.assertEqual( - load_plan_selection_memory(runtime_root=runtime_root, runtime_legacy_root=legacy_root), - {"feature/a.md": 3}, - ) - - def test_save_plan_selection_memory_writes_sorted_positive_counts_to_current_and_legacy(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - runtime_root = root / "runtime" - legacy_root = root / "legacy" - runtime_root.mkdir() - - save_plan_selection_memory( - runtime_root=runtime_root, - runtime_legacy_root=legacy_root, - selected_counts={"z.md": 2, "a.md": 1, "zero.md": 0}, - ) - - current_payload = json.loads((runtime_root / "planning_selection.json").read_text(encoding="utf-8")) - legacy_payload = json.loads((legacy_root / "planning_selection.json").read_text(encoding="utf-8")) - self.assertEqual(current_payload, legacy_payload) - self.assertEqual(list(current_payload["selected_counts"].items()), [("a.md", 1), ("z.md", 2)]) - self.assertIsInstance(current_payload["saved_at"], str) - - def test_initial_plan_selected_counts_uses_current_existing_counts_only(self) -> None: - self.assertEqual( - initial_plan_selected_counts( - planning_files=["feature/a.md", "feature/b.md", "feature/c.md"], - existing_counts={"feature/a.md": 2}, - remembered_counts={"feature/a.md": 9, "feature/b.md": 4, "feature/c.md": -1}, - ), - {"feature/a.md": 2, "feature/b.md": 0, "feature/c.md": 0}, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/python/runtime/engine_runtime_real_startup_test_support.py b/tests/python/runtime/engine_runtime_real_startup_test_support.py index 7f46ce704..7da958593 100644 --- a/tests/python/runtime/engine_runtime_real_startup_test_support.py +++ b/tests/python/runtime/engine_runtime_real_startup_test_support.py @@ -102,6 +102,7 @@ def __init__(self) -> None: self._next_pid = 4000 self.auto_track_listener_ports = False self._pid_listener_ports: dict[int, int] = {} + self._terminated_pids: set[int] = set() self.wait_for_port_result = False self.wait_for_port_calls: list[tuple[int, float]] = [] self.wait_for_pid_port_result = False @@ -199,6 +200,7 @@ def start_background(self, cmd, *, cwd=None, env=None, stdout_path=None, stderr_ _ = stderr_path self._next_pid += 1 pid = self._next_pid + self._terminated_pids.discard(pid) if parsed_port is not None and parsed_port in self.fail_start_ports: if stdout_path: @@ -269,6 +271,8 @@ def find_pid_listener_port( return self.find_pid_listener_port_result def is_pid_running(self, _pid: int) -> bool: + if _pid in self._terminated_pids: + return False if self.auto_track_listener_ports and _pid in self._pid_listener_ports: return True return self.pid_running_result @@ -281,6 +285,7 @@ def pid_owns_port(self, _pid: int, _port: int) -> bool: def terminate(self, _pid: int, *, term_timeout: float = 2.0, kill_timeout: float = 1.0) -> bool: _ = term_timeout, kill_timeout + self._terminated_pids.add(_pid) if self.auto_track_listener_ports: self._pid_listener_ports.pop(_pid, None) return True diff --git a/tests/python/runtime/lifecycle_parity_test_support.py b/tests/python/runtime/lifecycle_parity_test_support.py index 6b1c15376..fc53df8da 100644 --- a/tests/python/runtime/lifecycle_parity_test_support.py +++ b/tests/python/runtime/lifecycle_parity_test_support.py @@ -33,7 +33,6 @@ dump_state = state_module.dump_state - class _NoopPlanner: def __init__(self) -> None: self.released = False @@ -46,6 +45,7 @@ def release(self, port: int, owner: str | None = None) -> None: _ = owner self.released_ports.append(port) + class _TrackingPlanner: def __init__(self) -> None: self.released_session = False @@ -57,6 +57,7 @@ def release_session(self) -> None: def release_all(self) -> None: self.released_all = True + class _TrackingRunner: def __init__(self) -> None: self.terminated: list[int] = [] @@ -72,7 +73,7 @@ def terminate(self, pid: int, *, term_timeout: float = 2.0, kill_timeout: float return True def is_pid_running(self, pid: int) -> bool: - return pid > 0 + return pid > 0 and pid not in self.terminated def pid_owns_port(self, pid: int, port: int) -> bool: _ = pid, port @@ -94,18 +95,21 @@ def run(self, cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 return type("Result", (), {"returncode": 0, "stdout": stdout, "stderr": ""})() return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + class _OwnershipDenyRunner(_TrackingRunner): def pid_owns_port(self, pid: int, port: int) -> bool: _ = pid, port return False + class _ResumeRestoreRunner: - def __init__(self) -> None: + def __init__(self, *, listeners_ready: bool = False) -> None: self.run_calls: list[tuple[str, ...]] = [] self.start_calls: list[tuple[str, ...]] = [] self.terminated: list[int] = [] self._next_pid = 42000 self._running: set[int] = set() + self.listeners_ready = listeners_ready self.fail_alembic = False def run(self, cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 @@ -164,7 +168,7 @@ def wait_for_pid_port( debug_pid_wait_group: str = "", ) -> bool: _ = host, timeout, debug_pid_wait_group - return False + return self.listeners_ready and self.pid_owns_port(_pid, _port) def pid_owns_port(self, pid: int, _port: int) -> bool: return pid in self._running @@ -175,6 +179,7 @@ def terminate(self, pid: int, *, term_timeout: float = 2.0, kill_timeout: float self._running.discard(pid) return True + __all__ = [ "Path", "PYTHON_ROOT", diff --git a/tests/python/runtime/test_command_flag_storage.py b/tests/python/runtime/test_command_flag_storage.py index 9c05e94a4..d4d93398d 100644 --- a/tests/python/runtime/test_command_flag_storage.py +++ b/tests/python/runtime/test_command_flag_storage.py @@ -19,6 +19,7 @@ def test_parse_projects_splits_comma_separated_targets(self) -> None: def test_boolean_flag_name_maps_known_runtime_flags(self) -> None: self.assertEqual(boolean_flag_name("--headless"), "batch") self.assertEqual(boolean_flag_name("--cmux"), "cmux") + self.assertEqual(boolean_flag_name("--ignore-service-deps"), "ignore_service_deps") def test_store_value_flag_extends_repeatable_values(self) -> None: flags: dict[str, object] = {"services": ["backend"]} diff --git a/tests/python/runtime/test_docker_service_runtime.py b/tests/python/runtime/test_docker_service_runtime.py new file mode 100644 index 000000000..22dd0ddd6 --- /dev/null +++ b/tests/python/runtime/test_docker_service_runtime.py @@ -0,0 +1,1322 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path +from types import SimpleNamespace + +from envctl_engine.runtime.docker_service_runtime import ( + DockerServiceLaunchCleanupError, + DockerServiceRuntime, + docker_service_container_name, + refresh_container_log_snapshots, + stop_container_log_followers, +) +from envctl_engine.runtime.lifecycle_service_termination import terminate_service_record +from envctl_engine.runtime.service_status_truth import service_truth_status +from envctl_engine.state.models import ServiceRecord + + +class _Runner: + def __init__(self) -> None: + self.calls: list[list[str]] = [] + self.env_file_payload = "" + self.running = True + self.port_ready = True + self.terminated: list[int] = [] + self.build_env: dict[str, str] = {} + self.existing_container: tuple[bool, str] | None = None + self.launched_container: dict[str, str] | None = None + self.local_images: set[str] = set() + self.remote_exposed_ports: dict[str, dict[str, object]] = {} + + def run(self, command, **_kwargs): + argv = [str(part) for part in command] + self.calls.append(argv) + if argv[:2] == ["docker", "build"]: + self.build_env = dict(_kwargs.get("env", {})) + self.local_images.add(argv[argv.index("--tag") + 1]) + if argv[:2] == ["docker", "run"]: + env_file = Path(argv[argv.index("--env-file") + 1]) + self.env_file_payload = env_file.read_text(encoding="utf-8") + labels = [argv[index + 1] for index, value in enumerate(argv) if value == "--label"] + self.launched_container = { + "id": "d" * 64, + "name": argv[argv.index("--name") + 1], + "scope": next(value.split("=", 1)[1] for value in labels if value.startswith("io.envctl.runtime-scope=")), + "token": next(value.split("=", 1)[1] for value in labels if value.startswith("io.envctl.launch-token=")), + } + return SimpleNamespace(returncode=0, stdout=f"{'d' * 64}\n", stderr="") + if argv[:3] == ["docker", "container", "inspect"]: + reference = argv[-1] + launched = self.launched_container + if launched is not None and reference in {launched["id"], launched["name"]}: + format_value = argv[argv.index("--format") + 1] + if ".State.Running" in format_value: + return SimpleNamespace( + returncode=0, + stdout=f"{launched['id']}|true|{launched['scope']}\n", + stderr="", + ) + label_value = launched["token"] if "io.envctl.launch-token" in format_value else launched["scope"] + return SimpleNamespace( + returncode=0, + stdout=f"{launched['id']}|{label_value}\n", + stderr="", + ) + if self.existing_container is None: + return SimpleNamespace(returncode=1, stdout="", stderr="No such container") + running, owner = self.existing_container + format_value = argv[argv.index("--format") + 1] + if ".State.Running" not in format_value: + return SimpleNamespace(returncode=0, stdout=f"{'c' * 64}|{owner}\n", stderr="") + return SimpleNamespace( + returncode=0, + stdout=f"{'c' * 64}|{str(running).lower()}|{owner}\n", + stderr="", + ) + if argv[:3] == ["docker", "container", "ls"]: + launched = self.launched_container + token_filter = argv[argv.index("--filter") + 1] + expected_token = token_filter.rpartition("=")[2] + stdout = ( + f"{launched['id']}\n" + if launched is not None and launched["token"] == expected_token + else "" + ) + return SimpleNamespace(returncode=0, stdout=stdout, stderr="") + if argv[:3] == ["docker", "inspect", "--format"]: + return SimpleNamespace(returncode=0 if self.running else 1, stdout="true\n" if self.running else "", stderr="") + if argv[:3] == ["docker", "image", "inspect"]: + image = argv[-1] + if image not in self.local_images: + return SimpleNamespace(returncode=1, stdout="", stderr="missing") + if "--format" in argv: + payload = self.remote_exposed_ports.get(image, {}) + return SimpleNamespace(returncode=0, stdout=json.dumps(payload) + "\n", stderr="") + return SimpleNamespace(returncode=0, stdout="[]\n", stderr="") + if argv[:2] == ["docker", "pull"]: + self.local_images.add(argv[-1]) + return SimpleNamespace(returncode=0, stdout="pulled\n", stderr="") + if argv[:3] == ["docker", "rm", "--force"]: + if self.launched_container is not None and argv[-1] in { + self.launched_container["id"], + self.launched_container["name"], + }: + self.launched_container = None + return SimpleNamespace(returncode=0, stdout="", stderr="") + if argv[:2] == ["docker", "logs"]: + return SimpleNamespace(returncode=0, stdout="container log\n", stderr="") + return SimpleNamespace(returncode=0, stdout="ok\n", stderr="") + + def start_background(self, command, **_kwargs): + self.calls.append([str(part) for part in command]) + return SimpleNamespace(pid=4321) + + def wait_for_port(self, _port, **_kwargs): + return self.port_ready + + def terminate_process_group(self, pid, **_kwargs): + self.terminated.append(pid) + return True + + +def _runtime(tmp_path: Path, runner: _Runner, *, env: dict[str, str] | None = None): + events: list[tuple[str, dict[str, object]]] = [] + runtime = SimpleNamespace( + env={"DOCKER_MODE": "true", **(env or {})}, + config=SimpleNamespace(raw={}, runtime_scope_dir=tmp_path / "runtime-scope"), + process_runner=runner, + _command_exists=lambda command: command == "docker", + _run_dir_path=lambda run_id: tmp_path / "runs" / str(run_id), + _emit=lambda event, **payload: events.append((event, payload)), + _service_truth_timeout=lambda: 0.01, + ) + return runtime, events + + +def test_container_names_preserve_root_identity_for_long_project_names(tmp_path: Path) -> None: + first = docker_service_container_name( + project_name="feature-" + ("very-long-name-" * 8), + project_root=tmp_path / "one", + service_name="backend", + ) + second = docker_service_container_name( + project_name="feature-" + ("very-long-name-" * 8), + project_root=tmp_path / "two", + service_name="backend", + ) + + assert len(first) <= 63 + assert len(second) <= 63 + assert first != second + + +def test_builds_with_docker_layer_cache_and_runs_managed_container(tmp_path: Path) -> None: + runner = _Runner() + runtime, events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_COMMAND_MODE": "service"}) + service_root = tmp_path / "backend" + service_root.mkdir() + (service_root / "Dockerfile").write_text("FROM python:3.12-slim\nWORKDIR /app\n", encoding="utf-8") + + launch = DockerServiceRuntime(runtime, runner).launch( + project_name="Feature One", + project_root=tmp_path, + service_name="backend", + cwd=service_root, + command=["python", "-m", "uvicorn", "app:api", "--host", "127.0.0.1", "--port", "8123"], + env={ + "PATH": "/host/bin", + "DATABASE_URL": "postgres://u:p@127.0.0.1:5432/db", + "PORT": "8123", + "VITE_API_URL": "http://localhost:8123/api/v1", + }, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + + build_index = next(index for index, call in enumerate(runner.calls) if call[:2] == ["docker", "build"]) + run_index = next(index for index, call in enumerate(runner.calls) if call[:2] == ["docker", "run"]) + assert build_index < run_index + docker_build = runner.calls[build_index] + assert docker_build[docker_build.index("--build-arg") + 1] == "VITE_API_URL" + assert "http://localhost:8123/api/v1" not in docker_build + assert runner.build_env["VITE_API_URL"] == "http://localhost:8123/api/v1" + docker_run = runner.calls[run_index] + publish_index = docker_run.index("--publish") + assert docker_run[publish_index : publish_index + 2] == ["--publish", "127.0.0.1:8123:8123"] + assert "0.0.0.0" in docker_run + runtime_label_index = docker_run.index("io.envctl.service=backend") + 2 + assert docker_run[runtime_label_index].startswith("io.envctl.runtime-scope=") + assert "PATH=" not in runner.env_file_payload + assert "DATABASE_URL=postgres://u:p@host.docker.internal:5432/db" in runner.env_file_payload + assert launch.container_id == "d" * 64 + assert any(event == "service.container.build" and payload["cache"] == "docker_layer_cache" for event, payload in events) + + +def test_docker_env_files_are_unique_private_and_independently_scoped(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner) + docker_runtime = DockerServiceRuntime(runtime, runner) + + first = docker_runtime._write_env_file(project_name="Main", service_name="backend", values={"SECRET": "one"}) + second = docker_runtime._write_env_file(project_name="Main", service_name="backend", values={"SECRET": "two"}) + try: + assert first != second + assert first.read_text(encoding="utf-8") == "SECRET=one\n" + assert second.read_text(encoding="utf-8") == "SECRET=two\n" + assert first.stat().st_mode & 0o777 == 0o600 + assert second.stat().st_mode & 0o777 == 0o600 + finally: + assert docker_runtime._scrub_env_file(first) is None + assert docker_runtime._scrub_env_file(second) is None + + +def test_stale_env_file_from_dead_process_is_scrubbed_before_new_write(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner) + docker_runtime = DockerServiceRuntime(runtime, runner) + run_dir = tmp_path / "runs" / "docker" + run_dir.mkdir(parents=True) + stale = run_dir / ".docker-env-aaaaaaaaaaaa-p99999999-stale123" + stale.write_text("SECRET=stale\n", encoding="utf-8") + monkeypatch.setattr(DockerServiceRuntime, "_pid_is_running", staticmethod(lambda _pid: False)) + + current = docker_runtime._write_env_file( + project_name="Main", + service_name="backend", + values={"SECRET": "current"}, + ) + try: + assert not stale.exists() + assert current.read_text(encoding="utf-8") == "SECRET=current\n" + finally: + assert docker_runtime._scrub_env_file(current) is None + + +def test_old_env_file_from_live_process_is_never_scrubbed(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_DOCKER_ENV_STALE_SECONDS": "0"}) + docker_runtime = DockerServiceRuntime(runtime, runner) + run_dir = tmp_path / "runs" / "docker" + run_dir.mkdir(parents=True) + active = run_dir / f".docker-env-aaaaaaaaaaaa-p{os.getpid()}-active123" + active.write_text("SECRET=active\n", encoding="utf-8") + os.utime(active, (1, 1)) + + current = docker_runtime._write_env_file( + project_name="Main", + service_name="backend", + values={"SECRET": "current"}, + ) + try: + assert active.read_text(encoding="utf-8") == "SECRET=active\n" + finally: + assert docker_runtime._scrub_env_file(active) is None + assert docker_runtime._scrub_env_file(current) is None + + +def test_old_legacy_env_file_is_scrubbed_after_grace_period(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner) + docker_runtime = DockerServiceRuntime(runtime, runner) + run_dir = tmp_path / "runs" / "docker" + run_dir.mkdir(parents=True) + stale = run_dir / ".docker-env-legacy" + stale.write_text("SECRET=legacy\n", encoding="utf-8") + os.utime(stale, (1, 1)) + + current = docker_runtime._write_env_file( + project_name="Main", + service_name="backend", + values={"SECRET": "current"}, + ) + try: + assert not stale.exists() + finally: + assert docker_runtime._scrub_env_file(current) is None + + +def test_unscrubbable_stale_env_file_blocks_new_secret_write(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner) + docker_runtime = DockerServiceRuntime(runtime, runner) + run_dir = tmp_path / "runs" / "docker" + run_dir.mkdir(parents=True) + stale = run_dir / ".docker-env-aaaaaaaaaaaa-p99999999-stale123" + stale.write_text("SECRET=stale\n", encoding="utf-8") + original_unlink = Path.unlink + original_open = os.open + + def fail_env_unlink(path: Path, *args, **kwargs): # noqa: ANN002, ANN003, ANN202 + if path == stale: + raise OSError("unlink unavailable") + return original_unlink(path, *args, **kwargs) + + def fail_env_truncate(path, flags, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + if Path(path) == stale and flags & os.O_TRUNC: + raise OSError("truncate unavailable") + return original_open(path, flags, *args, **kwargs) + + with monkeypatch.context() as scoped: + scoped.setattr(DockerServiceRuntime, "_pid_is_running", staticmethod(lambda _pid: False)) + scoped.setattr(Path, "unlink", fail_env_unlink) + scoped.setattr(os, "open", fail_env_truncate) + try: + docker_runtime._write_env_file( + project_name="Main", + service_name="backend", + values={"SECRET": "current"}, + ) + except RuntimeError as exc: + assert "stale docker env file cleanup failed" in str(exc).lower() + else: + raise AssertionError("an unscrubbable stale secret must block a new write") + + assert list(run_dir.glob(".docker-env-*")) == [stale] + assert docker_runtime._scrub_env_file(stale) is None + assert list(run_dir.glob(".docker-env-*")) == [] + + +def test_partial_env_write_failure_cannot_mask_unscrubbable_secret_file(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + runner = _Runner() + runtime, events = _runtime(tmp_path, runner) + docker_runtime = DockerServiceRuntime(runtime, runner) + original_unlink = Path.unlink + original_open = os.open + original_fdopen = os.fdopen + + class PartialWriteFailure: + def __init__(self, handle) -> None: # noqa: ANN001 + self.handle = handle + + def __enter__(self): # noqa: ANN204 + return self + + def write(self, value: str) -> None: + self.handle.write(value[:10]) + self.handle.flush() + raise OSError("partial write failed") + + def __exit__(self, *_args) -> None: # noqa: ANN002 + self.handle.close() + + def partial_fdopen(*args, **kwargs): # noqa: ANN002, ANN003, ANN202 + return PartialWriteFailure(original_fdopen(*args, **kwargs)) + + def fail_env_unlink(path: Path, *args, **kwargs): # noqa: ANN002, ANN003, ANN202 + if path.name.startswith(".docker-env-"): + raise OSError("unlink unavailable") + return original_unlink(path, *args, **kwargs) + + def fail_env_truncate(path, flags, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + if Path(path).name.startswith(".docker-env-") and flags & os.O_TRUNC: + raise OSError("truncate unavailable") + return original_open(path, flags, *args, **kwargs) + + with monkeypatch.context() as scoped: + scoped.setattr(os, "fdopen", partial_fdopen) + scoped.setattr(Path, "unlink", fail_env_unlink) + scoped.setattr(os, "open", fail_env_truncate) + try: + docker_runtime._write_env_file( + project_name="Main", + service_name="backend", + values={"SECRET": "do-not-leak"}, + ) + except RuntimeError as exc: + detail = str(exc).lower() + assert "partial write failed" in detail + assert "secret scrub failed" in detail + else: + raise AssertionError("the partial write must report the secret cleanup failure") + + env_files = list((tmp_path / "runs" / "docker").glob(".docker-env-*")) + assert len(env_files) == 1 + assert env_files[0].read_text(encoding="utf-8") + assert any(event == "service.container.env_file_cleanup_warning" for event, _payload in events) + assert docker_runtime._scrub_env_file(env_files[0]) is None + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_interrupt_during_env_write_scrubs_partial_secret_before_reraising(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner) + docker_runtime = DockerServiceRuntime(runtime, runner) + original_fdopen = os.fdopen + + class InterruptedWrite: + def __init__(self, handle) -> None: # noqa: ANN001 + self.handle = handle + + def __enter__(self): # noqa: ANN204 + return self + + def write(self, value: str) -> None: + self.handle.write(value) + self.handle.flush() + raise KeyboardInterrupt + + def __exit__(self, *_args) -> None: # noqa: ANN002 + self.handle.close() + + monkeypatch.setattr( + os, + "fdopen", + lambda *args, **kwargs: InterruptedWrite(original_fdopen(*args, **kwargs)), + ) + + try: + docker_runtime._write_env_file( + project_name="Main", + service_name="backend", + values={"SECRET": "must-not-remain"}, + ) + except KeyboardInterrupt: + pass + else: + raise AssertionError("the write interrupt must be restored after secret cleanup") + + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_successful_docker_run_with_empty_stdout_uses_owned_container_name(tmp_path: Path) -> None: + class EmptyOutputRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + result = super().run(command, **kwargs) + if [str(part) for part in command][:2] == ["docker", "run"]: + return SimpleNamespace(returncode=0, stdout="", stderr="") + return result + + runner = EmptyOutputRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + launch = DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + + assert launch.container_id == "d" * 64 + assert launch.launch_token + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + + +def test_successful_docker_run_with_malformed_stdout_resolves_owned_immutable_id(tmp_path: Path) -> None: + class DiagnosticOutputRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + result = super().run(command, **kwargs) + if [str(part) for part in command][:2] == ["docker", "run"]: + return SimpleNamespace(returncode=0, stdout="unexpected diagnostic line\n", stderr="") + return result + + runner = DiagnosticOutputRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + launch = DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + + assert launch.container_id == "d" * 64 + assert launch.container_id != "unexpected diagnostic line" + + +def test_env_file_unlink_failure_scrubs_secrets_without_masking_success(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + runner = _Runner() + runtime, events = _runtime( + tmp_path, + runner, + env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}, + ) + original_unlink = Path.unlink + + def fail_env_unlink(path: Path, *args, **kwargs): # noqa: ANN002, ANN003, ANN202 + if path.name.startswith(".docker-env-"): + raise OSError("unlink unavailable") + return original_unlink(path, *args, **kwargs) + + monkeypatch.setattr(Path, "unlink", fail_env_unlink) + + launch = DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={"SECRET": "do-not-leak"}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + + env_files = list((tmp_path / "runs" / "docker").glob(".docker-env-*")) + assert launch.container_id == "d" * 64 + assert len(env_files) == 1 + assert env_files[0].read_text(encoding="utf-8") == "" + assert any(event == "service.container.env_file_cleanup_warning" for event, _payload in events) + + +def test_unscrubbable_env_file_fails_launch_and_removes_owned_container(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + runner = _Runner() + runtime, events = _runtime( + tmp_path, + runner, + env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}, + ) + docker_runtime = DockerServiceRuntime(runtime, runner) + original_unlink = Path.unlink + original_open = os.open + + def fail_env_unlink(path: Path, *args, **kwargs): # noqa: ANN002, ANN003, ANN202 + if path.name.startswith(".docker-env-"): + raise OSError("unlink unavailable") + return original_unlink(path, *args, **kwargs) + + def fail_env_truncate(path, flags, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + if Path(path).name.startswith(".docker-env-") and flags & os.O_TRUNC: + raise OSError("truncate unavailable") + return original_open(path, flags, *args, **kwargs) + + with monkeypatch.context() as scoped: + scoped.setattr(Path, "unlink", fail_env_unlink) + scoped.setattr(os, "open", fail_env_truncate) + try: + docker_runtime.launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={"SECRET": "do-not-leak"}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except RuntimeError as exc: + assert "env file cleanup failed" in str(exc).lower() + else: + raise AssertionError("an unscrubbable secret file must fail the launch") + + env_files = list((tmp_path / "runs" / "docker").glob(".docker-env-*")) + assert runner.launched_container is None + assert ["docker", "rm", "--force", "d" * 64] in runner.calls + assert any(event == "service.container.env_file_cleanup_warning" for event, _payload in events) + assert len(env_files) == 1 + assert docker_runtime._scrub_env_file(env_files[0]) is None + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_pre_run_command_error_cannot_mask_unscrubbable_secret_file(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + runner = _Runner() + runtime, events = _runtime( + tmp_path, + runner, + env={ + "ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev", + "ENVCTL_BACKEND_DOCKER_COMMAND_MODE": "invalid", + }, + ) + docker_runtime = DockerServiceRuntime(runtime, runner) + original_unlink = Path.unlink + original_open = os.open + + def fail_env_unlink(path: Path, *args, **kwargs): # noqa: ANN002, ANN003, ANN202 + if path.name.startswith(".docker-env-"): + raise OSError("unlink unavailable") + return original_unlink(path, *args, **kwargs) + + def fail_env_truncate(path, flags, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + if Path(path).name.startswith(".docker-env-") and flags & os.O_TRUNC: + raise OSError("truncate unavailable") + return original_open(path, flags, *args, **kwargs) + + with monkeypatch.context() as scoped: + scoped.setattr(Path, "unlink", fail_env_unlink) + scoped.setattr(os, "open", fail_env_truncate) + try: + docker_runtime.launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=["python", "app.py"], + env={"SECRET": "do-not-leak"}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except RuntimeError as exc: + detail = str(exc).lower() + assert "invalid docker command mode" in detail + assert "env file cleanup failed" in detail + assert "secret scrub failed" in detail + else: + raise AssertionError("the pre-run error must report the secret cleanup failure") + + env_files = list((tmp_path / "runs" / "docker").glob(".docker-env-*")) + assert not any(call[:2] == ["docker", "run"] for call in runner.calls) + assert any(event == "service.container.env_file_cleanup_warning" for event, _payload in events) + assert len(env_files) == 1 + assert docker_runtime._scrub_env_file(env_files[0]) is None + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_tokenized_stop_treats_already_absent_container_as_confirmed(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner) + + stopped = DockerServiceRuntime(runtime, runner).stop( + "envctl-app-main-backend", + expected_launch_token="launch-token", + ) + + assert stopped + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + + +def test_tokenized_stop_removes_inspected_immutable_id_not_reused_name(tmp_path: Path) -> None: + container_id = "a" * 64 + + class NameReuseRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:3] == ["docker", "container", "inspect"]: + self.calls.append(argv) + return SimpleNamespace( + returncode=0, + stdout=f"{container_id}|launch-token\n", + stderr="", + ) + return super().run(command, **kwargs) + + runner = NameReuseRunner() + runtime, _events = _runtime(tmp_path, runner) + + stopped = DockerServiceRuntime(runtime, runner).stop( + "envctl-app-main-backend", + expected_launch_token="launch-token", + ) + + assert stopped + remove = next(call for call in runner.calls if call[:3] == ["docker", "rm", "--force"]) + assert remove == ["docker", "rm", "--force", container_id] + assert "envctl-app-main-backend" not in remove + + +def test_tokenized_stop_refuses_foreign_launch_token_without_deleting(tmp_path: Path) -> None: + class ForeignTokenRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:3] == ["docker", "container", "inspect"]: + self.calls.append(argv) + return SimpleNamespace( + returncode=0, + stdout=f"{'b' * 64}|foreign-token\n", + stderr="", + ) + return super().run(command, **kwargs) + + runner = ForeignTokenRunner() + runtime, _events = _runtime(tmp_path, runner) + + stopped = DockerServiceRuntime(runtime, runner).stop( + "envctl-app-main-backend", + expected_launch_token="expected-token", + ) + + assert not stopped + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + + +def test_tokenized_stop_rejects_short_container_id_without_deleting(tmp_path: Path) -> None: + class ShortIdentityRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:3] == ["docker", "container", "inspect"]: + self.calls.append(argv) + return SimpleNamespace(returncode=0, stdout="abcdef123456|launch-token\n", stderr="") + return super().run(command, **kwargs) + + runner = ShortIdentityRunner() + runtime, _events = _runtime(tmp_path, runner) + + stopped = DockerServiceRuntime(runtime, runner).stop( + "envctl-app-main-backend", + expected_launch_token="launch-token", + ) + + assert not stopped + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + + +def test_launch_transport_and_ownership_probe_failure_preserves_durable_identity(tmp_path: Path) -> None: + class InspectFailureRunner(_Runner): + def __init__(self) -> None: + super().__init__() + self.container_inspects = 0 + + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:3] == ["docker", "container", "inspect"]: + self.container_inspects += 1 + if self.container_inspects > 1: + raise OSError("inspect transport failed") + if argv[:2] == ["docker", "run"]: + env_file = Path(argv[argv.index("--env-file") + 1]) + self.env_file_payload = env_file.read_text(encoding="utf-8") + raise OSError("docker run transport failed") + return super().run(command, **kwargs) + + runner = InspectFailureRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + try: + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={"SECRET": "scrub-me"}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except DockerServiceLaunchCleanupError as exc: + assert "inspect transport failed" not in str(exc) + assert "cleanup could not confirm" in str(exc).lower() + assert exc.launch.container_id == exc.launch.container_name + assert exc.launch.container_name.startswith("envctl-app-main-backend") + assert exc.launch.launch_token + else: + raise AssertionError("unconfirmed launch cleanup must preserve durable identity") + + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_timeout_accepted_container_is_found_by_token_after_delayed_visibility(tmp_path: Path) -> None: + class DelayedVisibilityRunner(_Runner): + def __init__(self) -> None: + super().__init__() + self.pending_container: dict[str, str] | None = None + self.token_queries = 0 + + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:2] == ["docker", "run"]: + self.calls.append(argv) + env_file = Path(argv[argv.index("--env-file") + 1]) + self.env_file_payload = env_file.read_text(encoding="utf-8") + labels = [argv[index + 1] for index, value in enumerate(argv) if value == "--label"] + self.pending_container = { + "id": "f" * 64, + "name": argv[argv.index("--name") + 1], + "scope": next( + value.split("=", 1)[1] + for value in labels + if value.startswith("io.envctl.runtime-scope=") + ), + "token": next( + value.split("=", 1)[1] + for value in labels + if value.startswith("io.envctl.launch-token=") + ), + } + return SimpleNamespace(returncode=124, stdout="", stderr="Command timed out") + if argv[:3] == ["docker", "container", "ls"]: + self.token_queries += 1 + if self.token_queries == 2: + self.launched_container = self.pending_container + return super().run(command, **kwargs) + return super().run(command, **kwargs) + + runner = DelayedVisibilityRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + try: + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={"SECRET": "scrub-me"}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except RuntimeError as exc: + assert "timed out" in str(exc).lower() + assert not isinstance(exc, DockerServiceLaunchCleanupError) + else: + raise AssertionError("a timed-out Docker launch must fail after cleanup") + + assert runner.token_queries >= 2 + assert runner.launched_container is None + assert ["docker", "rm", "--force", "f" * 64] in runner.calls + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_timeout_without_visible_container_preserves_pending_cleanup_authority(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + class TimeoutRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:2] == ["docker", "run"]: + self.calls.append(argv) + env_file = Path(argv[argv.index("--env-file") + 1]) + self.env_file_payload = env_file.read_text(encoding="utf-8") + return SimpleNamespace(returncode=124, stdout="", stderr="Command timed out") + return super().run(command, **kwargs) + + clock = [0.0] + monkeypatch.setattr( + "envctl_engine.runtime.docker_service_runtime.time.monotonic", + lambda: clock[0], + ) + monkeypatch.setattr( + "envctl_engine.runtime.docker_service_runtime.time.sleep", + lambda seconds: clock.__setitem__(0, clock[0] + seconds), + ) + runner = TimeoutRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + docker_runtime = DockerServiceRuntime(runtime, runner) + + try: + docker_runtime.launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={"SECRET": "scrub-me"}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except DockerServiceLaunchCleanupError as exc: + pending = exc.launch + else: + raise AssertionError("an uncertain timed-out launch must retain cleanup authority") + + assert pending.cleanup_pending_since is not None + assert pending.container_id == pending.container_name + assert not docker_runtime.stop( + pending.container_id, + expected_launch_token=pending.launch_token, + pending_cleanup_since=pending.cleanup_pending_since, + ) + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_interrupt_after_daemon_accepts_run_removes_tokenized_container(tmp_path: Path) -> None: + class InterruptAfterCreateRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + result = super().run(command, **kwargs) + if [str(part) for part in command][:2] == ["docker", "run"]: + raise KeyboardInterrupt + return result + + runner = InterruptAfterCreateRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + try: + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={"SECRET": "scrub-me"}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except KeyboardInterrupt: + pass + else: + raise AssertionError("the interrupt must be restored after confirmed cleanup") + + assert runner.launched_container is None + assert ["docker", "rm", "--force", "d" * 64] in runner.calls + assert any(call[:3] == ["docker", "container", "ls"] for call in runner.calls) + assert list((tmp_path / "runs" / "docker").glob(".docker-env-*")) == [] + + +def test_interrupt_during_cleanup_discovery_preserves_pending_authority(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001 + class TimeoutRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:2] == ["docker", "run"]: + self.calls.append(argv) + return SimpleNamespace(returncode=124, stdout="", stderr="Command timed out") + return super().run(command, **kwargs) + + monkeypatch.setattr( + "envctl_engine.runtime.docker_service_runtime.time.sleep", + lambda _seconds: (_ for _ in ()).throw(KeyboardInterrupt()), + ) + runner = TimeoutRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + try: + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except DockerServiceLaunchCleanupError as exc: + pending = exc.launch + else: + raise AssertionError("cleanup cancellation must surface durable launch authority") + + assert pending.cleanup_pending_since is not None + assert pending.launch_token + assert pending.container_id == pending.container_name + + +def test_interrupt_during_post_run_identity_probe_cannot_orphan_container(tmp_path: Path) -> None: + class IdentityProbeInterruptRunner(_Runner): + def __init__(self) -> None: + super().__init__() + self.interrupted = False + + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if ( + argv[:3] == ["docker", "container", "inspect"] + and self.launched_container is not None + and not self.interrupted + ): + self.calls.append(argv) + self.interrupted = True + raise KeyboardInterrupt + return super().run(command, **kwargs) + + runner = IdentityProbeInterruptRunner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + try: + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except KeyboardInterrupt: + pass + else: + raise AssertionError("the identity-probe interrupt must be restored after cleanup") + + assert runner.launched_container is None + assert ["docker", "rm", "--force", "d" * 64] in runner.calls + + +def test_interrupting_best_effort_start_event_cannot_preempt_authority_handoff(tmp_path: Path) -> None: + runner = _Runner() + runner.local_images.add("example/backend:dev") + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + + def interrupt_start_event(event: str, **_payload: object) -> None: + if event == "service.container.start": + raise KeyboardInterrupt + + runtime._emit = interrupt_start_event + confirmed: list[object] = [] + docker_runtime = DockerServiceRuntime(runtime, runner) + + launch = docker_runtime.launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + on_confirmed=confirmed.append, + ) + + assert confirmed == [launch] + assert runner.launched_container is not None + assert docker_runtime.stop( + launch.container_id, + expected_launch_token=launch.launch_token, + ) + assert runner.launched_container is None + + +def test_running_untracked_container_is_not_replaced_or_removed(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + docker_runtime = DockerServiceRuntime(runtime, runner) + runner.existing_container = (True, docker_runtime._runtime_scope_identity()) + + try: + docker_runtime.launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except RuntimeError as exc: + assert "already running" in str(exc) + else: + raise AssertionError("expected the untracked running container to block startup") + + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + assert not any(call[:2] == ["docker", "run"] for call in runner.calls) + + +def test_container_owned_by_another_runtime_scope_is_not_replaced(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + runner.existing_container = (False, "another-runtime") + + try: + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except RuntimeError as exc: + assert "another envctl runtime scope" in str(exc) + else: + raise AssertionError("expected the foreign runtime scope to block startup") + + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + + +def test_stopped_container_preflight_rejects_invalid_running_state_and_missing_owner(tmp_path: Path) -> None: + cases = (("not-a-bool", True, "invalid running state"), ("false", False, "another envctl runtime scope")) + for running, include_owner, expected_error in cases: + class InvalidPreflightRunner(_Runner): + def run(self, command, **kwargs): # noqa: ANN001, ANN202 + argv = [str(part) for part in command] + if argv[:3] == ["docker", "container", "inspect"]: + self.calls.append(argv) + owner = DockerServiceRuntime(runtime, self)._runtime_scope_identity() if include_owner else "" + return SimpleNamespace(returncode=0, stdout=f"{'e' * 64}|{running}|{owner}\n", stderr="") + return super().run(command, **kwargs) + + runner = InvalidPreflightRunner() + runtime, _events = _runtime( + tmp_path, + runner, + env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}, + ) + + try: + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + except RuntimeError as exc: + assert expected_error in str(exc) + else: + raise AssertionError("invalid preflight identity must fail closed") + + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + assert not any(call[:2] == ["docker", "run"] for call in runner.calls) + + +def test_stopped_container_from_same_runtime_scope_is_removed(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}) + docker_runtime = DockerServiceRuntime(runtime, runner) + runner.existing_container = (False, docker_runtime._runtime_scope_identity()) + + docker_runtime.launch( + project_name="Main", + project_root=tmp_path, + service_name="backend", + cwd=tmp_path, + command=[], + env={}, + host_port=8123, + container_port=8123, + listener_expected=True, + log_path=str(tmp_path / "backend.log"), + ) + + inspect_index = next(index for index, call in enumerate(runner.calls) if call[:3] == ["docker", "container", "inspect"]) + remove_index = next(index for index, call in enumerate(runner.calls) if call[:3] == ["docker", "rm", "--force"]) + run_index = next(index for index, call in enumerate(runner.calls) if call[:2] == ["docker", "run"]) + assert inspect_index < remove_index < run_index + assert runner.calls[remove_index][-1] == "c" * 64 + + +def test_configured_image_runs_without_build_and_preserves_browser_public_url(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime( + tmp_path, + runner, + env={ + "ENVCTL_FRONTEND_DOCKER_IMAGE": "example/frontend:dev", + "ENVCTL_FRONTEND_DOCKER_COMMAND": "npm run dev -- --host 0.0.0.0 --port 9000", + "ENVCTL_FRONTEND_DOCKER_PORT": "9000", + }, + ) + + launch = DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="frontend", + cwd=tmp_path, + command=["host-only-command"], + env={ + "VITE_API_URL": "http://localhost:8000/api/v1", + "REDIS_URL": "redis://localhost:6379", + "PORT": "9100", + "HOST": "127.0.0.1", + }, + host_port=9100, + container_port=9100, + listener_expected=True, + log_path=str(tmp_path / "frontend.log"), + ) + + assert not any(call[:2] == ["docker", "build"] for call in runner.calls) + docker_run = next(call for call in runner.calls if call[:2] == ["docker", "run"]) + assert "example/frontend:dev" in docker_run + assert "127.0.0.1:9100:9000" in docker_run + assert docker_run[-8:] == ["npm", "run", "dev", "--", "--host", "0.0.0.0", "--port", "9000"] + assert "VITE_API_URL=http://localhost:8000/api/v1" in runner.env_file_payload + assert "REDIS_URL=redis://host.docker.internal:6379" in runner.env_file_payload + assert "PORT=9000" in runner.env_file_payload + assert "HOST=0.0.0.0" in runner.env_file_payload + assert launch.image == "example/frontend:dev" + + +def test_container_env_preserves_browser_contract_urls_and_rewrites_internal_urls(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime( + tmp_path, + runner, + env={"ENVCTL_BACKEND_CORS_ENV_KEY": "ALLOWED_BROWSER_ORIGINS"}, + ) + + projected = DockerServiceRuntime(runtime, runner)._container_env( + { + "FRONTEND_BASE_URL": "http://localhost:5173", + "ENVCTL_SOURCE_FRONTEND_URL": "http://127.0.0.1:5173", + "CORS_ORIGINS_RAW": "http://localhost:5173,http://127.0.0.1:5173", + "ALLOWED_BROWSER_ORIGINS": "http://localhost:5173", + "DATABASE_URL": "postgres://user:pass@localhost:5432/app", + "REDIS_URL": "redis://127.0.0.1:6379/0", + } + ) + + assert projected["FRONTEND_BASE_URL"] == "http://localhost:5173" + assert projected["ENVCTL_SOURCE_FRONTEND_URL"] == "http://127.0.0.1:5173" + assert projected["CORS_ORIGINS_RAW"] == "http://localhost:5173,http://127.0.0.1:5173" + assert projected["ALLOWED_BROWSER_ORIGINS"] == "http://localhost:5173" + assert projected["DATABASE_URL"] == "postgres://user:pass@host.docker.internal:5432/app" + assert projected["REDIS_URL"] == "redis://host.docker.internal:6379/0" + + +def test_remote_image_is_pulled_before_exposed_port_detection(tmp_path: Path) -> None: + runner = _Runner() + image = "example/frontend:remote" + runner.remote_exposed_ports[image] = {"80/tcp": {}} + runtime, _events = _runtime(tmp_path, runner, env={"ENVCTL_FRONTEND_DOCKER_IMAGE": image}) + + DockerServiceRuntime(runtime, runner).launch( + project_name="Main", + project_root=tmp_path, + service_name="frontend", + cwd=tmp_path, + command=[], + env={}, + host_port=9100, + container_port=9100, + listener_expected=True, + log_path=str(tmp_path / "frontend.log"), + ) + + pull_index = runner.calls.index(["docker", "pull", image]) + exposed_inspect_index = next( + index + for index, call in enumerate(runner.calls) + if call[:3] == ["docker", "image", "inspect"] and "--format" in call + ) + run_index = next(index for index, call in enumerate(runner.calls) if call[:2] == ["docker", "run"]) + assert pull_index < exposed_inspect_index < run_index + assert "127.0.0.1:9100:80" in runner.calls[run_index] + + +def test_verified_stop_refuses_container_from_another_runtime_scope(tmp_path: Path) -> None: + runner = _Runner() + runner.existing_container = (True, "another-runtime") + runtime, _events = _runtime(tmp_path, runner) + + stopped = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd=str(tmp_path), + runtime_kind="docker", + container_id="reused-container-id", + ), + aggressive=False, + verify_ownership=True, + ) + + assert not stopped + assert not any(call[:3] == ["docker", "rm", "--force"] for call in runner.calls) + + +def test_container_state_drives_health_and_stop(tmp_path: Path) -> None: + runner = _Runner() + runtime, events = _runtime(tmp_path, runner) + runner.existing_container = ( + True, + DockerServiceRuntime(runtime, runner)._runtime_scope_identity(), + ) + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd=str(tmp_path), + pid=4321, + requested_port=8123, + actual_port=8123, + status="running", + runtime_kind="docker", + container_id="container-id", + container_name="envctl-app-main-backend", + container_image="example/backend:dev", + ) + + assert service_truth_status(runtime, service) == "running" + runner.port_ready = False + assert service_truth_status(runtime, service) == "unreachable" + assert terminate_service_record(runtime, service, aggressive=False, verify_ownership=True) + assert ["docker", "rm", "--force", "c" * 64] in runner.calls + assert runner.terminated == [4321] + assert any(event == "service.container.stop" for event, _payload in events) + + +def test_container_logs_are_snapshotted_and_followed_only_on_demand(tmp_path: Path) -> None: + runner = _Runner() + runtime, _events = _runtime(tmp_path, runner) + log_path = tmp_path / "backend.log" + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd=str(tmp_path), + log_path=str(log_path), + runtime_kind="docker", + container_id="container-id", + ) + state = SimpleNamespace(services={service.name: service}) + + assert refresh_container_log_snapshots(runtime, state, tail=20, follow=False) == [] + assert log_path.read_text(encoding="utf-8") == "container log\n" + assert not any(call[:3] == ["docker", "logs", "--follow"] for call in runner.calls) + + followers = refresh_container_log_snapshots(runtime, state, tail=20, follow=True) + assert followers == [4321] + assert any(call[:3] == ["docker", "logs", "--follow"] for call in runner.calls) + stop_container_log_followers(runtime, followers) + assert runner.terminated == [4321] diff --git a/tests/python/runtime/test_engine_runtime_artifacts.py b/tests/python/runtime/test_engine_runtime_artifacts.py index cf3973fee..f36d0920e 100644 --- a/tests/python/runtime/test_engine_runtime_artifacts.py +++ b/tests/python/runtime/test_engine_runtime_artifacts.py @@ -21,7 +21,7 @@ ) from envctl_engine.runtime.engine_runtime_event_support import persist_events_snapshot # noqa: E402 from envctl_engine.runtime.runtime_readiness import RuntimeReadinessResult # noqa: E402 -from envctl_engine.state.models import PortPlan, RunState # noqa: E402 +from envctl_engine.state.models import PortPlan, RunState, ServiceRecord # noqa: E402 class EngineRuntimeArtifactsTests(unittest.TestCase): @@ -59,8 +59,42 @@ def test_print_summary_includes_project_ports(self) -> None: self.assertIn("envctl Python engine run summary", rendered) self.assertIn("- Main: backend=8000 frontend=9000 db=5432 redis=6379 n8n=5678", rendered) + def test_print_summary_prefers_preserved_service_truth_over_unused_planned_port(self) -> None: + context = SimpleNamespace( + name="Main", + ports={ + name: PortPlan(project="Main", requested=port, assigned=port, final=port, source="fixed") + for name, port in { + "backend": 8001, + "frontend": 9001, + "db": 5432, + "redis": 6379, + "n8n": 5678, + }.items() + }, + ) + state = RunState( + run_id="run-restart", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", type="backend", cwd="/repo/backend", actual_port=8001 + ), + "Main Frontend": ServiceRecord( + name="Main Frontend", type="frontend", cwd="/repo/frontend", actual_port=9000 + ), + }, + ) + + buffer = io.StringIO() + with redirect_stdout(buffer): + print_summary(SimpleNamespace(), state, [context]) + + self.assertIn("- Main: backend=8001 frontend=9000", buffer.getvalue()) + def test_write_artifacts_forwards_expected_state_repository_payload(self) -> None: captured: dict[str, object] = {} + committed: list[bool] = [] def save_run(**kwargs): # noqa: ANN003 captured.update(kwargs) @@ -76,13 +110,22 @@ def save_run(**kwargs): # noqa: ANN003 ) state = RunState(run_id="run-1", mode="main") - write_artifacts(runtime, state, [SimpleNamespace(name="Main")], errors=["boom"]) + write_artifacts( + runtime, + state, + [SimpleNamespace(name="Main")], + errors=["boom"], + on_commit=lambda: committed.append(True), + ) self.assertIs(captured["state"], state) self.assertEqual(captured["errors"], ["boom"]) self.assertEqual(captured["events"], [{"event": "x"}]) self.assertTrue(callable(captured["runtime_map_builder"])) self.assertTrue(callable(captured["write_runtime_readiness_report"])) + self.assertTrue(callable(captured["on_commit"])) + captured["on_commit"]() + self.assertEqual(committed, [True]) def test_persist_events_snapshot_updates_bound_run_directory(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -107,6 +150,34 @@ def test_persist_events_snapshot_updates_bound_run_directory(self) -> None: self.assertEqual((legacy_root / "events.jsonl").read_text(encoding="utf-8"), expected) self.assertEqual((run_dir / "events.jsonl").read_text(encoding="utf-8"), expected) + def test_persist_events_snapshot_uses_repository_fence_for_bound_run(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_root = Path(tmpdir) / "runtime" + legacy_root = Path(tmpdir) / "legacy" + calls: list[dict[str, object]] = [] + runtime = SimpleNamespace( + events=[{"event": "test"}], + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + env={"ENVCTL_DEBUG_UI_RUN_ID": "run-1"}, + state_repository=SimpleNamespace(write_runtime_artifact=lambda **kwargs: calls.append(kwargs) or False), + ) + + persist_events_snapshot(runtime) + + self.assertEqual( + calls, + [ + { + "run_id": "run-1", + "artifact_name": "events.jsonl", + "text": '{"event": "test"}\n', + } + ], + ) + self.assertFalse((runtime_root / "events.jsonl").exists()) + self.assertFalse((legacy_root / "events.jsonl").exists()) + def test_write_artifacts_writes_runtime_readiness_report_synchronously_without_explicit_async_opt_in(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: runtime_root = Path(tmpdir) / "runtime" @@ -235,6 +306,51 @@ def test_write_runtime_readiness_report_writes_runtime_and_run_reports(self) -> self.assertEqual(run_report["gap_report"]["sha256"], "gap123") self.assertEqual(run_report["feature_matrix"]["sha256"], "matrix123") + def test_readiness_report_does_not_bypass_repository_when_stale_run_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_root = Path(tmpdir) / "runtime" + legacy_root = Path(tmpdir) / "legacy" + run_dir = runtime_root / "runs" / "run-retired" + calls: list[dict[str, object]] = [] + result = RuntimeReadinessResult( + passed=True, + report_path=Path(tmpdir) / "gap.json", + report_generated_at="now", + report_sha256="gap", + matrix_path=Path(tmpdir) / "matrix.json", + matrix_generated_at="now", + matrix_sha256="matrix", + parity_manifest_path=Path(tmpdir) / "manifest.json", + parity_manifest_generated_at="now", + parity_manifest_sha256="manifest", + blocking_gap_count=0, + high_gap_count=0, + medium_gap_count=0, + low_gap_count=0, + total_gap_count=0, + errors=[], + warnings=[], + ) + runtime = SimpleNamespace( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + config=SimpleNamespace(base_dir=Path(tmpdir)), + state_repository=SimpleNamespace(write_runtime_artifact=lambda **kwargs: calls.append(kwargs) or False), + ) + + persisted = write_runtime_readiness_report( + runtime, + run_dir=run_dir, + readiness_result=result, + ) + + self.assertFalse(persisted) + self.assertEqual(calls[0]["run_id"], "run-retired") + self.assertEqual(calls[0]["artifact_name"], "runtime_readiness_report.json") + self.assertFalse((runtime_root / "runtime_readiness_report.json").exists()) + self.assertFalse((legacy_root / "runtime_readiness_report.json").exists()) + self.assertFalse((run_dir / "runtime_readiness_report.json").exists()) + def test_write_runtime_readiness_report_reuses_cached_payload_when_report_hash_matches(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo_root = Path(tmpdir) / "repo" @@ -249,10 +365,16 @@ def test_write_runtime_readiness_report_reuses_cached_payload_when_report_hash_m run_dir.mkdir() report_text = json.dumps({"generated_at": "now", "summary": {"total_gap_count": 0}}, sort_keys=True) report_hash = hashlib.sha256(report_text.encode("utf-8")).hexdigest() - matrix_text = json.dumps({"generated_at": "now", "features": [], "summary": {"feature_count": 0}}, sort_keys=True) - matrix_hash = hashlib.sha256((json.dumps(json.loads(matrix_text), indent=2, sort_keys=True) + "\n").encode("utf-8")).hexdigest() + matrix_text = json.dumps( + {"generated_at": "now", "features": [], "summary": {"feature_count": 0}}, sort_keys=True + ) + matrix_hash = hashlib.sha256( + (json.dumps(json.loads(matrix_text), indent=2, sort_keys=True) + "\n").encode("utf-8") + ).hexdigest() gap_report_path.write_text(report_text, encoding="utf-8") - matrix_path.write_text(json.dumps(json.loads(matrix_text), indent=2, sort_keys=True) + "\n", encoding="utf-8") + matrix_path.write_text( + json.dumps(json.loads(matrix_text), indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) manifest_path.write_text( json.dumps({"generated_at": "now", "commands": {}, "modes": {}}, sort_keys=True), encoding="utf-8" ) diff --git a/tests/python/runtime/test_engine_runtime_command_parity_state_config.py b/tests/python/runtime/test_engine_runtime_command_parity_state_config.py index a63b11e1b..b830b37cb 100644 --- a/tests/python/runtime/test_engine_runtime_command_parity_state_config.py +++ b/tests/python/runtime/test_engine_runtime_command_parity_state_config.py @@ -2,9 +2,29 @@ from __future__ import annotations from tests.python.runtime.engine_runtime_command_parity_test_support import * +from envctl_engine.runtime.inspection_support import _running_tree_project_names +from envctl_engine.state.run_index import StateSelector class EngineRuntimeCommandParityStateConfigTests(EngineRuntimeCommandParityTestCase): + def test_running_tree_projects_prefers_additional_service_project_metadata(self) -> None: + state = RunState( + run_id="run-1", + mode="trees", + services={ + "Customer Platform Voice Runtime": ServiceRecord( + name="Customer Platform Voice Runtime", + type="voice-runtime", + cwd=".", + status="running", + project="Customer Platform", + service_slug="voice-runtime", + ) + }, + ) + runtime = SimpleNamespace(_project_name_from_service=lambda _name: "") + + self.assertEqual(_running_tree_project_names(runtime=runtime, state=state), {"Customer Platform"}) def test_list_targets_discovers_projects_in_main_mode(self) -> None: tmpdir = tempfile.TemporaryDirectory() @@ -82,7 +102,11 @@ def test_list_trees_json_reports_selected_state_presence_and_service_truth(self) state=RunState( run_id="run-prev", mode="trees", - services={"feature-1 Backend": ServiceRecord(name="feature-1 Backend", type="backend", cwd=".", status="unknown")}, + services={ + "feature-1 Backend": ServiceRecord( + name="feature-1 Backend", type="backend", cwd=".", status="unknown" + ) + }, requirements={}, metadata={"repo_scope_id": config.runtime_scope_id, "project_roots": {"feature-1": str(feature_dir)}}, ), @@ -104,6 +128,66 @@ def test_list_trees_json_reports_selected_state_presence_and_service_truth(self) self.assertFalse(feature["services_running"]) self.assertFalse(feature["running"]) + def test_tree_list_json_reports_discovery_and_integrity_warning_for_tampered_active_run(self) -> None: + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + repo = Path(tmpdir.name) / "repo" + runtime_root = Path(tmpdir.name) / "runtime" + (repo / ".git").mkdir(parents=True, exist_ok=True) + feature_dirs = { + name: repo / "trees" / "feature" / suffix for name, suffix in (("feature-1", "1"), ("feature-2", "2")) + } + for feature_dir in feature_dirs.values(): + feature_dir.mkdir(parents=True, exist_ok=True) + (feature_dir / ".envctl").write_text("ENVCTL_DEFAULT_MODE=trees\n") + config = load_config({"RUN_REPO_ROOT": str(repo), "RUN_SH_RUNTIME_DIR": str(runtime_root)}) + runtime = PythonEngineRuntime(config, env={}) + + for run_id, project_name in (("run-valid", "feature-1"), ("run-tampered", "feature-2")): + runtime.state_repository.save_resume_state( + state=RunState( + run_id=run_id, + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=str(feature_dirs[project_name]), + project=project_name, + ) + }, + requirements={}, + metadata={ + "repo_scope_id": config.runtime_scope_id, + "project_roots": {project_name: str(feature_dirs[project_name])}, + }, + ), + emit=lambda *args, **kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + tampered = next( + candidate + for candidate in runtime.state_repository.run_index.candidates( + StateSelector(mode="trees", project_names=()) + ) + if candidate.run_id == "run-tampered" + ) + tampered.state_path.write_text('{"broken":', encoding="utf-8") + + for args in (("--list-trees", "--json"), ("--list-targets", "--trees", "--json")): + with self.subTest(args=args): + buffer = StringIO() + with redirect_stdout(buffer): + code = runtime.dispatch(parse_route(list(args), env={})) + + self.assertEqual(code, 0) + payload = json.loads(buffer.getvalue()) + self.assertEqual({project["name"] for project in payload["projects"]}, set(feature_dirs)) + self.assertEqual(payload["state_error"]["code"], "runtime_state_integrity_error") + self.assertIn("run-tampered", payload["state_error"]["message"]) + self.assertEqual(payload["warnings"], [payload["state_error"]]) + self.assertFalse(any(project["state_present"] for project in payload["projects"])) + def test_list_trees_json_reports_imported_branch_worktrees_individually(self) -> None: tmpdir = tempfile.TemporaryDirectory() self.addCleanup(tmpdir.cleanup) @@ -289,7 +373,10 @@ def test_show_config_json_reports_additional_service_validation_errors_and_metad code = runtime.dispatch(parse_route(["--show-config", "--json"], env={})) self.assertEqual(code, 0) payload = json.loads(buffer.getvalue()) - self.assertIn("additional service dependency cycle: voice-runtime -> worker -> voice-runtime", payload["additional_service_errors"]) + self.assertIn( + "additional service dependency cycle: voice-runtime -> worker -> voice-runtime", + payload["additional_service_errors"], + ) self.assertTrue(payload["service_dependency_env_section_present"]["voice-runtime"]) self.assertEqual(payload["service_dependency_env_templates"]["voice-runtime"][0]["name"], "VOICE_PUBLIC") diff --git a/tests/python/runtime/test_engine_runtime_commands.py b/tests/python/runtime/test_engine_runtime_commands.py index 9b1af759b..0d58043d6 100644 --- a/tests/python/runtime/test_engine_runtime_commands.py +++ b/tests/python/runtime/test_engine_runtime_commands.py @@ -2,6 +2,7 @@ import os from pathlib import Path +import sys import tempfile import unittest from types import SimpleNamespace @@ -14,11 +15,77 @@ command_env, command_override_value, default_python_executable, + service_start_command_resolved, split_command, ) class EngineRuntimeCommandsTests(unittest.TestCase): + def test_docker_image_command_skips_host_command_resolution(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime = SimpleNamespace( + env={ + "DOCKER_MODE": "true", + "ENVCTL_FRONTEND_DOCKER_IMAGE": "example/frontend:dev", + }, + config=SimpleNamespace( + base_dir=root, + raw={"FRONTEND_DIR": "frontend"}, + ), + _command_exists=lambda _executable: self.fail("host command lookup must be skipped"), + ) + + command, source = service_start_command_resolved( + runtime, + service_name="frontend", + project_root=root, + port=8010, + ) + + self.assertEqual(command, []) + self.assertEqual(source, "docker_image") + + def test_explicit_docker_command_skips_host_command_resolution(self) -> None: + runtime = SimpleNamespace( + env={ + "DOCKER_MODE": "true", + "ENVCTL_BACKEND_DOCKER_COMMAND": "uvicorn app:api --port 8000", + }, + config=SimpleNamespace(base_dir=Path("/repo"), raw={}), + _command_exists=lambda _executable: self.fail("host command lookup must be skipped"), + ) + + command, source = service_start_command_resolved(runtime, service_name="backend", port=8000) + + self.assertEqual(command, []) + self.assertEqual(source, "docker_command") + + def test_docker_service_command_still_requires_executable_on_host(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + (root / "frontend").mkdir() + runtime = SimpleNamespace( + env={ + "DOCKER_MODE": "true", + "ENVCTL_FRONTEND_DOCKER_COMMAND_MODE": "service", + "ENVCTL_FRONTEND_START_CMD": "missing-command --port {port}", + }, + config=SimpleNamespace( + base_dir=root, + raw={"FRONTEND_DIR": "frontend"}, + ), + _command_exists=lambda _executable: False, + ) + + with self.assertRaises(RuntimeError): + service_start_command_resolved( + runtime, + service_name="frontend", + project_root=root, + port=8010, + ) + def test_command_override_value_prefers_runtime_env(self) -> None: runtime = SimpleNamespace( env={"DB_HOST": "env-db"}, @@ -49,6 +116,22 @@ def test_command_env_sets_port_and_overrides(self) -> None: self.assertEqual(env["PORT"], "9000") self.assertEqual(env["A"], "1") self.assertEqual(env["B"], "2") + self.assertEqual(env["ENVCTL_PYTHON_EXECUTABLE"], sys.executable) + + def test_command_env_keeps_running_interpreter_authoritative(self) -> None: + runtime = SimpleNamespace(env={"ENVCTL_PYTHON_EXECUTABLE": "/stale/runtime/python"}) + + with patch( + "envctl_engine.runtime.engine_runtime_commands.sys.executable", + "/opt/envctl/bin/python", + ): + env = command_env( + runtime, + port=9000, + extra={"ENVCTL_PYTHON_EXECUTABLE": "/spoofed/project/python"}, + ) + + self.assertEqual(env["ENVCTL_PYTHON_EXECUTABLE"], "/opt/envctl/bin/python") def test_command_env_strips_github_actions_cleanup_variables(self) -> None: runtime = SimpleNamespace(env={"A": "1"}) @@ -113,12 +196,32 @@ def test_split_command_accepts_relative_executable_from_cwd(self) -> None: script = root / "scripts" / "start-service.sh" script.parent.mkdir(parents=True, exist_ok=True) script.write_text("#!/usr/bin/env sh\n", encoding="utf-8") + script.chmod(0o755) runtime = SimpleNamespace(_command_exists=lambda executable: False) parsed = split_command(runtime, "scripts/start-service.sh {port}", port=8123, cwd=root) self.assertEqual(parsed, ["scripts/start-service.sh", "8123"]) + def test_split_command_rejects_non_executable_file_and_directory(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + file_path = root / "start-service.sh" + file_path.write_text("#!/usr/bin/env sh\n", encoding="utf-8") + directory_path = root / "service-dir" + directory_path.mkdir() + runtime = SimpleNamespace(_command_exists=lambda _executable: True) + + for command in (str(file_path), str(directory_path)): + with ( + self.subTest(command=command), + self.assertRaisesRegex( + RuntimeError, + "Resolved command executable not found", + ), + ): + split_command(runtime, command, cwd=root) + def test_default_python_executable_falls_back_to_python3(self) -> None: runtime = SimpleNamespace(env={}, _command_exists=lambda executable: False) diff --git a/tests/python/runtime/test_engine_runtime_dispatch.py b/tests/python/runtime/test_engine_runtime_dispatch.py index ec0b61560..27d5a4367 100644 --- a/tests/python/runtime/test_engine_runtime_dispatch.py +++ b/tests/python/runtime/test_engine_runtime_dispatch.py @@ -1,29 +1,40 @@ from __future__ import annotations import importlib -from pathlib import Path +import tempfile +import threading import unittest -from io import StringIO from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path from types import SimpleNamespace -import tempfile from unittest.mock import patch +from envctl_engine.state.repository import RuntimeStateRepository + REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" dispatch_module = importlib.import_module("envctl_engine.runtime.engine_runtime_dispatch") +command_policy_module = importlib.import_module("envctl_engine.runtime.command_policy") +lifecycle_lease_module = importlib.import_module("envctl_engine.runtime.lifecycle_operation_lease") utility_module = importlib.import_module("envctl_engine.runtime.utility_command_support") dispatch = dispatch_module.dispatch dispatch_command = dispatch_module.dispatch_command class EngineRuntimeDispatchTests(unittest.TestCase): + def _runtime_root(self) -> Path: + temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(temporary_directory.cleanup) + return Path(temporary_directory.name) + def test_dispatch_entry_configures_probe_debug_recorder_and_route_events(self) -> None: events: list[tuple[str, dict[str, object]]] = [] calls: list[str] = [] route = SimpleNamespace(command="start", mode="main") runtime = SimpleNamespace( + runtime_root=self._runtime_root(), process_probe=None, _build_process_probe_backend=lambda: calls.append("probe_backend") or "backend", _effective_start_mode=lambda value: calls.append(f"effective:{value.command}") or "trees", @@ -87,6 +98,213 @@ def test_dispatch_entry_uses_route_mode_for_non_startup_commands(self) -> None: self.assertEqual(code, 4) self.assertTrue(all(payload["effective_mode"] == "trees" for _event, payload in events)) + def test_exclusive_operation_commands_follow_mutating_command_policy_sets(self) -> None: + expected = { + *command_policy_module.LIFECYCLE_CLEANUP_COMMANDS, + *command_policy_module.STARTUP_COMMANDS, + "blast-worktree", + "delete-worktree", + "ensure-worktree", + "resume", + "self-destruct-worktree", + } + + self.assertEqual(dispatch_module._EXCLUSIVE_OPERATION_COMMANDS, frozenset(expected)) + self.assertIn("import", expected) + self.assertIn("self-destruct-worktree", expected) + self.assertNotIn("test-focused", expected) + self.assertNotIn("ship", expected) + + def test_nested_lifecycle_dispatch_is_reentrant_for_owning_runtime_thread(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + commands: list[str] = [] + runtime = SimpleNamespace( + runtime_root=self._runtime_root(), + process_probe=None, + _build_process_probe_backend=lambda: "backend", + _effective_start_mode=lambda route: route.mode, + _configure_debug_recorder=lambda _route: None, + _emit=lambda event, **payload: events.append((event, payload)), + ) + outer_route = SimpleNamespace(command="start", mode="trees") + nested_route = SimpleNamespace(command="stop", mode="trees") + + def command_dispatch(_runtime: object, route: object) -> int: + command = str(getattr(route, "command", "")) + commands.append(command) + if command == "start": + return dispatch(runtime, nested_route) + return 7 + + with ( + patch("envctl_engine.runtime.engine_runtime_dispatch.ProcessProbe", return_value="probe"), + patch( + "envctl_engine.runtime.engine_runtime_dispatch.dispatch_command", + side_effect=command_dispatch, + ), + ): + code = dispatch(runtime, outer_route) + + self.assertEqual(code, 7) + self.assertEqual(commands, ["start", "stop"]) + self.assertFalse(any(event == "lifecycle.operation.busy" for event, _payload in events)) + self.assertIsNone(getattr(runtime, "_lifecycle_operation_owner", None)) + self.assertEqual(getattr(runtime, "_lifecycle_operation_depth", 0), 0) + + def test_concurrent_lifecycle_dispatch_on_another_thread_fails_busy_without_overlap(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + commands: list[str] = [] + entered = threading.Event() + release = threading.Event() + results: list[int] = [] + errors: list[BaseException] = [] + runtime = SimpleNamespace( + runtime_root=self._runtime_root(), + process_probe=None, + _build_process_probe_backend=lambda: "backend", + _effective_start_mode=lambda route: route.mode, + _configure_debug_recorder=lambda _route: None, + _emit=lambda event, **payload: events.append((event, payload)), + ) + outer_route = SimpleNamespace(command="start", mode="trees") + concurrent_route = SimpleNamespace(command="stop", mode="trees") + + def command_dispatch(_runtime: object, route: object) -> int: + command = str(getattr(route, "command", "")) + commands.append(command) + if command == "start": + entered.set() + if not release.wait(timeout=2): + raise AssertionError("timed out waiting to release lifecycle operation") + return 0 + + def run_outer() -> None: + try: + results.append(dispatch(runtime, outer_route)) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + with ( + patch("envctl_engine.runtime.engine_runtime_dispatch.ProcessProbe", return_value="probe"), + patch( + "envctl_engine.runtime.engine_runtime_dispatch.dispatch_command", + side_effect=command_dispatch, + ), + ): + thread = threading.Thread(target=run_outer, name="envctl-dispatch-lock-test") + thread.start() + try: + self.assertTrue(entered.wait(timeout=2)) + output = StringIO() + with redirect_stdout(output): + concurrent_code = dispatch(runtime, concurrent_route) + finally: + release.set() + thread.join(timeout=2) + + self.assertFalse(thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(results, [0]) + self.assertEqual(concurrent_code, 1) + self.assertEqual(commands, ["start"]) + self.assertIn("Another envctl lifecycle operation is already active", output.getvalue()) + self.assertTrue(any(event == "lifecycle.operation.busy" for event, _payload in events)) + + def test_committed_start_releases_lease_before_interactive_wait_so_stop_can_run(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + commands: list[str] = [] + interactive_wait_entered = threading.Event() + release_interactive_wait = threading.Event() + results: list[int] = [] + runtime = SimpleNamespace( + runtime_root=self._runtime_root(), + process_probe=None, + _build_process_probe_backend=lambda: "backend", + _effective_start_mode=lambda route: route.mode, + _configure_debug_recorder=lambda _route: None, + _emit=lambda event, **payload: events.append((event, payload)), + ) + start_route = SimpleNamespace(command="start", mode="trees") + stop_route = SimpleNamespace(command="stop", mode="trees") + + def command_dispatch(_runtime: object, route: object) -> int: + command = str(getattr(route, "command", "")) + commands.append(command) + if command == "start": + released = lifecycle_lease_module.release_lifecycle_operation(runtime) + self.assertTrue(released) + interactive_wait_entered.set() + self.assertTrue(release_interactive_wait.wait(timeout=2)) + return 0 + + def run_start() -> None: + results.append(dispatch(runtime, start_route)) + + with ( + patch("envctl_engine.runtime.engine_runtime_dispatch.ProcessProbe", return_value="probe"), + patch( + "envctl_engine.runtime.engine_runtime_dispatch.dispatch_command", + side_effect=command_dispatch, + ), + ): + thread = threading.Thread(target=run_start, name="envctl-interactive-start-test") + thread.start() + try: + self.assertTrue(interactive_wait_entered.wait(timeout=2)) + stop_code = dispatch(runtime, stop_route) + finally: + release_interactive_wait.set() + thread.join(timeout=2) + + self.assertFalse(thread.is_alive()) + self.assertEqual(results, [0]) + self.assertEqual(stop_code, 0) + self.assertEqual(commands, ["start", "stop"]) + self.assertFalse(any(event == "lifecycle.operation.busy" for event, _payload in events)) + + def test_operation_lock_rejects_retargeted_repository_root_before_external_write(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + runtime_root.mkdir(parents=True) + legacy_root.mkdir() + repository = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + external = Path(tmpdir) / "external" + external.mkdir() + sentinel = external / "sentinel.txt" + sentinel.write_text("untouched", encoding="utf-8") + runtime_root.rmdir() + runtime_root.symlink_to(external, target_is_directory=True) + runtime = SimpleNamespace( + runtime_root=runtime_root, + state_repository=repository, + process_probe=None, + _build_process_probe_backend=lambda: "backend", + _effective_start_mode=lambda route: route.mode, + _configure_debug_recorder=lambda _route: None, + _emit=lambda *_args, **_kwargs: None, + ) + + with ( + patch("envctl_engine.runtime.engine_runtime_dispatch.ProcessProbe", return_value="probe"), + patch( + "envctl_engine.runtime.engine_runtime_dispatch.dispatch_command", + side_effect=AssertionError("command must not run"), + ), + self.assertRaisesRegex(RuntimeError, "symlink"), + ): + dispatch(runtime, SimpleNamespace(command="start", mode="trees")) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "untouched") + self.assertEqual(list(external.iterdir()), [sentinel]) + def test_list_commands_dispatch_prints_supported_commands(self) -> None: runtime = SimpleNamespace() route = SimpleNamespace(command="list-commands", mode="main") diff --git a/tests/python/runtime/test_engine_runtime_event_support.py b/tests/python/runtime/test_engine_runtime_event_support.py index c0812ebe1..04db99b74 100644 --- a/tests/python/runtime/test_engine_runtime_event_support.py +++ b/tests/python/runtime/test_engine_runtime_event_support.py @@ -53,6 +53,32 @@ def test_emit_records_event_and_notifies_listener(self) -> None: self.assertEqual(received[0][0], "ui.input.submit") self.assertIn("command_hash", received[0][1]) + def test_emit_disables_failed_debug_recorder_without_interrupting_lifecycle(self) -> None: + received: list[str] = [] + + class FailedRecorder: + def record(self, *_args: object, **_kwargs: object) -> None: + raise OSError("debug disk unavailable") + + runtime = SimpleNamespace( + _active_command_id="cmd-1", + _debug_hash_salt="salt", + _emit_lock=threading.Lock(), + events=[], + _debug_recorder=FailedRecorder(), + _debug_recorder_failure=None, + _emit_listeners=[lambda event, _payload: received.append(event)], + env={}, + config=SimpleNamespace(raw={}), + ) + + emit(runtime, "service.start", project="Main") + + self.assertEqual(runtime.events[0]["event"], "service.start") + self.assertEqual(received, ["service.start"]) + self.assertIsNone(runtime._debug_recorder) + self.assertEqual(runtime._debug_recorder_failure, "debug disk unavailable") + def test_debug_should_auto_pack_policies(self) -> None: runtime = SimpleNamespace(env={"ENVCTL_DEBUG_AUTO_PACK": "anomaly"}, config=SimpleNamespace(raw={})) diff --git a/tests/python/runtime/test_engine_runtime_hooks.py b/tests/python/runtime/test_engine_runtime_hooks.py index d0af728d2..a6bd1a68f 100644 --- a/tests/python/runtime/test_engine_runtime_hooks.py +++ b/tests/python/runtime/test_engine_runtime_hooks.py @@ -126,13 +126,24 @@ def test_services_from_hook_payload_builds_records_and_updates_service_ports(sel "port": 9010, "status": "healthy", }, + { + "name": "Opaque Worker Process", + "type": "voice-runtime", + "pid": 103, + "port": 9020, + }, ] }, ) - self.assertEqual(sorted(records.keys()), ["feature/a-1 Backend", "feature/a-1 Frontend"]) + self.assertEqual( + sorted(records.keys()), + ["Opaque Worker Process", "feature/a-1 Backend", "feature/a-1 Frontend"], + ) self.assertEqual(records["feature/a-1 Backend"].actual_port, 8010) self.assertEqual(records["feature/a-1 Frontend"].requested_port, 9010) + self.assertEqual(records["Opaque Worker Process"].project, "feature/a-1") + self.assertEqual(records["Opaque Worker Process"].service_slug, "voice-runtime") self.assertEqual(context.ports["backend"].final, 8010) self.assertEqual(context.ports["frontend"].final, 9010) diff --git a/tests/python/runtime/test_engine_runtime_lifecycle_support.py b/tests/python/runtime/test_engine_runtime_lifecycle_support.py index 78647f164..9ac92e916 100644 --- a/tests/python/runtime/test_engine_runtime_lifecycle_support.py +++ b/tests/python/runtime/test_engine_runtime_lifecycle_support.py @@ -36,12 +36,17 @@ def test_release_requirement_ports_releases_enabled_final_ports(self) -> None: project="Main", db={"enabled": True, "final": 5432}, redis={"enabled": False, "final": 6379}, + supabase={ + "enabled": True, + "final": 15432, + "resources": {"db": 15432, "api": 15433}, + }, n8n={"enabled": True, "final": 5678}, ) release_requirement_ports(runtime, requirements) - self.assertEqual(released, [5432, 5678]) + self.assertEqual(released, [5432, 5678, 15432, 15433]) def test_release_requirement_ports_skips_external_dependency_ports(self) -> None: released: list[int] = [] @@ -108,10 +113,16 @@ def test_terminate_services_from_state_releases_ports_for_terminated_services(se def test_terminate_service_record_prefers_process_group_termination(self) -> None: calls: list[tuple[str, int]] = [] + live_pids = {3210} runtime = SimpleNamespace( _emit=lambda *args, **kwargs: None, process_runner=SimpleNamespace( - terminate_process_group=lambda pid, **kwargs: calls.append(("group", pid)) or True, + is_pid_running=lambda pid: pid in live_pids, + terminate_process_group=lambda pid, **kwargs: ( + calls.append(("group", pid)), + live_pids.discard(pid), + True, + )[-1], terminate=lambda pid, **kwargs: calls.append(("single", pid)) or True, ), ) @@ -129,6 +140,7 @@ def test_terminate_service_record_prefers_process_group_termination(self) -> Non def test_terminate_service_record_accepts_verified_listener_child_ownership(self) -> None: calls: list[tuple[str, int]] = [] skips: list[dict[str, object]] = [] + live_pids = {111, 222} def pid_owns_port(pid: int, port: int) -> bool: return pid == 222 and port == 9000 @@ -136,8 +148,13 @@ def pid_owns_port(pid: int, port: int) -> bool: runtime = SimpleNamespace( _emit=lambda event, **payload: skips.append(payload) if event == "cleanup.skip" else None, process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, pid_owns_port=pid_owns_port, - terminate_process_group=lambda pid, **kwargs: calls.append(("group", pid)) or True, + terminate_process_group=lambda pid, **kwargs: ( + calls.append(("group", pid)), + live_pids.discard(pid), + True, + )[-1], terminate=lambda pid, **kwargs: calls.append(("single", pid)) or True, ), ) @@ -150,7 +167,7 @@ def pid_owns_port(pid: int, port: int) -> bool: ) self.assertTrue(terminated) - self.assertEqual(calls, [("group", 111)]) + self.assertEqual(calls, [("group", 111), ("group", 222)]) self.assertEqual(skips, []) def test_blast_worktree_before_delete_updates_state_and_emits_finish(self) -> None: @@ -173,7 +190,7 @@ def test_blast_worktree_before_delete_updates_state_and_emits_finish(self) -> No _emit=lambda event, **payload: events.append((event, payload)), _try_load_existing_state=lambda **kwargs: state if kwargs["mode"] == "trees" else None, _project_name_from_service=lambda name: "Feature" if "Feature" in name else "", - _terminate_services_from_state=lambda *args, **kwargs: None, + _terminate_services_from_state=lambda *args, **kwargs: set(), port_planner=SimpleNamespace(release=lambda port: released.append(port)), state_repository=SimpleNamespace( save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) @@ -282,7 +299,7 @@ def run(self, cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001, ARG002 _emit=lambda event, **payload: events.append((event, payload)), _try_load_existing_state=lambda **kwargs: state if kwargs["mode"] == "trees" else None, _project_name_from_service=lambda name: "Feature" if "Feature" in name else "", - _terminate_services_from_state=lambda *args, **kwargs: None, + _terminate_services_from_state=lambda *args, **kwargs: set(), _blast_all_process_command=lambda pid: f"python service {pid}", _blast_all_kill_pid_tree=lambda pid: killed_pids.append(pid), _collect_container_volume_candidates=lambda cid, volumes: volumes.append(f"vol-{cid}"), @@ -366,7 +383,10 @@ def _resolve_proc_path(path_obj: Path, strict: bool = False) -> Path: # noqa: A return path_obj with ( - patch("envctl_engine.runtime.engine_runtime_lifecycle_support.Path.iterdir", return_value=[proc_root / "111", proc_root / "222"]), + patch( + "envctl_engine.runtime.engine_runtime_lifecycle_support.Path.iterdir", + return_value=[proc_root / "111", proc_root / "222"], + ), patch("pathlib.Path.resolve", autospec=True, side_effect=_resolve_proc_path), patch( "envctl_engine.runtime.engine_runtime_lifecycle_support._remove_tree_containers", diff --git a/tests/python/runtime/test_engine_runtime_plan_resume_startup.py b/tests/python/runtime/test_engine_runtime_plan_resume_startup.py index 1c7b37174..1a239fd61 100644 --- a/tests/python/runtime/test_engine_runtime_plan_resume_startup.py +++ b/tests/python/runtime/test_engine_runtime_plan_resume_startup.py @@ -100,12 +100,14 @@ def fake_start_project_context(*, context, mode, route, run_id): # noqa: ANN001 with ( patch.object(engine, "_try_load_existing_state", return_value=state), patch.object(engine, "_resume", return_value=1) as resume_mock, + patch.object(engine, "_terminate_services_from_state", return_value=set()) as terminate_mock, patch.object(engine, "_start_project_context", side_effect=fake_start_project_context), ): code = engine.dispatch(route) self.assertEqual(code, 0) self.assertEqual(resume_mock.call_count, 1) + self.assertEqual(terminate_mock.call_count, 1) self.assertEqual(started_projects, ["feature-a-1"]) def test_plan_auto_resumes_existing_run_when_selected_projects_are_subset_of_state(self) -> None: @@ -282,6 +284,7 @@ def test_plan_no_resume_flag_disables_auto_resume(self) -> None: with ( patch.object(engine, "_try_load_existing_state", return_value=state), patch.object(engine, "_resume", return_value=0) as resume_mock, + patch.object(engine, "_terminate_services_from_state", return_value=set()) as terminate_mock, patch.object(engine, "_start_project_context", side_effect=RuntimeError("expected startup path")), ): out = StringIO() @@ -290,6 +293,7 @@ def test_plan_no_resume_flag_disables_auto_resume(self) -> None: self.assertEqual(code, 1) self.assertEqual(resume_mock.call_count, 0) + self.assertEqual(terminate_mock.call_count, 1) self.assertIn("expected startup path", out.getvalue()) def test_exact_tree_plan_match_reuses_prior_run_id(self) -> None: @@ -339,4 +343,3 @@ def fake_result(context: ProjectContext) -> ProjectStartupResult: assert second_state is not None self.assertEqual(second_state.run_id, first_state.run_id) self.assertNotEqual(first_engine._current_session_id(), second_session_id) - diff --git a/tests/python/runtime/test_engine_runtime_plan_selection_startup.py b/tests/python/runtime/test_engine_runtime_plan_selection_startup.py index e518f4842..6d1da16b2 100644 --- a/tests/python/runtime/test_engine_runtime_plan_selection_startup.py +++ b/tests/python/runtime/test_engine_runtime_plan_selection_startup.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import os import re import tempfile @@ -75,36 +74,6 @@ def test_plan_without_selection_and_without_tty_fails_when_planning_files_exist( self.assertEqual(code, 1) - def test_initial_plan_selected_counts_prefers_existing_counts(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - repo = Path(tmpdir) / "repo" - runtime = Path(tmpdir) / "runtime" - (repo / ".git").mkdir(parents=True, exist_ok=True) - engine = PythonEngineRuntime(self._config(repo, runtime), env={}) - selected = engine._initial_plan_selected_counts( - planning_files=["backend/task-a.md", "backend/task-b.md"], - existing_counts={"backend/task-a.md": 2, "backend/task-b.md": 1}, - ) - self.assertEqual(selected, {"backend/task-a.md": 2, "backend/task-b.md": 1}) - - def test_initial_plan_selected_counts_ignores_memory_when_existing_missing(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - repo = Path(tmpdir) / "repo" - runtime = Path(tmpdir) / "runtime" - (repo / ".git").mkdir(parents=True, exist_ok=True) - engine = PythonEngineRuntime(self._config(repo, runtime), env={}) - memory_path = runtime / "python-engine" / "planning_selection.json" - memory_path.parent.mkdir(parents=True, exist_ok=True) - memory_path.write_text( - json.dumps({"selected_counts": {"backend/task-a.md": 3}}, indent=2, sort_keys=True), - encoding="utf-8", - ) - selected = engine._initial_plan_selected_counts( - planning_files=["backend/task-a.md", "backend/task-b.md"], - existing_counts={}, - ) - self.assertEqual(selected, {"backend/task-a.md": 0, "backend/task-b.md": 0}) - def test_planning_menu_apply_key_supports_arrow_navigation_and_count_adjustment(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo = Path(tmpdir) / "repo" diff --git a/tests/python/runtime/test_engine_runtime_real_startup.py b/tests/python/runtime/test_engine_runtime_real_startup.py index e65fcd7a8..b985eae31 100644 --- a/tests/python/runtime/test_engine_runtime_real_startup.py +++ b/tests/python/runtime/test_engine_runtime_real_startup.py @@ -72,7 +72,7 @@ def test_startup_emits_port_lock_reclaim_event_with_previous_owner_metadata(self lock_path.write_text( json.dumps( { - "owner": "old-worker", + "owner": "feature-a-1:backend", "session": "old-session", "pid": 999999, "created_at": "2000-01-01T00:00:00+00:00", @@ -95,7 +95,7 @@ def test_startup_emits_port_lock_reclaim_event_with_previous_owner_metadata(self self.assertTrue(reclaim_events) latest = reclaim_events[-1] self.assertEqual(latest.get("port"), planned_backend_port) - self.assertEqual(latest.get("reclaimed_owner"), "old-worker") + self.assertEqual(latest.get("reclaimed_owner"), "feature-a-1:backend") self.assertEqual(latest.get("reclaimed_session"), "old-session") self.assertEqual(latest.get("reclaimed_pid"), 999999) @@ -253,10 +253,11 @@ def capture_emit(event: str, **payload: object) -> None: rendered = out.getvalue() self.assertIn("\x1b]8;;file://", rendered) visible = strip_ansi(rendered) - self.assertIn("backend listener not detected", visible) + self.assertIn("backend is missing a required executable or module", visible) self.assertIn("log_path:", visible) failure_events = [payload for event, payload in captured_events if event == "service.failure"] self.assertTrue(failure_events) + self.assertEqual(failure_events[0]["failure_class"], "dependency_missing") detail = str(failure_events[0]["detail"]) self.assertIn("log_path:", detail) self.assertNotIn("\x1b]8;;", detail) diff --git a/tests/python/runtime/test_engine_runtime_resume_startup.py b/tests/python/runtime/test_engine_runtime_resume_startup.py index b7a62022f..8bc3e98c9 100644 --- a/tests/python/runtime/test_engine_runtime_resume_startup.py +++ b/tests/python/runtime/test_engine_runtime_resume_startup.py @@ -214,7 +214,9 @@ def fake_result(context: ProjectContext) -> ProjectStartupResult: first_engine = PythonEngineRuntime(self._config(repo, runtime), env={}) first_engine._reconcile_state_truth = lambda _state: [] # type: ignore[assignment] - with patch.object(first_engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"])): + with patch.object( + first_engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"]) + ): first_code = first_engine.dispatch(parse_route(["--plan", "feature-a", "--batch"], env={})) self.assertEqual(first_code, 0) first_state = first_engine.state_repository.load_latest(mode="trees", strict_mode_match=True) @@ -223,10 +225,20 @@ def fake_result(context: ProjectContext) -> ProjectStartupResult: second_engine = PythonEngineRuntime(self._config(repo, runtime, extra={"BACKEND_DIR": "api"}), env={}) second_engine._reconcile_state_truth = lambda _state: [] # type: ignore[assignment] - with patch.object(second_engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"])): + with ( + patch.object( + second_engine, + "_terminate_services_from_state", + return_value=set(), + ) as terminate_mock, + patch.object( + second_engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"]) + ), + ): second_code = second_engine.dispatch(parse_route(["--plan", "feature-a", "--batch"], env={})) self.assertEqual(second_code, 0) + self.assertEqual(terminate_mock.call_count, 1) second_state = second_engine.state_repository.load_latest(mode="trees", strict_mode_match=True) self.assertIsNotNone(second_state) assert second_state is not None @@ -242,7 +254,9 @@ def test_project_root_change_forces_fresh_run(self) -> None: (tree / "frontend").mkdir(parents=True, exist_ok=True) engine = PythonEngineRuntime(self._config(repo, runtime), env={}) context = engine._discover_projects(mode="trees")[0] - wrong_root_context = ProjectContext(name=context.name, root=repo / "trees" / "feature-a" / "99", ports=context.ports) + wrong_root_context = ProjectContext( + name=context.name, root=repo / "trees" / "feature-a" / "99", ports=context.ports + ) metadata = startup_support.build_startup_identity_metadata( engine, runtime_mode="trees", @@ -287,7 +301,9 @@ def fake_result(selected_context: ProjectContext) -> ProjectStartupResult: ) engine._reconcile_state_truth = lambda _state: [] # type: ignore[assignment] - with patch.object(engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"])): + with patch.object( + engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"]) + ): code = engine.dispatch(parse_route(["--plan", "feature-a", "--batch"], env={})) self.assertEqual(code, 0) @@ -324,6 +340,9 @@ def test_resume_rejects_state_without_resumable_services(self) -> None: with ( patch.object(engine, "_try_load_existing_state", return_value=state), patch.object(engine, "_run_interactive_dashboard_loop", return_value=0) as loop_mock, + patch( + "envctl_engine.startup.resume_orchestrator.release_lifecycle_operation" + ) as release_lifecycle_operation_mock, patch("sys.stdin.isatty", return_value=True), patch("sys.stdout.isatty", return_value=True), patch.dict(os.environ, {"TERM": "xterm-256color"}, clear=False), @@ -333,6 +352,7 @@ def test_resume_rejects_state_without_resumable_services(self) -> None: self.assertEqual(code, 1) self.assertEqual(loop_mock.call_count, 0) + release_lifecycle_operation_mock.assert_not_called() self.assertIn("No active services to resume.", output.getvalue()) def test_dashboard_interactive_flag_enters_loop(self) -> None: @@ -441,4 +461,3 @@ def test_resume_batch_flag_skips_interactive_loop(self) -> None: self.assertEqual(code, 0) self.assertEqual(loop_mock.call_count, 0) - diff --git a/tests/python/runtime/test_engine_runtime_service_truth.py b/tests/python/runtime/test_engine_runtime_service_truth.py index a02e39b04..24d04dcfd 100644 --- a/tests/python/runtime/test_engine_runtime_service_truth.py +++ b/tests/python/runtime/test_engine_runtime_service_truth.py @@ -16,6 +16,7 @@ listener_pids_for_port, rebind_stale_service_pid, refresh_service_listener_pids, + service_listener_failure_class, service_listener_failure_detail, service_truth_fallback_enabled, service_truth_status, @@ -34,6 +35,7 @@ def __init__(self) -> None: self.listener_pids_result: list[int] = [] self.process_tree_listener_pids_result: list[int] = [] self.running_result = True + self.wait_for_port_calls: list[tuple[int, float]] = [] def wait_for_pid_port( self, @@ -49,6 +51,7 @@ def wait_for_pid_port( def wait_for_port(self, port: int, *, host: str = "127.0.0.1", timeout: float = 30.0) -> bool: # noqa: ARG002 _ = host + self.wait_for_port_calls.append((port, timeout)) return self.wait_for_port_result def find_pid_listener_port(self, pid: int, requested_port: int, *, max_delta: int) -> int | None: # noqa: ARG002 @@ -120,6 +123,16 @@ def test_service_listener_failure_detail_reports_startup_progress_without_callin self.assertIn("startup still in progress: INFO: Waiting for application startup.", detail) self.assertNotIn("log: INFO: Waiting for application startup.", detail) + def test_service_listener_failure_class_recognizes_missing_nested_executable(self) -> None: + detail = ( + "process 2304354 exited; log_path: /tmp/voice-runtime.txt; " + "log: scripts/envctl/start-voice-runtime.sh: line 20: exec: python: not found" + ) + + self.assertEqual(service_listener_failure_class(detail), "dependency_missing") + self.assertEqual(service_listener_failure_class("process 42 exited"), "process_exited") + self.assertEqual(service_listener_failure_class("listener timed out"), "listener_not_detected") + def test_wait_for_service_listener_uses_port_probe_fallback(self) -> None: events: list[tuple[str, dict[str, object]]] = [] runner = _RunnerStub() @@ -138,6 +151,25 @@ def test_wait_for_service_listener_uses_port_probe_fallback(self) -> None: self.assertTrue(ok) self.assertEqual(events[0][0], "service.bind.port_fallback") self.assertEqual(events[0][1]["service"], "Main Backend") + self.assertEqual(runner.wait_for_port_calls, [(8000, 2.0)]) + + def test_wait_for_service_listener_never_falls_back_after_launch_pid_exits(self) -> None: + runner = _RunnerStub() + runner.running_result = False + runner.wait_for_port_result = True + runner.process_tree_probe_supported = False + runtime = SimpleNamespace( + process_runner=runner, + config=SimpleNamespace(runtime_truth_mode="auto"), + _listener_probe_supported=True, + _service_listener_timeout=lambda: 300.0, + _emit=lambda *_args, **_kwargs: None, + ) + + ok = wait_for_service_listener(runtime, 2304354, 8251, service_name="voice-runtime") + + self.assertFalse(ok) + self.assertEqual(runner.wait_for_port_calls, []) def test_wait_for_service_listener_rejects_generic_port_when_process_tree_probe_is_available(self) -> None: events: list[tuple[str, dict[str, object]]] = [] @@ -236,8 +268,7 @@ def wait_for_pid_port( self.wait_calls += 1 if self.wait_calls == 2: self.log_path.write_text( - "INFO: Waiting for application startup.\n" - "INFO: Application startup complete.\n", + "INFO: Waiting for application startup.\nINFO: Application startup complete.\n", encoding="utf-8", ) return True @@ -309,9 +340,7 @@ def wait_for_pid_port( ) self.assertEqual(detected, 8000) - self.assertTrue( - any(event == "service.startup.progress.listener_accepted" for event, _payload in events) - ) + self.assertTrue(any(event == "service.startup.progress.listener_accepted" for event, _payload in events)) self.assertFalse(any(event == "service.startup.progress.timeout" for event, _payload in events)) self.assertFalse(any(event == "service.bind.actual.discovered" for event, _payload in events)) diff --git a/tests/python/runtime/test_engine_runtime_startup_support.py b/tests/python/runtime/test_engine_runtime_startup_support.py index ca8a73908..68f53ffcc 100644 --- a/tests/python/runtime/test_engine_runtime_startup_support.py +++ b/tests/python/runtime/test_engine_runtime_startup_support.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json +import tempfile import unittest from pathlib import Path from types import SimpleNamespace @@ -61,17 +63,15 @@ def _reuse_runtime( "backend": backend_enabled, "frontend": frontend_enabled, }.get(str(service_name).strip().lower(), False), - requirement_enabled_for_mode=lambda _mode, requirement_name: str(requirement_name).strip().lower() - in enabled_dependencies, + requirement_enabled_for_mode=lambda _mode, requirement_name: ( + str(requirement_name).strip().lower() in enabled_dependencies + ), ) return SimpleNamespace( env=dict(external_env or {}), config=config, _try_load_existing_state=lambda **_kwargs: state, - _project_name_from_service=lambda name: str(name) - .replace(" Backend", "") - .replace(" Frontend", "") - .strip(), + _project_name_from_service=lambda name: str(name).replace(" Backend", "").replace(" Frontend", "").strip(), ) def test_effective_start_mode_switches_main_to_trees_for_setup_worktree(self) -> None: @@ -144,6 +144,61 @@ def test_run_reuse_accepts_exact_project_roots_and_startup_identity_match(self) self.assertEqual(decision.reason, "exact_match") self.assertIs(decision.candidate_state, state) + def test_run_reuse_forwards_selected_project_names_to_state_lookup(self) -> None: + context = ProjectContext(name="feature-a", root=Path("/repo/trees/feature-a"), ports={}) + state = RunState( + run_id="run-a", + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/repo/trees/feature-a/backend", + ) + }, + metadata={"project_roots": {"feature-a": str(context.root)}}, + ) + runtime = self._reuse_runtime(state=state) + calls: list[dict[str, object]] = [] + runtime._try_load_existing_state = lambda **kwargs: calls.append(kwargs) or state + + startup_support.evaluate_run_reuse( + runtime, + runtime_mode="trees", + route=Route(command="start", mode="trees"), + contexts=[context], + ) + + self.assertEqual( + calls, + [{"mode": "trees", "strict_mode_match": True, "project_names": ["feature-a"]}], + ) + + def test_explicit_no_resume_keeps_matching_candidate_for_safe_replacement(self) -> None: + context = ProjectContext(name="feature-a", root=Path("/repo/trees/feature-a"), ports={}) + state = RunState( + run_id="run-a", + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/repo/trees/feature-a/backend", + ) + }, + metadata={"project_roots": {"feature-a": str(context.root)}}, + ) + decision = startup_support.evaluate_run_reuse( + self._reuse_runtime(state=state), + runtime_mode="trees", + route=Route(command="start", mode="trees", flags={"no_resume": True}), + contexts=[context], + ) + + self.assertEqual(decision.decision_kind, "fresh_run") + self.assertEqual(decision.reason, "explicit_no_resume") + self.assertIs(decision.candidate_state, state) + def test_run_reuse_rejects_project_root_mismatch_even_when_names_match(self) -> None: context = ProjectContext(name="feature-a-1", root=Path("/repo/trees/feature-a/1"), ports={}) route = Route(command="plan", mode="trees", raw_args=[], passthrough_args=[], projects=["feature-a"], flags={}) @@ -319,6 +374,41 @@ def test_run_reuse_rejects_startup_fingerprint_mismatch(self) -> None: self.assertEqual(decision.decision_kind, "fresh_run") self.assertEqual(decision.reason, "startup_fingerprint_mismatch") + def test_explicit_no_resume_retains_candidate_for_safe_replacement(self) -> None: + context = ProjectContext(name="Main", root=Path("/repo"), ports={}) + route = Route( + command="start", + mode="main", + raw_args=["start", "--no-resume"], + passthrough_args=[], + projects=[], + flags={"no_resume": True}, + ) + state = RunState( + run_id="run-existing", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + ) + }, + metadata={"project_roots": {"Main": "/repo"}}, + ) + runtime = self._reuse_runtime(state=state) + + decision = startup_support.evaluate_run_reuse( + runtime, + runtime_mode="main", + route=route, + contexts=[context], + ) + + self.assertEqual(decision.decision_kind, "fresh_run") + self.assertEqual(decision.reason, "explicit_no_resume") + self.assertIs(decision.candidate_state, state) + def test_run_reuse_rejects_dependency_external_mode_mismatch(self) -> None: context = ProjectContext(name="Main", root=Path("/repo"), ports={}) route = Route(command="start", mode="main", raw_args=[], passthrough_args=[], projects=[], flags={}) @@ -571,6 +661,236 @@ def test_discover_projects_and_reserve_project_ports(self) -> None: reserve_project_ports(runtime, context) self.assertEqual(events[-1][0], "port.reserved") + def test_reserving_selected_project_reclaims_only_its_candidate_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_dir = Path(tmpdir) + foreign_payload = json.dumps( + { + "owner": "Other:backend", + "session": "historical-other", + "pid": 99999999, + "created_at": 0, + }, + sort_keys=True, + ) + selected_payload = json.dumps( + { + "owner": "Selected:backend", + "session": "historical-selected", + "pid": 99999998, + "created_at": 0, + }, + sort_keys=True, + ) + (lock_dir / "18000.lock").write_text(foreign_payload, encoding="utf-8") + (lock_dir / "18001.lock").write_text(selected_payload, encoding="utf-8") + planner = PortPlanner( + lock_dir=lock_dir, + session_id="selected-start", + availability_checker=lambda _port: True, + ) + runtime = SimpleNamespace( + port_planner=planner, + _emit=lambda *_args, **_kwargs: None, + _lock_inventory=lambda: [], + ) + context = ProjectContext( + name="Selected", + root=Path("/repo/trees/selected"), + ports={ + "backend": PortPlan( + project="Selected", + requested=18001, + assigned=18001, + final=18001, + source="fixed", + ) + }, + ) + + reserve_project_ports(runtime, context) + + self.assertEqual((lock_dir / "18000.lock").read_text(encoding="utf-8"), foreign_payload) + selected_lock = json.loads((lock_dir / "18001.lock").read_text(encoding="utf-8")) + self.assertEqual(selected_lock["owner"], "Selected:backend") + self.assertEqual(selected_lock["session"], "selected-start") + + def test_no_infra_reserves_no_application_or_dependency_ports(self) -> None: + reserve_calls: list[tuple[int, str]] = [] + events: list[tuple[str, dict[str, object]]] = [] + plans = { + name: PortPlan(project="Main", requested=port, assigned=port, final=port, source="fixed") + for name, port in { + "backend": 8000, + "frontend": 9000, + "db": 5432, + "redis": 6379, + "n8n": 5678, + "supabase_api": 54321, + "worker": 9100, + }.items() + } + runtime = SimpleNamespace( + port_planner=SimpleNamespace( + reserve_next=lambda assigned, owner: reserve_calls.append((assigned, owner)) or assigned, + max_port=65000, + availability_mode="lock", + update_final_port=lambda plan, reserved, source: None, + ), + _emit=lambda event, **payload: events.append((event, payload)), + _lock_inventory=lambda: [], + _requirement_enabled=lambda *_args, **_kwargs: self.fail("disabled dependencies need no lookup"), + ) + route = Route( + command="start", + mode="main", + flags={"launch_backend": False, "launch_frontend": False, "launch_dependencies": False}, + ) + + reserve_project_ports( + runtime, + ProjectContext(name="Main", root=Path("/repo"), ports=plans), + route=route, + ) + + self.assertEqual(reserve_calls, []) + self.assertEqual( + [payload["service"] for event, payload in events if event == "port.reservation.skipped"], + list(plans), + ) + + def test_disabled_requirements_do_not_reserve_dependency_ports(self) -> None: + reserve_calls: list[tuple[int, str]] = [] + plans = { + "backend": PortPlan("Main", 8000, 8000, 8000, "fixed"), + "db": PortPlan("Main", 5432, 5432, 5432, "fixed"), + "redis": PortPlan("Main", 6379, 6379, 6379, "fixed"), + "n8n": PortPlan("Main", 5678, 5678, 5678, "fixed"), + "supabase_api": PortPlan("Main", 54321, 54321, 54321, "fixed"), + } + runtime = SimpleNamespace( + port_planner=SimpleNamespace( + reserve_next=lambda assigned, owner: reserve_calls.append((assigned, owner)) or assigned, + max_port=65000, + availability_mode="lock", + update_final_port=lambda plan, reserved, source: None, + ), + _emit=lambda *_args, **_kwargs: None, + _lock_inventory=lambda: [], + _requirement_enabled=lambda *_args, **_kwargs: False, + ) + + reserve_project_ports( + runtime, + ProjectContext(name="Main", root=Path("/repo"), ports=plans), + route=Route(command="start", mode="main", flags={"runtime_scope": "entire-system"}), + ) + + self.assertEqual(reserve_calls, [(8000, "Main:backend")]) + + def test_setup_worktree_reserves_dependencies_for_effective_trees_mode(self) -> None: + reserve_calls: list[tuple[int, str]] = [] + enabled_modes: list[str] = [] + runtime = SimpleNamespace( + port_planner=SimpleNamespace( + reserve_next=lambda assigned, owner: reserve_calls.append( + (assigned, owner) + ) + or assigned, + max_port=65000, + availability_mode="lock", + update_final_port=lambda plan, reserved, source: None, + ), + _emit=lambda *_args, **_kwargs: None, + _lock_inventory=lambda: [], + _setup_worktree_requested=lambda _route: True, + _requirement_enabled=lambda _dependency, *, mode, route: ( + enabled_modes.append(mode) or mode == "trees" + ), + ) + route = Route( + command="start", + mode="main", + flags={"setup_worktree": True, "runtime_scope": "entire-system"}, + ) + + reserve_project_ports( + runtime, + ProjectContext( + name="FeatureA", + root=Path("/repo/trees/FeatureA"), + ports={"db": PortPlan("FeatureA", 5432, 5432, 5432, "fixed")}, + ), + route=route, + ) + + self.assertEqual(enabled_modes, ["trees"]) + self.assertEqual(reserve_calls, [(5432, "FeatureA:db")]) + + def test_port_reservations_match_the_selected_runtime_surface(self) -> None: + voice = SimpleNamespace( + name="voice", + depends_on=(), + enabled_for_project_root=lambda _mode, _root: True, + ) + cases = ( + ("frontend only", {"runtime_scope": "frontend"}, {"frontend", "db"}), + ("backend only", {"runtime_scope": "backend"}, {"backend", "db"}), + ("dependencies only", {"runtime_scope": "dependencies"}, {"db"}), + ( + "no infra", + {"launch_backend": False, "launch_frontend": False, "launch_dependencies": False}, + set(), + ), + ("selected additional service", {"services": ["Main Voice"]}, {"voice", "db"}), + ) + + for label, flags, expected_services in cases: + with self.subTest(label): + reserve_calls: list[tuple[int, str]] = [] + plans = { + name: PortPlan("Main", port, port, port, "fixed") + for name, port in { + "backend": 8000, + "frontend": 9000, + "voice": 9100, + "db": 5432, + "redis": 6379, + "n8n": 5678, + "supabase_api": 54321, + }.items() + } + runtime = SimpleNamespace( + config=SimpleNamespace( + additional_services=(voice,), + all_app_service_names_for_mode=lambda _mode, project_root=None: ( + "backend", + "frontend", + "voice", + ), + ), + port_planner=SimpleNamespace( + reserve_next=lambda assigned, owner: reserve_calls.append((assigned, owner)) or assigned, + max_port=65000, + availability_mode="lock", + update_final_port=lambda plan, reserved, source: None, + ), + _emit=lambda *_args, **_kwargs: None, + _lock_inventory=lambda: [], + _requirement_enabled=lambda requirement, **_kwargs: requirement == "postgres", + ) + + reserve_project_ports( + runtime, + ProjectContext(name="Main", root=Path("/repo"), ports=plans), + route=Route(command="start", mode="main", flags=flags), + ) + + self.assertEqual( + {owner.removeprefix("Main:") for _port, owner in reserve_calls}, + expected_services, + ) + def test_reserve_project_ports_releases_matching_restart_lock_before_reuse(self) -> None: events: list[tuple[str, dict[str, object]]] = [] released: list[tuple[int, str | None]] = [] @@ -627,7 +947,9 @@ def update_final_port(plan: PortPlan, reserved: int, source: str) -> None: ) context = ProjectContext(name="Main", root=Path("/repo"), ports={"backend": port_plan}) - reserve_project_ports(runtime, context, route=Route(command="start", mode="main", flags={"interactive_command": True})) + reserve_project_ports( + runtime, context, route=Route(command="start", mode="main", flags={"interactive_command": True}) + ) rebound_payloads = [payload for event, payload in events if event == "port.rebound"] self.assertEqual(len(rebound_payloads), 1) diff --git a/tests/python/runtime/test_engine_runtime_state_lookup.py b/tests/python/runtime/test_engine_runtime_state_lookup.py index 4e3eba439..3f9b5c2a1 100644 --- a/tests/python/runtime/test_engine_runtime_state_lookup.py +++ b/tests/python/runtime/test_engine_runtime_state_lookup.py @@ -18,19 +18,29 @@ class EngineRuntimeStateLookupTests(unittest.TestCase): def test_try_load_existing_state_emits_fingerprint_after_reload(self) -> None: state = RunState(run_id="run-1", mode="main") events: list[tuple[str, dict[str, object]]] = [] + load_calls: list[dict[str, object]] = [] runtime = SimpleNamespace( - state_repository=SimpleNamespace(load_latest=lambda **kwargs: state), + state_repository=SimpleNamespace(load_latest=lambda **kwargs: load_calls.append(kwargs) or state), _emit=lambda event, **payload: events.append((event, payload)), _state_fingerprint=lambda loaded: f"fp:{loaded.run_id}", ) - loaded = try_load_existing_state(runtime, mode="main", strict_mode_match=True) + loaded = try_load_existing_state( + runtime, + mode="main", + strict_mode_match=True, + project_names=["Main"], + ) self.assertIs(loaded, state) self.assertEqual( events, [("state.fingerprint.after_reload", {"run_id": "run-1", "state_fingerprint": "fp:run-1"})], ) + self.assertEqual( + load_calls, + [{"mode": "main", "strict_mode_match": True, "project_names": ["Main"]}], + ) def test_state_matches_scope_accepts_missing_scope_and_matches_exact_scope(self) -> None: runtime = SimpleNamespace(config=SimpleNamespace(runtime_scope_id="repo-123")) diff --git a/tests/python/runtime/test_engine_runtime_state_truth.py b/tests/python/runtime/test_engine_runtime_state_truth.py index b4f6e3d72..0af60b107 100644 --- a/tests/python/runtime/test_engine_runtime_state_truth.py +++ b/tests/python/runtime/test_engine_runtime_state_truth.py @@ -18,10 +18,25 @@ requirement_truth_issues, state_fingerprint, ) +from envctl_engine.runtime.requirement_reconcile_truth import requirement_truth_identity # noqa: E402 from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord # noqa: E402 class EngineRuntimeStateTruthTests(unittest.TestCase): + def test_collision_requirement_truth_uses_authoritative_project_and_root(self) -> None: + requirements = RequirementsResult(project="Main", redis={"enabled": True, "final": 6381}) + state = RunState( + run_id="run-collision", + mode="main", + requirements={"Main Restart Collision": requirements}, + metadata={"project_roots": {"Main": "/repo/main"}}, + ) + + project, root = requirement_truth_identity(state, "Main Restart Collision", requirements) + + self.assertEqual(project, "Main") + self.assertEqual(root, Path("/repo/main")) + def test_state_fingerprint_changes_when_state_changes(self) -> None: first = RunState(run_id="run-1", mode="main", services={}) second = RunState( diff --git a/tests/python/runtime/test_help_topics.py b/tests/python/runtime/test_help_topics.py index b84c8ebad..97dc17a99 100644 --- a/tests/python/runtime/test_help_topics.py +++ b/tests/python/runtime/test_help_topics.py @@ -29,6 +29,15 @@ def test_pr_preview_controller_help_topic_is_available(self) -> None: self.assertIn("envctl pr-preview-controller - run the GitHub PR-label preview controller", text) self.assertIn("--command ", text) + def test_stop_help_distinguishes_detach_from_managed_docker_removal(self) -> None: + stop_text = render_command_help(COMMAND_HELP_TOPICS["stop"]) + stop_all_text = render_command_help(COMMAND_HELP_TOPICS["stop-all"]) + + self.assertIn("managed Docker stacks and storage remain reusable", stop_text) + self.assertIn("stop-all --remove-volumes", stop_text) + self.assertIn("releases dependency records and port locks", stop_all_text) + self.assertIn("remove supported managed containers and volumes", stop_all_text) + def test_help_text_for_route_returns_none_for_general_help(self) -> None: self.assertIsNone(help_text_for_route(parse_route(["--help"], env={}))) diff --git a/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py b/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py index d28f7f304..69c412d74 100644 --- a/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py +++ b/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py @@ -1,17 +1,26 @@ from __future__ import annotations +import json +import tempfile +import unittest from contextlib import contextmanager, redirect_stdout from io import StringIO from pathlib import Path from types import SimpleNamespace -import unittest from unittest.mock import patch REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.lifecycle_cleanup_orchestrator import LifecycleCleanupOrchestrator +from envctl_engine.runtime.lifecycle_service_termination import ( + terminate_service_record, + terminate_services_from_state, +) +from envctl_engine.shared.ports import PortPlanner from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord +from envctl_engine.state.repository import RuntimeStateRepository +from envctl_engine.state.run_index import StateSelector from envctl_engine.ui.spinner_service import SpinnerPolicy from envctl_engine.ui.target_selector import TargetSelection @@ -19,16 +28,43 @@ class _StateRepoStub: def __init__(self) -> None: self.saved_states: list[RunState] = [] + self.saved_authoritative_project_names: list[list[str] | None] = [] self.purge_calls: list[bool] = [] + self.active_states: list[RunState] = [] + self.deactivated_run_ids: list[str] = [] - def save_selected_stop_state(self, *, state, emit, runtime_map_builder): # noqa: ANN001 + def save_selected_stop_state( # noqa: ANN001 + self, + *, + state, + emit, + runtime_map_builder, + authoritative_project_names=None, + ): _ = emit, runtime_map_builder self.saved_states.append(state) + self.saved_authoritative_project_names.append(authoritative_project_names) + self.active_states = [ + state if active_state.run_id == state.run_id else active_state for active_state in self.active_states + ] return {} def purge(self, *, aggressive: bool = False) -> None: self.purge_calls.append(aggressive) + def load_all(self, *, mode: str | None = None) -> list[RunState]: + return [state for state in self.active_states if mode is None or state.mode == mode] + + def deactivate_runs(self, run_ids: list[str]) -> bool: + selected = set(run_ids) + removed = [state for state in self.active_states if state.run_id in selected] + self.active_states = [state for state in self.active_states if state.run_id not in selected] + self.deactivated_run_ids.extend(state.run_id for state in removed) + return bool(removed) + + def has_active_runs(self) -> bool: + return bool(self.active_states) + class _RuntimeStub: def __init__(self) -> None: @@ -52,7 +88,7 @@ def _state_lookup_strict_mode_match(_route): # noqa: ANN001 return False def _terminate_services_from_state(self, *args, **kwargs): # noqa: ANN001, ARG002 - return None + return set() def _release_port_session(self) -> None: return None @@ -106,6 +142,149 @@ def _terminate_services_from_state(self, *args, **kwargs): # noqa: ANN001, ARG0 class LifecycleCleanupSpinnerIntegrationTests(unittest.TestCase): + def test_full_cleanup_retains_dependency_state_after_container_cleanup_failure(self) -> None: + for command, aggressive in (("stop-all", False), ("blast-all", True)): + with self.subTest(command=command): + runtime = _RuntimeStub() + state = RunState( + run_id="run-1", + mode="main", + requirements={ + "Main": RequirementsResult( + project="Main", + redis={ + "id": "redis", + "enabled": True, + "container_name": "envctl-redis-main", + }, + ) + }, + ) + runtime.state_repository.active_states = [state] + released_requirements: list[str] = [] + runtime._release_requirement_ports = ( # type: ignore[attr-defined] + lambda requirements: released_requirements.append(requirements.project) + ) + orchestrator = LifecycleCleanupOrchestrator(runtime) + orchestrator.blast_all_ecosystem_enabled = lambda: False # type: ignore[method-assign] + orchestrator.blast_all_purge_legacy_state_artifacts = lambda: None # type: ignore[method-assign] + output = StringIO() + + with ( + patch( + "envctl_engine.runtime.lifecycle_cleanup_orchestrator.stop_requirement_component_containers", + side_effect=RuntimeError("Docker daemon is unavailable"), + ), + redirect_stdout(output), + ): + code = orchestrator.execute(Route(command=command, mode="main")) + + self.assertEqual(code, 1) + self.assertEqual(runtime.state_repository.purge_calls, []) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(runtime.state_repository.active_states, [state]) + self.assertEqual(runtime.state_repository.saved_states, [state]) + self.assertEqual(set(state.requirements), {"Main"}) + self.assertEqual(released_requirements, []) + self.assertIn("could not stop redis container", output.getvalue()) + warnings = [ + event + for event in runtime.events + if event.get("event") == "cleanup.dependency_container.warning" + ] + self.assertEqual( + warnings, + [ + { + "event": "cleanup.dependency_container.warning", + "component": "redis", + "detail": "Docker daemon is unavailable", + } + ], + ) + incomplete = [event for event in runtime.events if event.get("event") == "cleanup.incomplete"] + self.assertEqual(incomplete[-1].get("failed_services"), {}) + self.assertEqual(incomplete[-1].get("failed_requirements"), {"run-1": ["Main"]}) + + retry_code = orchestrator.execute(Route(command=command, mode="main")) + + self.assertEqual(retry_code, 0) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-1"]) + self.assertEqual(runtime.state_repository.active_states, []) + self.assertEqual(released_requirements, ["Main"]) + self.assertEqual(runtime.state_repository.purge_calls, [aggressive]) + + def test_targeted_runtime_cleanup_remains_fail_closed(self) -> None: + runtime = _RuntimeStub() + runtime._try_load_existing_state = lambda *args, **kwargs: RunState( # type: ignore[method-assign] + run_id="run-1", + mode="main", + requirements={ + "Main": RequirementsResult( + project="Main", + redis={ + "id": "redis", + "enabled": True, + "container_name": "envctl-redis-main", + }, + ) + }, + ) + orchestrator = LifecycleCleanupOrchestrator(runtime) + + with ( + patch( + "envctl_engine.runtime.lifecycle_cleanup_orchestrator.stop_requirement_component_containers", + side_effect=RuntimeError("Docker daemon is unavailable"), + ), + self.assertRaisesRegex(RuntimeError, "Docker daemon is unavailable"), + ): + orchestrator.clear_runtime_state(command="stop", aggressive=False) + + self.assertEqual(runtime.state_repository.purge_calls, []) + + def test_full_cleanup_retains_state_when_requirement_port_release_fails(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-port-release", + mode="main", + requirements={ + "Main": RequirementsResult( + project="Main", + redis={"id": "redis", "enabled": True, "external": True, "final": 6379}, + ) + }, + ) + runtime.state_repository.active_states = [state] + release_attempts = 0 + + def release_ports(_requirements: RequirementsResult) -> None: + nonlocal release_attempts + release_attempts += 1 + if release_attempts == 1: + raise OSError("lock unlink failed") + + runtime._release_requirement_ports = release_ports # type: ignore[attr-defined] + orchestrator = LifecycleCleanupOrchestrator(runtime) + + first_code = orchestrator.execute(Route(command="stop-all", mode="main")) + + self.assertEqual(first_code, 1) + self.assertEqual(runtime.state_repository.active_states, [state]) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(runtime.state_repository.purge_calls, []) + self.assertEqual(set(state.requirements), {"Main"}) + warning = next(event for event in runtime.events if event.get("event") == "cleanup.requirement_ports.warning") + self.assertEqual(warning.get("detail"), "lock unlink failed") + + retry_code = orchestrator.execute(Route(command="stop-all", mode="main")) + + self.assertEqual(retry_code, 0) + self.assertEqual(release_attempts, 2) + self.assertEqual(runtime.state_repository.active_states, []) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-port-release"]) + self.assertEqual(runtime.state_repository.purge_calls, [False]) + def test_stop_all_emits_spinner_policy_and_lifecycle(self) -> None: runtime = _RuntimeStub() orchestrator = LifecycleCleanupOrchestrator(runtime) @@ -165,6 +344,627 @@ def test_stop_all_tolerates_runtime_without_port_session_hook(self) -> None: self.assertEqual(runtime.state_repository.purge_calls, [False]) self.assertTrue(any(item.get("event") == "cleanup.stop_all" for item in runtime.events)) + def test_stop_all_releases_every_scoped_port_lock_after_last_run_is_removed(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime = _RuntimeStub() + scoped_planner = PortPlanner( + lock_dir=str(Path(tmpdir) / "scope-a"), + availability_checker=lambda _port: True, + ) + foreign_planner = PortPlanner( + lock_dir=str(Path(tmpdir) / "scope-b"), + availability_checker=lambda _port: True, + ) + scoped_planner.reserve_next(8100, owner="old-startup") + foreign_planner.reserve_next(8200, owner="other-repository") + runtime.port_planner = scoped_planner + + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(code, 0) + self.assertEqual(list(scoped_planner.lock_dir.glob("*.lock")), []) + self.assertEqual( + [path.name for path in foreign_planner.lock_dir.glob("*.lock")], + ["8200.lock"], + ) + self.assertEqual(runtime.state_repository.purge_calls, [False]) + + def test_stop_all_purge_failure_preserves_scoped_port_locks(self) -> None: + runtime = _RuntimeStub() + releases: list[str] = [] + runtime.port_planner = SimpleNamespace(release_all=lambda: releases.append("all")) + + with ( + patch.object(runtime.state_repository, "purge", side_effect=OSError("registry unavailable")), + self.assertRaisesRegex(OSError, "registry unavailable"), + ): + LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(releases, []) + + def test_stop_all_terminates_every_independent_active_run_once(self) -> None: + runtime = _RuntimeStub() + runtime.state_repository.active_states = [ + RunState(run_id="run-a", mode="trees"), + RunState(run_id="run-b", mode="trees"), + RunState(run_id="run-a", mode="trees"), + ] + terminated: list[str] = [] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: terminated.append(state.run_id) or set() + ) + + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="trees")) + + self.assertEqual(code, 0) + self.assertEqual(terminated, ["run-a", "run-b"]) + self.assertEqual(runtime.state_repository.purge_calls, [False]) + + def test_plain_stop_all_terminates_active_runs_across_modes(self) -> None: + runtime = _RuntimeStub() + runtime.state_repository.active_states = [ + RunState(run_id="run-main", mode="main"), + RunState(run_id="run-tree", mode="trees"), + ] + terminated: list[str] = [] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: terminated.append(state.run_id) or set() + ) + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route(command="stop-all", mode="main") + ) + + self.assertEqual(code, 0) + self.assertEqual(terminated, ["run-main", "run-tree"]) + self.assertEqual(runtime.state_repository.active_states, []) + + def test_stop_all_trees_preserves_main_registry_alias_and_port_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime_dir = root / "runtime" + repository = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + for run_id, mode, project in ( + ("run-main", "main", "Main"), + ("run-alpha", "trees", "FeatureA"), + ("run-beta", "trees", "FeatureB"), + ): + repository.save_resume_state( + state=RunState( + run_id=run_id, + mode=mode, + services={ + f"{project} Backend": ServiceRecord( + name=f"{project} Backend", + type="backend", + cwd=str(root / project / "backend"), + project=project, + pid=100 + len(project), + ) + }, + metadata={ + "project_names": [project], + "project_roots": {project: str(root / project)}, + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, + ) + + planner = PortPlanner( + lock_dir=str(repository.runtime_root / "locks"), + session_id="main-owner", + availability_checker=lambda _port: True, + ) + main_port = planner.reserve_next(8111, owner="Main:backend") + runtime = _RuntimeStub() + runtime.state_repository = repository # type: ignore[assignment] + runtime._state_lookup_strict_mode_match = lambda _route: True # type: ignore[method-assign] + runtime.port_planner = planner + terminated: list[str] = [] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: terminated.append(state.run_id) or set() + ) + + with patch.object(repository, "purge", wraps=repository.purge) as purge: + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="trees")) + + self.assertEqual(code, 0) + self.assertEqual(terminated, ["run-beta", "run-alpha"]) + purge.assert_not_called() + remaining = repository.load_all() + self.assertEqual([state.run_id for state in remaining], ["run-main"]) + self.assertEqual(repository.load_latest(mode="main", strict_mode_match=True).run_id, "run-main") + current = json.loads(repository.run_state_path().read_text(encoding="utf-8")) + self.assertEqual((current["run_id"], current["mode"]), ("run-main", "main")) + self.assertTrue((planner.lock_dir / f"{main_port}.lock").exists()) + + def test_mode_scoped_stop_all_ignores_tampered_other_mode_and_preserves_it(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repository = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + for run_id, mode, project, pid in ( + ("run-valid", "main", "Main", 101), + ("run-tampered", "trees", "FeatureA", 202), + ): + service_name = f"{project} Backend" + repository.save_resume_state( + state=RunState( + run_id=run_id, + mode=mode, + services={ + service_name: ServiceRecord( + name=service_name, + type="backend", + cwd=f"/tmp/{project}/backend", + project=project, + pid=pid, + ) + }, + metadata={"project_names": [project]}, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, + ) + + candidates = repository.run_index.candidates(StateSelector(mode=None, project_names=())) + tampered = next(candidate for candidate in candidates if candidate.run_id == "run-tampered") + tampered.state_path.write_text('{"broken":', encoding="utf-8") + + runtime = _RuntimeStub() + runtime.state_repository = repository # type: ignore[assignment] + runtime._state_lookup_strict_mode_match = lambda _route: True # type: ignore[method-assign] + termination_calls: list[str] = [] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: termination_calls.append(state.run_id) or set() + ) + + with patch.object(repository, "purge", wraps=repository.purge) as purge: + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(code, 0) + self.assertEqual(termination_calls, ["run-valid"]) + purge.assert_not_called() + remaining = repository.run_index.candidates(StateSelector(mode=None, project_names=())) + self.assertEqual({candidate.run_id for candidate in remaining}, {"run-tampered"}) + self.assertTrue(all(candidate.state_path.exists() for candidate in remaining)) + + def test_targeted_stop_deactivates_no_infra_state_with_only_disabled_requirements(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-no-infra", + mode="trees", + services={}, + requirements={"FeatureA": RequirementsResult(project="FeatureA")}, + metadata={ + "project_names": ["FeatureA"], + "project_roots": {"FeatureA": "/tmp/FeatureA"}, + }, + ) + runtime.state_repository.active_states = [state] + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"batch": True, "runtime_scope": "entire-system"}, + ) + ) + + self.assertEqual(code, 0) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-no-infra"]) + self.assertEqual(runtime.state_repository.active_states, []) + + def test_targeted_stop_removes_only_selected_project_from_shared_no_infra_state(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-no-infra-ab", + mode="trees", + services={}, + requirements={project: RequirementsResult(project=project) for project in ("FeatureA", "FeatureB")}, + metadata={ + "project_names": ["FeatureA", "FeatureB"], + "project_roots": { + "FeatureA": "/tmp/FeatureA", + "FeatureB": "/tmp/FeatureB", + }, + }, + ) + runtime.state_repository.active_states = [state] + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"batch": True, "runtime_scope": "entire-system"}, + ) + ) + + self.assertEqual(code, 0) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(len(runtime.state_repository.saved_states), 1) + saved = runtime.state_repository.saved_states[0] + self.assertEqual(set(saved.requirements), {"FeatureB"}) + self.assertEqual(saved.metadata["project_names"], ["FeatureB"]) + self.assertEqual(saved.metadata["project_roots"], {"FeatureB": "/tmp/FeatureB"}) + self.assertEqual(runtime.state_repository.saved_authoritative_project_names, [["FeatureB"]]) + self.assertEqual(runtime.state_repository.active_states, [saved]) + + def test_targeted_service_stop_preserves_unselected_owner_and_shared_main_requirements(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-services-ab", + mode="trees", + services={ + f"{project} Backend": ServiceRecord( + name=f"{project} Backend", + type="backend", + cwd=f"/tmp/{project}", + project=project, + pid=pid, + ) + for project, pid in (("FeatureA", 101), ("FeatureB", 202)) + }, + requirements={ + "Main": RequirementsResult( + project="Main", + components={ + "postgres": { + "enabled": True, + "success": True, + "final": 5432, + } + }, + ) + }, + metadata={ + "project_names": ["FeatureA", "FeatureB"], + "project_roots": { + "FeatureA": "/tmp/FeatureA", + "FeatureB": "/tmp/FeatureB", + }, + "shared_dependencies": True, + }, + ) + runtime.state_repository.active_states = [state] + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"batch": True, "runtime_scope": "backend"}, + ) + ) + + self.assertEqual(code, 0) + saved = runtime.state_repository.saved_states[0] + self.assertEqual(set(saved.services), {"FeatureB Backend"}) + self.assertEqual(set(saved.requirements), {"Main"}) + self.assertTrue(saved.metadata["shared_dependencies"]) + self.assertEqual(saved.metadata["project_names"], ["FeatureB"]) + self.assertEqual(saved.metadata["project_roots"], {"FeatureB": "/tmp/FeatureB"}) + self.assertEqual(runtime.state_repository.saved_authoritative_project_names, [["FeatureB"]]) + + def test_targeted_stop_terminates_and_deactivates_every_shadowed_run_for_project(self) -> None: + runtime = _RuntimeStub() + release_snapshots: list[list[str]] = [] + runtime.port_planner = SimpleNamespace( + release_all=lambda: release_snapshots.append( + [state.run_id for state in runtime.state_repository.active_states] + ) + ) + runtime.state_repository.active_states = [ + RunState( + run_id=run_id, + mode="trees", + services={ + f"FeatureA Backend {run_id}": ServiceRecord( + name=f"FeatureA Backend {run_id}", + type="backend", + cwd=f"/tmp/{run_id}/backend", + project="FeatureA", + pid=pid, + ) + }, + metadata={"project_names": ["FeatureA"]}, + ) + for run_id, pid in (("run-new", 202), ("run-old", 101)) + ] + terminated: list[str] = [] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: terminated.append(state.run_id) or set() + ) + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"batch": True}, + ) + ) + + self.assertEqual(code, 0) + self.assertEqual(terminated, ["run-new", "run-old"]) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-new", "run-old"]) + self.assertEqual(runtime.state_repository.active_states, []) + self.assertEqual(runtime.state_repository.purge_calls, []) + self.assertEqual(release_snapshots, [[]]) + + def test_targeted_stop_failure_preserves_service_and_run_tracking(self) -> None: + runtime = _RuntimeStub() + runtime.port_planner = SimpleNamespace( + release_all=lambda: self.fail("incomplete termination must retain scoped locks") + ) + state = RunState( + run_id="run-a", + mode="trees", + services={ + "FeatureA Backend": ServiceRecord( + name="FeatureA Backend", + type="backend", + cwd="/tmp/FeatureA/backend", + project="FeatureA", + pid=101, + ) + }, + metadata={"project_names": ["FeatureA"]}, + ) + runtime.state_repository.active_states = [state] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda _state, *, selected_services, **_kwargs: set(selected_services or ()) + ) + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"batch": True}, + ) + ) + + self.assertEqual(code, 1) + self.assertEqual(set(state.services), {"FeatureA Backend"}) + self.assertEqual(runtime.state_repository.saved_states, [state]) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(runtime.state_repository.purge_calls, []) + + def test_targeted_stop_missing_termination_result_preserves_service_and_requirements(self) -> None: + runtime = _RuntimeStub() + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + pid=101, + ) + requirements = RequirementsResult(project="Main", redis={"enabled": True, "final": 6379}) + state = RunState( + run_id="run-main", + mode="main", + services={service.name: service}, + requirements={"Main": requirements}, + ) + runtime._try_load_existing_state = lambda *_args, **_kwargs: state # type: ignore[method-assign] + runtime._terminate_services_from_state = lambda *_args, **_kwargs: None # type: ignore[method-assign] + runtime._release_requirement_ports = lambda _requirements: self.fail( # type: ignore[attr-defined] + "unconfirmed service must retain requirements" + ) + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="main", + flags={"services": [service.name], "batch": True}, + ) + ) + + self.assertEqual(code, 1) + self.assertIs(state.services[service.name], service) + self.assertIs(state.requirements["Main"], requirements) + + def test_stop_all_missing_termination_result_retains_active_run(self) -> None: + runtime = _RuntimeStub() + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + pid=101, + ) + state = RunState(run_id="run-main", mode="main", services={service.name: service}) + runtime.state_repository.active_states = [state] + runtime._terminate_services_from_state = lambda *_args, **_kwargs: None # type: ignore[method-assign] + + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(code, 1) + self.assertEqual(runtime.state_repository.active_states, [state]) + self.assertEqual(set(state.services), {service.name}) + self.assertEqual(runtime.state_repository.saved_states, [state]) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(runtime.state_repository.purge_calls, []) + + def test_implicit_mode_targeted_stop_cleans_every_tree_shadow_with_duplicate_pid(self) -> None: + runtime = _RuntimeStub() + main_state = RunState( + run_id="run-main", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + pid=303, + ) + }, + metadata={"project_names": ["Main"]}, + ) + shadow_states = [ + RunState( + run_id=run_id, + mode="trees", + services={ + f"FeatureA Backend {run_id}": ServiceRecord( + name=f"FeatureA Backend {run_id}", + type="backend", + cwd=f"/tmp/{run_id}/backend", + project="FeatureA", + pid=404, + actual_port=8404, + ) + }, + metadata={"project_names": ["FeatureA"]}, + ) + for run_id in ("run-tree-new", "run-tree-old") + ] + runtime.state_repository.active_states = [main_state, *shadow_states] + runtime._try_load_existing_state = ( # type: ignore[method-assign] + lambda *, mode=None, strict_mode_match=False, project_names=None: ( + shadow_states[0] if mode == "main" and not strict_mode_match and project_names == ["FeatureA"] else None + ) + ) + live_pids = {404} + termination_calls: list[int] = [] + released_ports: list[int] = [] + runtime.port_planner = SimpleNamespace(release=lambda port: released_ports.append(port)) + runtime.process_runner = SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + pid_owns_port=lambda pid, port: pid in live_pids and pid == 404 and port == 8404, + terminate_process_group=lambda pid, **_kwargs: termination_calls.append(pid) or not live_pids.discard(pid), + ) + runtime._terminate_service_record = ( # type: ignore[attr-defined] + lambda service, *, aggressive, verify_ownership: terminate_service_record( + runtime, + service, + aggressive=aggressive, + verify_ownership=verify_ownership, + ) + ) + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, *, selected_services, aggressive, verify_ownership: terminate_services_from_state( + runtime, + state, + selected_services=selected_services, + aggressive=aggressive, + verify_ownership=verify_ownership, + ) + ) + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="main", + projects=["FeatureA"], + flags={"batch": True}, + ) + ) + + self.assertEqual(code, 0) + self.assertEqual(termination_calls, [404]) + self.assertEqual(released_ports, [8404, 8404]) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-tree-new", "run-tree-old"]) + self.assertEqual(runtime.state_repository.active_states, [main_state]) + + def test_stop_all_failure_preserves_failed_run_and_never_purges_registry(self) -> None: + runtime = _RuntimeStub() + failed_state = RunState( + run_id="run-failed", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/backend", + project="Main", + pid=101, + ), + "FeatureC Backend": ServiceRecord( + name="FeatureC Backend", + type="backend", + cwd="/tmp/FeatureC/backend", + project="FeatureC", + pid=102, + ), + }, + requirements={ + "shared-main": RequirementsResult(project="Main", db={"enabled": True, "final": 5432}), + "FeatureC": RequirementsResult(project="FeatureC", redis={"enabled": True, "final": 6379}), + }, + ) + successful_state = RunState( + run_id="run-success", + mode="trees", + services={ + "FeatureB Backend": ServiceRecord( + name="FeatureB Backend", + type="backend", + cwd="/tmp/FeatureB/backend", + project="FeatureB", + pid=202, + ) + }, + requirements={ + "FeatureB": RequirementsResult(project="FeatureB", db={"enabled": True, "final": 5433}), + }, + ) + runtime.state_repository.active_states = [failed_state, successful_state] + released_requirements: list[str] = [] + runtime._release_requirement_ports = ( # type: ignore[attr-defined] + lambda requirements: released_requirements.append(requirements.project) + ) + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: {"Main Backend"} if state.run_id == "run-failed" else set() + ) + + runtime._state_lookup_strict_mode_match = lambda _route: True # type: ignore[method-assign] + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(code, 1) + self.assertEqual(runtime.state_repository.saved_states, [failed_state]) + self.assertEqual(set(failed_state.services), {"Main Backend"}) + self.assertEqual(set(failed_state.requirements), {"shared-main"}) + self.assertEqual(set(released_requirements), {"FeatureC"}) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(runtime.state_repository.active_states, [failed_state, successful_state]) + self.assertEqual(runtime.state_repository.purge_calls, []) + self.assertTrue(any(event.get("event") == "cleanup.incomplete" for event in runtime.events)) + + runtime._terminate_services_from_state = lambda _state, **_kwargs: set() # type: ignore[method-assign] + retry_code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(retry_code, 0) + self.assertEqual(released_requirements.count("Main"), 1) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-failed"]) + self.assertEqual(runtime.state_repository.active_states, [successful_state]) + self.assertEqual(runtime.state_repository.purge_calls, []) + + trees_code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="trees")) + + self.assertEqual(trees_code, 0) + self.assertEqual(released_requirements.count("FeatureB"), 1) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-failed", "run-success"]) + self.assertEqual(runtime.state_repository.active_states, []) + self.assertEqual(runtime.state_repository.purge_calls, [False]) + def test_stop_selector_miss_does_not_start_spinner(self) -> None: runtime = _RuntimeStub() runtime._try_load_existing_state = lambda *args, **kwargs: RunState( # type: ignore[method-assign] @@ -276,7 +1076,7 @@ def test_stop_dependencies_scope_releases_requirements_without_terminating_servi runtime.port_planner = SimpleNamespace(release=lambda port: released.append(port)) runtime._terminate_services_from_state = lambda _state, **kwargs: terminated.append( # type: ignore[method-assign] kwargs.get("selected_services") - ) + ) or set() state = RunState( run_id="run-1", mode="main", @@ -300,6 +1100,184 @@ def test_stop_dependencies_scope_releases_requirements_without_terminating_servi self.assertEqual(state.requirements, {}) self.assertEqual(runtime.state_repository.saved_states[0].services.keys(), {"Main Backend"}) + def test_targeted_dependencies_scope_preserves_other_project_requirements_and_state(self) -> None: + runtime = _RuntimeStub() + released: list[str] = [] + alpha = RequirementsResult(project="Alpha", redis={"enabled": True, "final": 6379}) + beta = RequirementsResult(project="Beta", redis={"enabled": True, "final": 6389}) + state = RunState( + run_id="run-multi", + mode="trees", + services={ + "Beta Backend": ServiceRecord( + name="Beta Backend", + type="backend", + cwd="/beta", + project="Beta", + pid=2, + ) + }, + requirements={"Alpha": alpha, "Beta": beta}, + metadata={"project_names": ["Alpha", "Beta"]}, + ) + runtime.state_repository.active_states = [state] + runtime._try_load_existing_state = lambda *_args, **_kwargs: state # type: ignore[method-assign] + runtime._release_requirement_ports = ( # type: ignore[attr-defined] + lambda requirements: released.append(requirements.project) + ) + orchestrator = LifecycleCleanupOrchestrator(runtime) + + code = orchestrator.execute( + Route( + command="stop", + mode="trees", + projects=["Alpha"], + flags={"runtime_scope": "dependencies", "batch": True}, + ) + ) + + self.assertEqual(code, 0) + self.assertEqual(released, ["Alpha"]) + self.assertEqual(set(state.requirements), {"Beta"}) + self.assertIn("Beta Backend", state.services) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(runtime.state_repository.saved_states[-1].metadata["project_names"], ["Beta"]) + + def test_dependency_scope_normalizes_positional_and_service_selectors(self) -> None: + routes = ( + Route( + command="stop", + mode="trees", + passthrough_args=["Alpha"], + flags={"runtime_scope": "dependencies", "batch": True}, + ), + Route( + command="stop", + mode="trees", + flags={ + "runtime_scope": "dependencies", + "services": ["Alpha Backend"], + "batch": True, + }, + ), + ) + for route in routes: + with self.subTest(route=route): + runtime = _RuntimeStub() + state = RunState( + run_id="run-selector", + mode="trees", + services={ + "Alpha Backend": ServiceRecord( + name="Alpha Backend", type="backend", project="Alpha", cwd="/alpha", pid=1 + ), + "Beta Backend": ServiceRecord( + name="Beta Backend", type="backend", project="Beta", cwd="/beta", pid=2 + ), + }, + requirements={ + "alpha-alias": RequirementsResult(project="Alpha", redis={"enabled": True}), + "beta-alias": RequirementsResult(project="Beta", redis={"enabled": True}), + }, + metadata={"project_names": ["Alpha", "Beta"]}, + ) + runtime.state_repository.active_states = [state] + runtime._try_load_existing_state = lambda *_args, **_kwargs: state # type: ignore[method-assign] + released: list[str] = [] + runtime._release_requirement_ports = ( # type: ignore[attr-defined] + lambda requirements: released.append(requirements.project) + ) + + code = LifecycleCleanupOrchestrator(runtime).execute(route) + + self.assertEqual(code, 0) + self.assertEqual(released, ["Alpha"]) + self.assertEqual(set(state.requirements), {"beta-alias"}) + self.assertEqual(set(state.services), {"Alpha Backend", "Beta Backend"}) + + def test_unmatched_dependency_service_selector_fails_without_broadening_scope(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-selector-miss", + mode="trees", + services={ + "Alpha Backend": ServiceRecord( + name="Alpha Backend", type="backend", project="Alpha", cwd="/alpha", pid=1 + ) + }, + requirements={"Alpha": RequirementsResult(project="Alpha", redis={"enabled": True})}, + ) + runtime.state_repository.active_states = [state] + runtime._try_load_existing_state = lambda *_args, **_kwargs: state # type: ignore[method-assign] + runtime._release_requirement_ports = lambda _requirements: self.fail("selector miss released ports") # type: ignore[attr-defined] + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + flags={"runtime_scope": "dependencies", "services": ["Missing Worker"], "batch": True}, + ) + ) + + self.assertEqual(code, 1) + self.assertEqual(set(state.requirements), {"Alpha"}) + self.assertEqual(runtime.state_repository.saved_states, []) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + + def test_entire_system_selector_does_not_remove_sibling_project_dependencies(self) -> None: + routes = ( + Route( + command="stop", + mode="trees", + passthrough_args=["Alpha"], + flags={"runtime_scope": "entire-system", "batch": True}, + ), + Route( + command="stop", + mode="trees", + flags={ + "runtime_scope": "entire-system", + "services": ["Alpha Backend"], + "batch": True, + }, + ), + ) + for route in routes: + with self.subTest(route=route): + runtime = _RuntimeStub() + released: list[int] = [] + runtime.port_planner = SimpleNamespace(release=lambda port: released.append(port)) + state = RunState( + run_id="run-entire-selector", + mode="trees", + services={ + "Alpha Backend": ServiceRecord( + name="Alpha Backend", type="backend", project="Alpha", cwd="/alpha", pid=1 + ), + "Beta Backend": ServiceRecord( + name="Beta Backend", type="backend", project="Beta", cwd="/beta", pid=2 + ), + }, + requirements={ + "alpha-alias": RequirementsResult( + project="Alpha", redis={"id": "redis", "enabled": True, "final": 6379} + ), + "beta-alias": RequirementsResult( + project="Beta", redis={"id": "redis", "enabled": True, "final": 6389} + ), + }, + metadata={"project_names": ["Alpha", "Beta"]}, + ) + runtime.state_repository.active_states = [state] + runtime._try_load_existing_state = lambda *_args, **_kwargs: state # type: ignore[method-assign] + + code = LifecycleCleanupOrchestrator(runtime).execute(route) + + self.assertEqual(code, 0) + self.assertEqual(released, [6379]) + self.assertEqual(set(state.services), {"Beta Backend"}) + self.assertEqual(set(state.requirements), {"beta-alias"}) + def test_stop_selected_dependency_component_preserves_services_and_other_dependencies(self) -> None: runtime = _RuntimeStub() released: list[int] = [] @@ -307,7 +1285,7 @@ def test_stop_selected_dependency_component_preserves_services_and_other_depende runtime.port_planner = SimpleNamespace(release=lambda port: released.append(port)) runtime._terminate_services_from_state = lambda _state, **kwargs: terminated.append( # type: ignore[method-assign] kwargs.get("selected_services") - ) + ) or set() state = RunState( run_id="run-1", mode="main", @@ -352,7 +1330,7 @@ def test_stop_selected_services_can_leave_dependencies_running(self) -> None: runtime.port_planner = SimpleNamespace(release=lambda port: released.append(port)) runtime._terminate_services_from_state = lambda _state, **kwargs: terminated.append( # type: ignore[method-assign] kwargs.get("selected_services") - ) + ) or set() state = RunState( run_id="run-1", mode="main", @@ -394,7 +1372,7 @@ def test_interactive_stop_all_services_preserves_stopped_dashboard_state(self) - terminated: list[set[str] | None] = [] runtime._terminate_services_from_state = lambda _state, **kwargs: terminated.append( # type: ignore[method-assign] kwargs.get("selected_services") - ) + ) or set() state = RunState( run_id="run-1", mode="main", diff --git a/tests/python/runtime/test_lifecycle_dependency_stop.py b/tests/python/runtime/test_lifecycle_dependency_stop.py index 59d176d1e..ac2dcdb91 100644 --- a/tests/python/runtime/test_lifecycle_dependency_stop.py +++ b/tests/python/runtime/test_lifecycle_dependency_stop.py @@ -1,6 +1,9 @@ from __future__ import annotations import unittest +from pathlib import Path +import tempfile +from types import SimpleNamespace from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.lifecycle_dependency_stop import ( @@ -8,11 +11,112 @@ release_selected_dependency_components, requirements_have_enabled_components, select_dependency_components_for_stop, + stop_requirement_component_containers, ) +from envctl_engine.runtime.lifecycle_requirement_ports import requirement_component_port_owners +from envctl_engine.shared.ports import PortPlanner from envctl_engine.state.models import RequirementsResult, RunState class LifecycleDependencyStopTests(unittest.TestCase): + def test_entire_system_selects_all_enabled_dependencies_for_requested_project(self) -> None: + state = RunState( + run_id="run-entire", + mode="trees", + requirements={ + "Alpha": RequirementsResult(project="Alpha", redis={"enabled": True}, n8n={"enabled": True}), + "Beta": RequirementsResult(project="Beta", redis={"enabled": True}), + }, + ) + route = Route(command="stop", mode="trees", projects=["Alpha"], flags={"runtime_scope": "entire-system"}) + + self.assertEqual(select_dependency_components_for_stop(state, route), {"Alpha": {"redis", "n8n"}}) + + def test_entire_system_selects_every_storage_record_by_authoritative_project(self) -> None: + state = RunState( + run_id="run-collision", + mode="main", + requirements={ + "Main": RequirementsResult(project="Main", redis={"enabled": True}), + "Main Restart Collision": RequirementsResult(project="Main", redis={"enabled": True}), + "Other": RequirementsResult(project="Other", redis={"enabled": True}), + }, + ) + route = Route(command="stop", mode="main", projects=["Main"], flags={"runtime_scope": "entire-system"}) + + selected = select_dependency_components_for_stop(state, route) + + self.assertEqual(selected, {"Main": {"redis"}, "Main Restart Collision": {"redis"}}) + self.assertEqual( + requirement_component_port_owners(state.requirements["Main Restart Collision"], "redis"), + ("Main:redis", "Main:requirements"), + ) + + def test_stop_requirement_component_removes_direct_and_compose_containers(self) -> None: + calls: list[list[str]] = [] + + def run(command, **_kwargs): # noqa: ANN001 + argv = [str(part) for part in command] + calls.append(argv) + if argv[:3] == ["docker", "ps", "--all"]: + return SimpleNamespace(returncode=0, stdout="one\ntwo\n", stderr="") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + runtime = SimpleNamespace(process_runner=SimpleNamespace(run=run)) + stop_requirement_component_containers( + runtime, + {"id": "redis", "enabled": True, "container_name": "envctl-redis-alpha"}, + ) + stop_requirement_component_containers( + runtime, + { + "id": "supabase", + "enabled": True, + "container_name": "envctl-supabase-alpha-supabase-db-1", + }, + ) + + self.assertIn(["docker", "rm", "--force", "envctl-redis-alpha"], calls) + self.assertIn( + ["docker", "ps", "--all", "--quiet", "--filter", "label=com.docker.compose.project=envctl-supabase-alpha"], + calls, + ) + self.assertIn(["docker", "rm", "--force", "one", "two"], calls) + + def test_stop_native_supabase_falls_back_to_saved_container_name(self) -> None: + calls: list[list[str]] = [] + + def run(command, **_kwargs): # noqa: ANN001 + argv = [str(part) for part in command] + calls.append(argv) + if argv[:3] == ["docker", "ps", "--all"]: + return SimpleNamespace(returncode=0, stdout="", stderr="") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + stop_requirement_component_containers( + SimpleNamespace(process_runner=SimpleNamespace(run=run)), + { + "id": "supabase", + "enabled": True, + "container_name": "envctl-feature-supabase-db-1", + }, + ) + + self.assertEqual( + calls, + [ + [ + "docker", + "ps", + "--all", + "--quiet", + "--filter", + "label=com.docker.compose.project=envctl-feature", + ], + ["docker", "rm", "--force", "envctl-feature-supabase-db-1"], + ], + ) + def test_select_dependency_components_filters_unknown_projects_dependencies_and_disabled_components(self) -> None: state = RunState( run_id="run-1", @@ -56,17 +160,22 @@ def test_release_selected_dependency_components_releases_internal_ports_and_prun }, ) released: list[int] = [] + stopped: list[str] = [] release_selected_dependency_components( state, {"Main": {"postgres", "redis"}, "Aux": {"postgres"}}, - release_component_ports_fn=lambda component: release_requirement_component_ports( - component, - release_port_fn=released.append, + release_component_ports_fn=lambda _requirements, _dependency_id, component: ( + release_requirement_component_ports( + component, + port_planner=SimpleNamespace(release=released.append), + ) ), + stop_component_fn=lambda component: stopped.append(str(component.get("id"))), ) self.assertEqual(released, [5432, 15432, 6543]) + self.assertEqual(stopped, ["postgres", "postgres"]) self.assertEqual(state.requirements, {}) def test_release_selected_dependency_components_preserves_projects_with_enabled_components(self) -> None: @@ -86,9 +195,11 @@ def test_release_selected_dependency_components_preserves_projects_with_enabled_ release_selected_dependency_components( state, {"Main": {"redis"}}, - release_component_ports_fn=lambda component: release_requirement_component_ports( - component, - release_port_fn=released.append, + release_component_ports_fn=lambda _requirements, _dependency_id, component: ( + release_requirement_component_ports( + component, + port_planner=SimpleNamespace(release=released.append), + ) ), ) @@ -103,11 +214,53 @@ def test_release_requirement_component_ports_dedupes_positive_final_and_resource release_requirement_component_ports( {"final": 5432, "resources": {"shadow": 15432, "duplicate": 5432, "ignored": 0}}, - release_port_fn=released.append, + port_planner=SimpleNamespace(release=released.append), ) self.assertEqual(released, [5432, 15432]) + def test_selected_dependency_stop_releases_exact_prior_session_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(5432, owner="Main:db") + requirements = RequirementsResult( + project="Main", + db={ + "enabled": True, + "final": 5432, + "port_lock_session": "prior-session", + }, + redis={"enabled": True, "final": 6379}, + ) + state = RunState( + run_id="run-prior", + mode="main", + requirements={"Main": requirements}, + ) + + release_selected_dependency_components( + state, + {"Main": {"postgres"}}, + release_component_ports_fn=lambda owner, dependency_id, component: release_requirement_component_ports( + component, + port_planner=current, + owner_candidates=requirement_component_port_owners(owner, dependency_id), + ), + ) + + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + self.assertFalse(requirements.db.get("enabled", False)) + self.assertTrue(requirements.redis.get("enabled", False)) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/runtime/test_lifecycle_parity_restore_startup.py b/tests/python/runtime/test_lifecycle_parity_restore_startup.py index 4d7517a51..aeeeee829 100644 --- a/tests/python/runtime/test_lifecycle_parity_restore_startup.py +++ b/tests/python/runtime/test_lifecycle_parity_restore_startup.py @@ -167,7 +167,7 @@ def test_resume_restore_does_not_reuse_requirements_when_project_root_reveals_ow "ENVCTL_DEBUG_RESTORE_TIMING": "true", }, ) - restore_runner = _ResumeRestoreRunner() + restore_runner = _ResumeRestoreRunner(listeners_ready=True) engine.process_runner = restore_runner # type: ignore[assignment] def fake_reconcile(project: str, requirements: RequirementsResult, *, project_root: Path | None = None): # noqa: ANN001 diff --git a/tests/python/runtime/test_lifecycle_parity_resume_policy.py b/tests/python/runtime/test_lifecycle_parity_resume_policy.py index ef6e7bd13..f08802cd1 100644 --- a/tests/python/runtime/test_lifecycle_parity_resume_policy.py +++ b/tests/python/runtime/test_lifecycle_parity_resume_policy.py @@ -174,6 +174,8 @@ def test_resume_reconciles_missing_service_status(self) -> None: runtime = Path(tmpdir) / "runtime" run_dir = runtime / "python-engine" (repo / ".git").mkdir(parents=True, exist_ok=True) + (repo / "backend").mkdir(parents=True, exist_ok=True) + (repo / "frontend").mkdir(parents=True, exist_ok=True) run_dir.mkdir(parents=True, exist_ok=True) state = RunState( @@ -199,7 +201,15 @@ def test_resume_reconciles_missing_service_status(self) -> None: "RUN_SH_RUNTIME_DIR": str(runtime), } ) - engine = PythonEngineRuntime(config, env={}) + engine = PythonEngineRuntime( + config, + env={ + "ENVCTL_BACKEND_START_CMD": "sh -lc 'sleep 1'", + "ENVCTL_FRONTEND_START_CMD": "sh -lc 'sleep 1'", + }, + ) + restore_runner = _ResumeRestoreRunner(listeners_ready=True) + engine.process_runner = restore_runner # type: ignore[assignment] code = engine.dispatch(parse_route(["--resume"], env={})) self.assertEqual(code, 0) diff --git a/tests/python/runtime/test_lifecycle_requirement_ports.py b/tests/python/runtime/test_lifecycle_requirement_ports.py index afa5fb5cc..c4977906a 100644 --- a/tests/python/runtime/test_lifecycle_requirement_ports.py +++ b/tests/python/runtime/test_lifecycle_requirement_ports.py @@ -1,6 +1,8 @@ from __future__ import annotations import unittest +import tempfile +from pathlib import Path from types import SimpleNamespace from envctl_engine.runtime.lifecycle_requirement_ports import ( @@ -10,6 +12,7 @@ requirement_port_values, ) from envctl_engine.state.models import RequirementsResult, RunState +from envctl_engine.shared.ports import PortPlanner class LifecycleRequirementPortsTests(unittest.TestCase): @@ -20,16 +23,21 @@ def test_requirement_key_for_project_matches_case_insensitive_key(self) -> None: self.assertIsNone(requirement_key_for_project(state, " ")) self.assertIsNone(requirement_key_for_project(state, "missing")) - def test_requirement_port_values_returns_enabled_internal_final_ports_only(self) -> None: + def test_requirement_port_values_returns_all_enabled_internal_resource_ports(self) -> None: requirements = RequirementsResult( project="Main", db={"enabled": True, "final": 5432}, redis={"enabled": False, "final": 6379}, - supabase={"enabled": True, "external": True, "final": 54321}, - n8n={"enabled": True, "final": 5678, "resources": {"ignored": 15678}}, + supabase={ + "enabled": True, + "external": True, + "final": 54321, + "resources": {"external_api": 55432}, + }, + n8n={"enabled": True, "final": 5678, "resources": {"secondary": 15678}}, ) - self.assertEqual(requirement_port_values(requirements), {5432, 5678}) + self.assertEqual(requirement_port_values(requirements), {5432, 5678, 15678}) def test_component_port_values_collects_final_and_resource_ports(self) -> None: self.assertEqual( @@ -43,12 +51,157 @@ def test_release_requirement_ports_uses_runtime_port_planner_in_sorted_order(sel requirements = RequirementsResult( project="Main", db={"enabled": True, "final": 5432}, + supabase={ + "enabled": True, + "final": 15432, + "resources": {"db": 15432, "api": 15433}, + }, n8n={"enabled": True, "final": 5678}, ) release_requirement_ports(runtime, requirements) - self.assertEqual(released, [5432, 5678]) + self.assertEqual(released, [5432, 5678, 15432, 15433]) + + def test_release_requirement_ports_can_release_verified_prior_session_owners(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(5432, owner="Main:db") + prior.reserve_next(5433, owner="Main:requirements") + requirements = RequirementsResult( + project="Main", + db={ + "enabled": True, + "final": 5432, + "resources": {"primary": 5432, "retry": 5433}, + "port_lock_session": "prior-session", + }, + ) + + release_requirement_ports(SimpleNamespace(port_planner=current), requirements) + + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + + def test_collision_storage_records_release_ports_with_shared_authoritative_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(6380, owner="Main:requirements") + prior.reserve_next(6381, owner="Main:requirements") + state = RunState( + run_id="run-collision", + mode="main", + requirements={ + "Main": RequirementsResult( + project="Main", + redis={ + "enabled": True, + "final": 6380, + "port_lock_session": "prior-session", + }, + ), + "Main Restart Collision": RequirementsResult( + project="Main", + redis={ + "enabled": True, + "final": 6381, + "port_lock_session": "prior-session", + }, + ), + }, + ) + + for requirements in state.requirements.values(): + release_requirement_ports(SimpleNamespace(port_planner=cleanup), requirements) + + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + + def test_stale_requirement_state_cannot_release_newer_live_same_owner_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + newer = PortPlanner( + lock_dir=tmpdir, + session_id="newer-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + newer.reserve_next(5432, owner="Main:db") + stale_requirements = RequirementsResult( + project="Main", + db={ + "enabled": True, + "final": 5432, + "port_lock_session": "stale-session", + }, + ) + + release_requirement_ports( + SimpleNamespace(port_planner=cleanup), + stale_requirements, + ) + + self.assertTrue((Path(tmpdir) / "5432.lock").exists()) + + def test_legacy_requirement_state_preserves_live_foreign_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + live = PortPlanner( + lock_dir=tmpdir, + session_id="live-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + live.reserve_next(5432, owner="Main:db") + + release_requirement_ports( + SimpleNamespace(port_planner=cleanup), + RequirementsResult(project="Main", db={"enabled": True, "final": 5432}), + ) + + self.assertTrue((Path(tmpdir) / "5432.lock").exists()) + + def test_legacy_requirement_state_reaps_only_proven_dead_foreign_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "5432.lock" + lock_path.write_text( + '{"created_at": 0, "owner": "Main:db", "pid": 99999999, "session": "dead-session"}', + encoding="utf-8", + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + + release_requirement_ports( + SimpleNamespace(port_planner=cleanup), + RequirementsResult(project="Main", db={"enabled": True, "final": 5432}), + ) + + self.assertFalse(lock_path.exists()) if __name__ == "__main__": diff --git a/tests/python/runtime/test_lifecycle_service_stop_selection.py b/tests/python/runtime/test_lifecycle_service_stop_selection.py index 2f4bb8306..28db7929f 100644 --- a/tests/python/runtime/test_lifecycle_service_stop_selection.py +++ b/tests/python/runtime/test_lifecycle_service_stop_selection.py @@ -34,6 +34,83 @@ def test_runtime_scope_selects_matching_services_without_interactive_prompt(self self.assertEqual(selected, {"Aux Backend", "Main Backend"}) + def test_runtime_scope_intersects_explicit_project_selector(self) -> None: + state = RunState( + run_id="run-1", + mode="trees", + services={ + "FeatureA Backend": ServiceRecord( + name="FeatureA Backend", + type="backend", + cwd=".", + pid=1, + project="FeatureA", + ), + "FeatureB Backend": ServiceRecord( + name="FeatureB Backend", + type="backend", + cwd=".", + pid=2, + project="FeatureB", + ), + }, + ) + + selected = select_services_for_stop( + state, + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"runtime_scope": "backend"}, + ), + project_name_from_service_fn=lambda name: name.split(" ", 1)[0], + selectors_from_passthrough_fn=lambda _args: set(), + interactive_stop_selection_fn=lambda _route, _state: self.fail("interactive prompt should not run"), + services_from_selection_fn=lambda _selection, _state: self.fail("selection projection should not run"), + ) + + self.assertEqual(selected, {"FeatureA Backend"}) + + def test_entire_system_project_stop_includes_additional_services(self) -> None: + state = RunState( + run_id="run-1", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + pid=1, + project="Main", + ), + "Main Optional Bad": ServiceRecord( + name="Main Optional Bad", + type="optional-bad", + cwd=".", + pid=2, + project="Main", + service_slug="optional-bad", + ), + }, + ) + + selected = select_services_for_stop( + state, + Route( + command="stop", + mode="main", + projects=["Main"], + flags={"runtime_scope": "entire-system"}, + ), + project_name_from_service_fn=lambda name: name.rsplit(" ", 1)[0], + selectors_from_passthrough_fn=lambda _args: set(), + interactive_stop_selection_fn=lambda _route, _state: self.fail("interactive prompt should not run"), + services_from_selection_fn=lambda _selection, _state: self.fail("selection projection should not run"), + ) + + self.assertEqual(selected, {"Main Backend", "Main Optional Bad"}) + def test_service_and_project_selectors_match_slug_display_name_and_passthrough_project(self) -> None: state = RunState( run_id="run-1", diff --git a/tests/python/runtime/test_lifecycle_service_termination.py b/tests/python/runtime/test_lifecycle_service_termination.py index eb43fa87d..19c2f24f7 100644 --- a/tests/python/runtime/test_lifecycle_service_termination.py +++ b/tests/python/runtime/test_lifecycle_service_termination.py @@ -1,30 +1,89 @@ from __future__ import annotations import os +import signal +import subprocess +import sys +import tempfile import unittest +from pathlib import Path from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import Mock, patch from envctl_engine.runtime.lifecycle_service_termination import ( + _wait_for_pid_exit, + failed_listener_pids, service_port, terminate_service_record, terminate_services_from_state, terminate_started_services, + unconfirmed_service_names, ) from envctl_engine.state.models import RunState, ServiceRecord +from envctl_engine.shared.ports import PortPlanner class LifecycleServiceTerminationTests(unittest.TestCase): + def test_pending_docker_cleanup_timestamp_reaches_tokenized_stop(self) -> None: + runtime = SimpleNamespace( + process_runner=SimpleNamespace(), + _emit=lambda *_args, **_kwargs: None, + ) + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + runtime_kind="docker", + container_id="envctl-app-main-backend", + container_name="envctl-app-main-backend", + container_launch_token="pending-token", + container_cleanup_pending_since=1_700_000_000.25, + ) + + with patch( + "envctl_engine.runtime.docker_service_runtime.DockerServiceRuntime.stop", + return_value=False, + ) as stop: + self.assertFalse( + terminate_service_record( + runtime, + service, + aggressive=False, + verify_ownership=True, + ) + ) + + stop.assert_called_once_with( + "envctl-app-main-backend", + verify_ownership=True, + expected_launch_token="pending-token", + pending_cleanup_since=1_700_000_000.25, + ) + + def test_termination_result_normalization_fails_closed_for_missing_or_unknown_identities(self) -> None: + requested = {"Main Backend", "Main Frontend"} + + self.assertEqual(unconfirmed_service_names(None, requested), requested) + self.assertEqual(unconfirmed_service_names("Main Backend", requested), requested) + self.assertEqual(unconfirmed_service_names({"unknown"}, requested), requested) + self.assertEqual(unconfirmed_service_names([" "], requested), requested) + self.assertEqual(unconfirmed_service_names(["", "Main Backend"], requested), requested) + self.assertEqual(unconfirmed_service_names({"Main Backend"}, requested), {"Main Backend"}) + self.assertEqual(unconfirmed_service_names([], requested), set()) + self.assertEqual(failed_listener_pids(set()), set()) + self.assertEqual(failed_listener_pids({4242}), {4242}) + self.assertIsNone(failed_listener_pids(None)) + self.assertIsNone(failed_listener_pids({"4242"})) + def test_terminate_started_services_uses_non_aggressive_unverified_shutdown(self) -> None: calls: list[tuple[int, bool, bool]] = [] runtime = SimpleNamespace( - _terminate_service_record=lambda service, *, aggressive, verify_ownership: calls.append( - (service.pid, aggressive, verify_ownership) + _terminate_service_record=lambda service, *, aggressive, verify_ownership: ( + calls.append((service.pid, aggressive, verify_ownership)) or True ) - or True ) - terminate_started_services( + failed = terminate_started_services( runtime, { "Main Backend": ServiceRecord(name="Main Backend", type="backend", cwd=".", pid=1), @@ -33,16 +92,56 @@ def test_terminate_started_services_uses_non_aggressive_unverified_shutdown(self ) self.assertEqual(calls, [(1, False, False), (2, False, False)]) + self.assertEqual(failed, set()) + + def test_terminate_started_services_reports_every_unconfirmed_exit(self) -> None: + runtime = SimpleNamespace(_terminate_service_record=lambda service, **_kwargs: service.name != "Main Backend") + + failed = terminate_started_services( + runtime, + { + "Main Backend": ServiceRecord(name="Main Backend", type="backend", cwd=".", pid=1), + "Main Frontend": ServiceRecord(name="Main Frontend", type="frontend", cwd=".", pid=2), + }, + ) + + self.assertEqual(failed, {"Main Backend"}) + + def test_terminate_started_services_continues_after_terminator_exception(self) -> None: + calls: list[str] = [] + events: list[tuple[str, dict[str, object]]] = [] + + def terminate(service: ServiceRecord, **_kwargs: object) -> bool: + calls.append(service.name) + if service.name == "Main Backend": + raise RuntimeError("probe failed") + return True + + runtime = SimpleNamespace( + _terminate_service_record=terminate, + _emit=lambda event, **payload: events.append((event, payload)), + ) + + failed = terminate_started_services( + runtime, + { + "Main Backend": ServiceRecord(name="Main Backend", type="backend", cwd="."), + "Main Frontend": ServiceRecord(name="Main Frontend", type="frontend", cwd="."), + }, + ) + + self.assertEqual(calls, ["Main Backend", "Main Frontend"]) + self.assertEqual(failed, {"Main Backend"}) + self.assertEqual(events[0][0], "cleanup.error") def test_terminate_services_from_state_filters_selected_services_and_releases_terminated_ports(self) -> None: released: list[int] = [] terminated: list[tuple[int, bool, bool]] = [] runtime = SimpleNamespace( port_planner=SimpleNamespace(release=lambda port: released.append(port)), - _terminate_service_record=lambda service, *, aggressive, verify_ownership: terminated.append( - (service.pid, aggressive, verify_ownership) - ) - or True, + _terminate_service_record=lambda service, *, aggressive, verify_ownership: ( + terminated.append((service.pid, aggressive, verify_ownership)) or True + ), ) state = RunState( run_id="run-1", @@ -55,7 +154,7 @@ def test_terminate_services_from_state_filters_selected_services_and_releases_te }, ) - terminate_services_from_state( + failed = terminate_services_from_state( runtime, state, selected_services={"Main Frontend"}, @@ -65,12 +164,275 @@ def test_terminate_services_from_state_filters_selected_services_and_releases_te self.assertEqual(terminated, [(2, True, True)]) self.assertEqual(released, [3000]) + self.assertEqual(failed, set()) + + def test_terminated_service_releases_verified_prior_session_port_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(8000, owner="Main:backend") + runtime = SimpleNamespace( + port_planner=current, + _terminate_service_record=lambda _service, **_kwargs: True, + _project_name_from_service=lambda _name: "Main", + ) + state = RunState( + run_id="run-prior", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + project="Main", + pid=1, + actual_port=8000, + port_lock_session="prior-session", + ) + }, + ) + + failed = terminate_services_from_state( + runtime, + state, + selected_services=None, + aggressive=False, + verify_ownership=True, + ) + + self.assertEqual(failed, set()) + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + + def test_legacy_collision_name_releases_original_project_owner_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(8001, owner="Main:backend") + collision_name = "Main Backend Restart Collision 2" + state = RunState( + run_id="run-collision", + mode="main", + services={ + collision_name: ServiceRecord( + name=collision_name, + type="backend", + cwd=".", + pid=2, + actual_port=8001, + port_lock_session="prior-session", + project=None, + ) + }, + ) + runtime = SimpleNamespace( + port_planner=current, + _terminate_service_record=lambda _service, **_kwargs: True, + _project_name_from_service=lambda _name: "Main Backend Restart Collision", + ) + + failed = terminate_services_from_state( + runtime, + state, + selected_services=None, + aggressive=False, + verify_ownership=True, + ) + + self.assertEqual(failed, set()) + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + + def test_terminated_rebound_frontend_releases_launch_owner_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(5173, owner="Main:frontend") + prior.reserve_next(5174, owner="Main:services:frontend-launch") + runtime = SimpleNamespace( + port_planner=current, + _terminate_service_record=lambda _service, **_kwargs: True, + _project_name_from_service=lambda _name: "Main", + ) + state = RunState( + run_id="run-prior", + mode="main", + services={ + "Main Frontend": ServiceRecord( + name="Main Frontend", + type="frontend", + cwd=".", + project="Main", + pid=1, + requested_port=5173, + actual_port=5174, + port_lock_session="prior-session", + ) + }, + ) + + failed = terminate_services_from_state( + runtime, + state, + selected_services=None, + aggressive=False, + verify_ownership=True, + ) + + self.assertEqual(failed, set()) + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + + def test_stale_service_state_cannot_release_newer_live_same_owner_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + newer = PortPlanner( + lock_dir=tmpdir, + session_id="newer-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + newer.reserve_next(8000, owner="Main:backend") + runtime = SimpleNamespace( + port_planner=cleanup, + _terminate_service_record=lambda _service, **_kwargs: True, + ) + state = RunState( + run_id="stale-run", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + project="Main", + pid=1, + actual_port=8000, + port_lock_session="stale-session", + ) + }, + ) + + failed = terminate_services_from_state( + runtime, + state, + selected_services=None, + aggressive=False, + verify_ownership=True, + ) + + self.assertEqual(failed, set()) + self.assertTrue((Path(tmpdir) / "8000.lock").exists()) + + def test_startup_rollback_releases_confirmed_service_ports_but_preserves_failed_service_ports(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="startup-session", + availability_checker=lambda _port: True, + ) + planner.reserve_next(5173, owner="Main:frontend") + planner.reserve_next(5174, owner="Main:services:frontend-launch") + planner.reserve_next(8000, owner="Main:backend") + frontend = ServiceRecord( + name="Main Frontend", + type="frontend", + cwd=".", + project="Main", + pid=1, + requested_port=5173, + actual_port=5174, + ) + backend = ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + project="Main", + pid=2, + requested_port=8000, + actual_port=8000, + ) + runtime = SimpleNamespace( + port_planner=planner, + _terminate_service_record=lambda service, **_kwargs: service is frontend, + ) + + failed = terminate_started_services( + runtime, + {frontend.name: frontend, backend.name: backend}, + ) + + self.assertEqual(failed, {backend.name}) + remaining = list(Path(tmpdir).glob("*.lock")) + self.assertEqual([path.name for path in remaining], ["8000.lock"]) + + def test_terminate_services_reports_failures_without_releasing_their_ports(self) -> None: + released: list[int] = [] + runtime = SimpleNamespace( + port_planner=SimpleNamespace(release=lambda port: released.append(port)), + _terminate_service_record=lambda service, **_kwargs: service.name != "Main Backend", + ) + state = RunState( + run_id="run-1", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + pid=1, + actual_port=8000, + ), + "Main Frontend": ServiceRecord( + name="Main Frontend", + type="frontend", + cwd=".", + pid=2, + actual_port=3000, + ), + }, + ) + + failed = terminate_services_from_state( + runtime, + state, + selected_services=None, + aggressive=False, + verify_ownership=True, + ) + + self.assertEqual(failed, {"Main Backend"}) + self.assertEqual(released, [3000]) def test_terminate_service_record_skips_self_parent_and_missing_ownership_port(self) -> None: events: list[dict[str, object]] = [] runtime = SimpleNamespace( _emit=lambda _event, **payload: events.append(payload), - process_runner=SimpleNamespace(), + process_runner=SimpleNamespace(is_pid_running=lambda _pid: True), ) with patch.object(os, "getpid", return_value=100), patch.object(os, "getppid", return_value=200): @@ -96,6 +458,7 @@ def test_terminate_service_record_skips_self_parent_and_missing_ownership_port(s def test_terminate_service_record_accepts_verified_listener_child_and_prefers_process_group(self) -> None: calls: list[tuple[str, int]] = [] + live_pids = {111, 222} def pid_owns_port(pid: int, port: int) -> bool: return pid == 222 and port == 9000 @@ -103,8 +466,13 @@ def pid_owns_port(pid: int, port: int) -> bool: runtime = SimpleNamespace( _emit=lambda *_args, **_kwargs: None, process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, pid_owns_port=pid_owns_port, - terminate_process_group=lambda pid, **_kwargs: calls.append(("group", pid)) or True, + terminate_process_group=lambda pid, **_kwargs: ( + calls.append(("group", pid)), + live_pids.discard(pid), + True, + )[-1], terminate=lambda pid, **_kwargs: calls.append(("single", pid)) or True, ), ) @@ -117,7 +485,428 @@ def pid_owns_port(pid: int, port: int) -> bool: ) self.assertTrue(terminated) - self.assertEqual(calls, [("group", 111)]) + self.assertEqual(calls, [("group", 111), ("group", 222)]) + + def test_terminate_service_record_stops_non_listener_with_matching_recorded_cwd(self) -> None: + calls: list[int] = [] + live_pids = {333} + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + process_cwd=lambda _pid: "/repo/worker", + terminate_process_group=lambda pid, **_kwargs: (calls.append(pid), live_pids.discard(pid), True)[-1], + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Worker", + type="worker", + cwd="/repo/worker", + pid=333, + listener_expected=False, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertTrue(terminated) + self.assertEqual(calls, [333]) + + def test_terminate_service_record_keeps_non_listener_when_cwd_cannot_be_verified(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + runtime = SimpleNamespace( + _emit=lambda event, **payload: events.append((event, payload)), + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + process_cwd=lambda _pid: "/foreign/worker", + terminate_process_group=lambda *_args, **_kwargs: self.fail("foreign PID must not be signalled"), + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Worker", + type="worker", + cwd="/repo/worker", + pid=334, + listener_expected=False, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertFalse(terminated) + self.assertEqual(events[0][1]["reason"], "missing_port_for_ownership") + + def test_parent_dead_live_listener_child_is_still_terminated(self) -> None: + calls: list[int] = [] + live_pids = {402} + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + pid_owns_port=lambda pid, port: pid == 402 and port == 8000, + terminate_process_group=lambda pid, **_kwargs: (calls.append(pid), live_pids.discard(pid), True)[-1], + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=401, + listener_pids=[402], + actual_port=8000, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertTrue(terminated) + self.assertEqual(calls, [402]) + + def test_missing_primary_pid_does_not_hide_live_listener_child(self) -> None: + calls: list[int] = [] + live_pids = {403} + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + pid_owns_port=lambda pid, port: pid == 403 and port == 8001, + terminate_process_group=lambda pid, **_kwargs: (calls.append(pid), live_pids.discard(pid), True)[-1], + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=None, + listener_pids=[403], + actual_port=8001, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertTrue(terminated) + self.assertEqual(calls, [403]) + + def test_dead_parent_with_unrecorded_live_listener_keeps_state_without_signalling_foreign_pid(self) -> None: + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _listener_pids_for_port=lambda _port: [406], + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid == 406, + pid_owns_port=lambda *_args, **_kwargs: self.fail("no recorded live PID should be probed"), + terminate_process_group=lambda *_args, **_kwargs: self.fail("foreign PID must not be signalled"), + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=404, + listener_pids=None, + actual_port=8002, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertFalse(terminated) + + def test_pidless_unconfirmed_launch_remains_tracked(self) -> None: + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace(), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Worker", + type="worker", + cwd="/repo/worker", + pid=None, + listener_expected=False, + status="termination_failed", + ), + aggressive=False, + verify_ownership=False, + ) + + self.assertFalse(terminated) + + def test_live_port_listener_after_termination_keeps_service_tracked(self) -> None: + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _listener_pids_for_port=lambda _port: [405], + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + pid_owns_port=lambda pid, port: pid == 404 and port == 8002, + terminate_process_group=lambda *_args, **_kwargs: True, + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=404, + actual_port=8002, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertFalse(terminated) + + def test_malformed_listener_probe_result_fails_closed(self) -> None: + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _listener_pids_for_port=lambda _port: None, + process_runner=SimpleNamespace(is_pid_running=lambda _pid: False), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=999, + actual_port=8003, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertFalse(terminated) + + def test_terminate_service_record_treats_already_dead_pid_as_success_before_ownership_check(self) -> None: + ownership_checks: list[tuple[int, int]] = [] + terminate_calls: list[int] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: False, + pid_owns_port=lambda pid, port: ownership_checks.append((pid, port)) or False, + terminate_process_group=lambda pid, **_kwargs: terminate_calls.append(pid) or True, + ), + ) + + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Shadow Backend", pid=777, actual_port=8000), + aggressive=False, + verify_ownership=True, + ) + + self.assertTrue(terminated) + self.assertEqual(ownership_checks, []) + self.assertEqual(terminate_calls, []) + + def test_terminate_service_record_keeps_live_non_owner_fail_closed(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + runtime = SimpleNamespace( + _emit=lambda event, **payload: events.append((event, payload)), + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + pid_owns_port=lambda _pid, _port: False, + terminate_process_group=lambda *_args, **_kwargs: self.fail("non-owner must not be signalled"), + ), + ) + + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Foreign Backend", pid=778, actual_port=8001), + aggressive=False, + verify_ownership=True, + ) + + self.assertFalse(terminated) + self.assertEqual( + events, + [ + ( + "cleanup.skip", + { + "service": "Foreign Backend", + "pid": 778, + "port": 8001, + "reason": "ownership_mismatch", + }, + ) + ], + ) + + def test_raw_signal_fallback_escalates_and_fails_when_pid_survives(self) -> None: + sent_signals: list[tuple[int, signal.Signals]] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + terminate=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + ), + ) + + with ( + patch( + "envctl_engine.shared.process_termination.os.kill", + side_effect=lambda pid, requested_signal: sent_signals.append((pid, requested_signal)), + ), + patch( + "envctl_engine.shared.process_termination.wait_for_pid_exit", + side_effect=[False, False], + ) as wait_for_exit, + ): + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Main Backend", pid=779, actual_port=8002), + aggressive=False, + verify_ownership=False, + ) + + self.assertFalse(terminated) + self.assertEqual(sent_signals, [(779, signal.SIGTERM), (779, signal.SIGKILL)]) + self.assertEqual( + [call.kwargs["timeout"] for call in wait_for_exit.call_args_list], + [2.0, 1.0], + ) + + def test_raw_signal_fallback_does_not_escalate_after_confirmed_sigterm_exit(self) -> None: + sent_signals: list[tuple[int, signal.Signals]] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + terminate=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + ), + ) + + with ( + patch( + "envctl_engine.shared.process_termination.os.kill", + side_effect=lambda pid, requested_signal: sent_signals.append((pid, requested_signal)), + ), + patch( + "envctl_engine.shared.process_termination.wait_for_pid_exit", + return_value=True, + ), + ): + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Main Backend", pid=780, actual_port=8003), + aggressive=True, + verify_ownership=False, + ) + + self.assertTrue(terminated) + self.assertEqual(sent_signals, [(780, signal.SIGTERM)]) + + def test_raw_signal_fallback_accepts_process_lookup_race_as_clean_exit(self) -> None: + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + terminate=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + ), + ) + + with patch( + "envctl_engine.shared.process_termination.os.kill", + side_effect=ProcessLookupError, + ): + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Main Backend", pid=781, actual_port=8004), + aggressive=False, + verify_ownership=False, + ) + + self.assertTrue(terminated) + + def test_raw_signal_exit_wait_polls_until_pid_is_confirmed_dead(self) -> None: + runtime = SimpleNamespace( + process_runner=SimpleNamespace(is_pid_running=Mock(side_effect=[True, False])), + ) + + with ( + patch( + "envctl_engine.shared.process_termination.time.monotonic", + side_effect=[10.0, 10.0], + ), + patch("envctl_engine.shared.process_termination.time.sleep") as sleep_mock, + ): + exited = _wait_for_pid_exit( + runtime, + 782, + timeout=1.0, + initial_identity=None, + ) + + self.assertTrue(exited) + sleep_mock.assert_called_once_with(0.05) + + def test_raw_signal_fallback_really_escalates_sigterm_resistant_process(self) -> None: + process = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "import signal,time; " + "signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "print('ready', flush=True); " + "time.sleep(30)" + ), + ], + stdout=subprocess.PIPE, + text=True, + ) + try: + assert process.stdout is not None + self.assertEqual(process.stdout.readline().strip(), "ready") + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: process.poll() is None, + terminate_process_group=lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("unsupported") + ), + terminate=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + ), + ) + + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Main Backend", pid=process.pid, actual_port=8005), + aggressive=True, + verify_ownership=False, + ) + + self.assertTrue(terminated) + self.assertEqual(process.wait(timeout=2.0), -signal.SIGKILL) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=2.0) def test_service_port_prefers_actual_then_requested(self) -> None: self.assertEqual(service_port(SimpleNamespace(actual_port=9000, requested_port=8000)), 9000) diff --git a/tests/python/runtime/test_lifecycle_worktree_cleanup.py b/tests/python/runtime/test_lifecycle_worktree_cleanup.py index 79773b828..ed7d6797a 100644 --- a/tests/python/runtime/test_lifecycle_worktree_cleanup.py +++ b/tests/python/runtime/test_lifecycle_worktree_cleanup.py @@ -4,6 +4,7 @@ from pathlib import Path from types import SimpleNamespace +from envctl_engine.runtime.docker_service_runtime import docker_service_container_name from envctl_engine.runtime.lifecycle_worktree_containers import legacy_container_name, remove_tree_containers from envctl_engine.runtime.lifecycle_worktree_cleanup import WorktreeCleanupDependencies, blast_worktree_before_delete from envctl_engine.runtime.lifecycle_worktree_metadata import cleanup_artifact_paths, prune_project_metadata @@ -12,6 +13,128 @@ class LifecycleWorktreeCleanupTests(unittest.TestCase): + def test_remove_tree_containers_removes_managed_application_container(self) -> None: + calls: list[list[str]] = [] + project_root = Path(".").resolve() + container_name = docker_service_container_name( + project_name="Feature", + project_root=project_root, + service_name="backend", + ) + + class Runner: + def run(self, command: list[str], **kwargs: object) -> SimpleNamespace: + calls.append(command) + if command[1:4] == ["ps", "-a", "--format"]: + return SimpleNamespace(returncode=0, stdout=f"app-id|{container_name}\n", stderr="") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + runtime = SimpleNamespace( + process_runner=Runner(), + env={}, + config=SimpleNamespace(additional_services=()), + _emit=lambda event, **payload: None, + ) + warnings: list[str] = [] + + remove_tree_containers( + runtime, + project_name="Feature", + project_root=project_root, + include_supabase=False, + remove_named_volumes=False, + warnings=warnings, + ) + + self.assertEqual(warnings, []) + self.assertIn(["docker", "rm", "-f", "-v", "app-id"], calls) + + def test_worktree_cleanup_persists_siblings_from_the_full_multi_project_run(self) -> None: + loader_calls: list[dict[str, object]] = [] + saved_states: list[RunState] = [] + target_service = ServiceRecord( + name="Feature Backend", + type="backend", + cwd="/repo/feature/backend", + actual_port=8000, + project="Feature", + ) + sibling_service = ServiceRecord( + name="Sibling Backend", + type="backend", + cwd="/repo/sibling/backend", + actual_port=8020, + project="Sibling", + ) + full_state = RunState( + run_id="run-multi", + mode="trees", + services={target_service.name: target_service, sibling_service.name: sibling_service}, + requirements={ + "Feature": RequirementsResult(project="Feature", db={"enabled": True, "final": 5432}), + "Sibling": RequirementsResult(project="Sibling", db={"enabled": True, "final": 5452}), + "SiblingInfra": RequirementsResult( + project="SiblingInfra", + redis={"enabled": True, "final": 6399}, + ), + }, + metadata={ + "project_names": ["Feature", "Sibling", "SiblingInfra"], + "project_roots": { + "Feature": "/repo/feature", + "Sibling": "/repo/sibling", + "SiblingInfra": "/repo/sibling-infra", + }, + }, + ) + + def load_state(**kwargs: object) -> RunState | None: + loader_calls.append(dict(kwargs)) + if kwargs.get("mode") != "trees": + return None + if kwargs.get("project_names"): + return RunState( + run_id=full_state.run_id, + mode=full_state.mode, + services={target_service.name: target_service}, + requirements={"Feature": full_state.requirements["Feature"]}, + metadata={ + "project_names": ["Feature"], + "project_roots": {"Feature": "/repo/feature"}, + }, + ) + return full_state + + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _try_load_existing_state=load_state, + _project_name_from_service=lambda name: str(name).removesuffix(" Backend"), + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + port_planner=SimpleNamespace(release=lambda _port: None), + state_repository=SimpleNamespace( + save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) + ), + process_runner=SimpleNamespace(), + env={}, + ) + + blast_worktree_before_delete( + runtime, + project_name="Feature", + project_root=Path("/repo/feature"), + source_command="delete-worktree", + ) + + self.assertTrue(loader_calls) + self.assertTrue(all("project_names" not in call for call in loader_calls)) + self.assertEqual(set(saved_states[0].services), {"Sibling Backend"}) + self.assertEqual(set(saved_states[0].requirements), {"Sibling", "SiblingInfra"}) + self.assertEqual(saved_states[0].metadata["project_names"], ["Sibling", "SiblingInfra"]) + self.assertEqual( + saved_states[0].metadata["project_roots"], + {"Sibling": "/repo/sibling", "SiblingInfra": "/repo/sibling-infra"}, + ) + def test_blast_worktree_before_delete_updates_matching_project_state(self) -> None: events: list[tuple[str, dict[str, object]]] = [] saved_states: list[RunState] = [] @@ -32,7 +155,7 @@ def test_blast_worktree_before_delete_updates_matching_project_state(self) -> No _emit=lambda event, **payload: events.append((event, payload)), _try_load_existing_state=lambda **kwargs: state if kwargs["mode"] == "trees" else None, _project_name_from_service=lambda name: "Feature" if "Feature" in name else "", - _terminate_services_from_state=lambda *args, **kwargs: None, + _terminate_services_from_state=lambda *args, **kwargs: set(), port_planner=SimpleNamespace(release=lambda port: released.append(port)), state_repository=SimpleNamespace( save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) @@ -54,6 +177,143 @@ def test_blast_worktree_before_delete_updates_matching_project_state(self) -> No self.assertEqual(saved_states[0].requirements, {}) self.assertEqual(events[-1][0], "cleanup.worktree.finish") + def test_worktree_cleanup_releases_every_requirement_storage_row_for_project(self) -> None: + saved_states: list[RunState] = [] + released: list[int] = [] + state = RunState( + run_id="run-collision", + mode="trees", + requirements={ + "Feature": RequirementsResult(project="Feature", redis={"enabled": True, "final": 6379}), + "Feature Restart Collision": RequirementsResult( + project="Feature", + redis={"enabled": True, "final": 6381}, + ), + "Sibling": RequirementsResult(project="Sibling", redis={"enabled": True, "final": 6391}), + }, + metadata={"project_names": ["Feature", "Sibling"]}, + ) + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _try_load_existing_state=lambda **kwargs: state if kwargs["mode"] == "trees" else None, + _project_name_from_service=lambda _name: "", + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + port_planner=SimpleNamespace(release=lambda port: released.append(port)), + state_repository=SimpleNamespace( + save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) + ), + process_runner=SimpleNamespace(), + env={}, + ) + + warnings = blast_worktree_before_delete( + runtime, + project_name="Feature", + project_root=Path("/repo/feature"), + source_command="delete-worktree", + ) + + self.assertEqual(warnings, []) + self.assertEqual(released, [6379, 6381]) + self.assertEqual(set(saved_states[0].requirements), {"Sibling"}) + + def test_worktree_cleanup_stops_saved_custom_requirement_container_before_dropping_state(self) -> None: + calls: list[list[str]] = [] + + class Runner: + def run(self, command: list[str], **_kwargs: object) -> SimpleNamespace: + calls.append(command) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + state = RunState( + run_id="run-container-collision", + mode="trees", + requirements={ + "Feature Restart Collision": RequirementsResult( + project="Feature", + redis={ + "enabled": True, + "success": True, + "final": 6381, + "container_name": "custom-collision-redis", + }, + ) + }, + ) + saved_states: list[RunState] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _try_load_existing_state=lambda **kwargs: state if kwargs["mode"] == "trees" else None, + _project_name_from_service=lambda _name: "", + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + port_planner=SimpleNamespace(release=lambda _port: None), + state_repository=SimpleNamespace( + save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) + ), + process_runner=Runner(), + config=SimpleNamespace(additional_services=()), + env={}, + ) + + warnings = blast_worktree_before_delete( + runtime, + project_name="Feature", + project_root=Path("/repo/feature"), + source_command="delete-worktree", + ) + + self.assertEqual(warnings, []) + self.assertIn(["docker", "rm", "--force", "custom-collision-redis"], calls) + self.assertEqual(saved_states[0].requirements, {}) + + def test_worktree_cleanup_uses_multiword_additional_service_project_metadata(self) -> None: + terminated: list[set[str]] = [] + saved_states: list[RunState] = [] + service = ServiceRecord( + name="Opaque Runtime Process", + type="voice-runtime", + cwd="/repo/customer-platform/voice", + project="Customer Platform", + actual_port=8010, + ) + legacy_service = ServiceRecord( + name="Customer Platform Voice Runtime", + type="voice-runtime", + cwd="/repo/customer-platform/voice-legacy", + project=None, + service_slug="voice-runtime", + actual_port=8011, + ) + state = RunState( + run_id="run-metadata-project", + mode="trees", + services={service.name: service, legacy_service.name: legacy_service}, + ) + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _try_load_existing_state=lambda **kwargs: state if kwargs["mode"] == "trees" else None, + _project_name_from_service=lambda _name: "Opaque Runtime", + _terminate_services_from_state=lambda _state, *, selected_services, **_kwargs: ( + terminated.append(set(selected_services)) or set() + ), + state_repository=SimpleNamespace( + save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) + ), + process_runner=SimpleNamespace(), + env={}, + ) + + warnings = blast_worktree_before_delete( + runtime, + project_name="Customer Platform", + project_root=Path("/repo/customer-platform"), + source_command="delete-worktree", + ) + + self.assertEqual(warnings, []) + self.assertEqual(terminated, [{service.name, legacy_service.name}]) + self.assertEqual(saved_states[0].services, {}) + def test_blast_worktree_before_delete_uses_injected_cleanup_dependencies(self) -> None: calls: list[str] = [] state = RunState(run_id="run-1", mode="trees") @@ -86,6 +346,56 @@ def test_blast_worktree_before_delete_uses_injected_cleanup_dependencies(self) - self.assertEqual(warnings, []) self.assertEqual(calls, ["ports", "cwd", "artifacts", "containers"]) + def test_worktree_delete_aborts_and_retains_authority_when_service_exit_is_unconfirmed(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + saved_states: list[RunState] = [] + released: list[int] = [] + service = ServiceRecord( + name="Feature Backend", + type="backend", + cwd="/repo/feature/backend", + pid=10, + actual_port=8000, + ) + requirements = RequirementsResult( + project="Feature", + db={"enabled": True, "final": 5432}, + ) + state = RunState( + run_id="run-1", + mode="trees", + services={service.name: service}, + requirements={"Feature": requirements}, + metadata={"project_roots": {"Feature": "/repo/feature"}}, + ) + runtime = SimpleNamespace( + _emit=lambda event, **payload: events.append((event, payload)), + _try_load_existing_state=lambda **kwargs: state if kwargs["mode"] == "trees" else None, + _project_name_from_service=lambda name: "Feature" if "Feature" in name else "", + _terminate_services_from_state=lambda *args, **kwargs: {service.name}, + port_planner=SimpleNamespace(release=lambda port: released.append(port)), + state_repository=SimpleNamespace( + save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) + ), + process_runner=SimpleNamespace(), + env={}, + ) + + with self.assertRaisesRegex(RuntimeError, "could not confirm service exit"): + blast_worktree_before_delete( + runtime, + project_name="Feature", + project_root=Path("/repo/feature"), + source_command="delete-worktree", + ) + + self.assertEqual(len(saved_states), 1) + self.assertIs(saved_states[0].services[service.name], service) + self.assertIs(saved_states[0].requirements["Feature"], requirements) + self.assertEqual(saved_states[0].metadata["project_roots"], {"Feature": "/repo/feature"}) + self.assertEqual(released, []) + self.assertEqual(events[-1][0], "cleanup.worktree.warning") + def test_prune_project_metadata_returns_test_artifacts_and_removes_empty_keys(self) -> None: state = RunState( run_id="run-1", @@ -152,7 +462,9 @@ def test_blast_tree_listener_ports_skips_docker_processes_and_kills_owned_listen self.assertEqual(warnings, []) self.assertEqual(killed, [222]) - self.assertEqual([event for event, _payload in events], ["cleanup.worktree.port.skip", "cleanup.worktree.port.kill"]) + self.assertEqual( + [event for event, _payload in events], ["cleanup.worktree.port.skip", "cleanup.worktree.port.kill"] + ) def test_remove_tree_containers_matches_hashed_and_legacy_names_and_removes_volumes(self) -> None: calls: list[list[str]] = [] diff --git a/tests/python/runtime/test_pr_preview_controller.py b/tests/python/runtime/test_pr_preview_controller.py index b11a61682..e617a6654 100644 --- a/tests/python/runtime/test_pr_preview_controller.py +++ b/tests/python/runtime/test_pr_preview_controller.py @@ -1,5 +1,6 @@ from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime import json from pathlib import Path @@ -11,6 +12,9 @@ import envctl_engine.pr_preview_controller as preview_controller +REAL_PROBE_PUBLIC_ROUTE_URL = preview_controller.probe_public_route_url + + def load_controller(): return preview_controller @@ -1308,6 +1312,113 @@ def test_manual_start_refreshes_preview_ttl_when_label_is_old(tmp_path): assert "2000-01-01" not in runner.comments[-1] +def test_manual_start_reconciles_same_head_instead_of_reusing_stale_runtime( + tmp_path, + monkeypatch, +): + controller = load_controller() + root = tmp_path / "control" / "trees" / "imported" / "feature-demo" + root.mkdir(parents=True) + runner = FakeRunner( + controller, + projects={"projects": [{"name": "feature/demo", "root": str(root), "running": True}]}, + endpoints={"frontend": {"port": 9000}, "backend": {"port": 8000}}, + root_to_branch={str(root): "feature/demo"}, + ) + instance = controller.PreviewController(make_config(controller, tmp_path), runner) + pr = controller.pr_from_event( + pr_payload( + action="labeled", + labels=["deploy-app"], + event_label="deploy-app", + ) + ) + instance.save_state( + controller.PreviewState( + pr_number=pr.number, + label="deploy-app", + project="feature/demo", + root=str(root), + head_ref=pr.head_ref, + head_sha=pr.head_sha, + status="running", + label_added_at="2026-06-14T00:00:00Z", + started_at="2026-06-14T00:00:00Z", + expires_at="2026-06-14T00:45:00Z", + updated_at="2026-06-14T00:00:00Z", + endpoints={"frontend": {"public_url": "https://preview.example"}}, + deployment_id="12345", + ) + ) + monkeypatch.setattr(instance, "get_pr", lambda _number: pr) + + exit_code = instance.run_command("start", pr.number) + + assert exit_code == 0 + sequence = [ + call["argv"][:2] + for call in runner.calls + if call["argv"][:2] in (["envctl", "stop"], ["envctl", "import"], ["envctl", "start"]) + ] + assert sequence == [ + ["envctl", "stop"], + ["envctl", "import"], + ["envctl", "stop"], + ["envctl", "start"], + ] + state = instance.load_state(pr.number) + assert state is not None + assert state.status == "running" + assert state.head_sha == pr.head_sha + assert state.started_at != "2026-06-14T00:00:00Z" + assert "already running for this head" not in runner.comments[-1] + + +@pytest.mark.parametrize( + ("payload", "expected"), + [ + ({"ready": True}, None), + ( + {"ready": False, "reason": "missing_provider_credentials"}, + "reported ready=false (reason=missing_provider_credentials)", + ), + ( + {"ready": "true", "reason": "invalid/value with spaces"}, + "returned a non-boolean readiness value (reason=invalid_value_with_spaces)", + ), + ], +) +def test_public_route_probe_honors_json_readiness_semantics( + monkeypatch, + payload, + expected, +): + controller = load_controller() + + class Response: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, limit): + body = json.dumps(payload).encode("utf-8") + assert len(body) <= limit + return body + + monkeypatch.setattr(controller.urllib.request, "urlopen", lambda *_args, **_kwargs: Response()) + + result = REAL_PROBE_PUBLIC_ROUTE_URL("https://preview.example/readyz") + + if expected is None: + assert result is None + else: + assert expected in str(result) + + def test_push_event_refreshes_preview_ttl_for_labeled_branch(tmp_path): controller = load_controller() root = tmp_path / "control" / "trees" / "imported" / "feature-demo" @@ -1337,6 +1448,27 @@ def test_push_event_refreshes_preview_ttl_for_labeled_branch(tmp_path): assert "- Reason: push to labeled PR branch feature/demo" in runner.comments[-1] +def test_controller_run_dispatches_github_push_payload(tmp_path, monkeypatch): + controller = load_controller() + event_path = tmp_path / "push.json" + event_path.write_text(json.dumps(push_payload(after="newsha")), encoding="utf-8") + config = make_config(controller, tmp_path) + instance = controller.PreviewController(config, FakeRunner(controller)) + received = [] + + monkeypatch.setattr( + instance, + "run_push_event", + lambda event: received.append(event) or 7, + ) + args = controller.build_parser().parse_args( + ["--event-name", "push", "--event-path", str(event_path)] + ) + + assert instance.run(args) == 7 + assert received == [push_payload(after="newsha")] + + def test_push_event_uses_pushed_sha_before_same_head_skip(tmp_path): controller = load_controller() root = tmp_path / "control" / "trees" / "imported" / "feature-demo" @@ -2029,6 +2161,117 @@ def test_start_keeps_backend_venv_created_with_same_python(tmp_path): assert venv.exists() +def test_preview_python_command_shim_supports_legacy_python_token(tmp_path): + controller = load_controller() + env = { + "PYTHON_BIN": sys.executable, + "PATH": "/usr/bin:/bin", + } + + shim = controller.prepare_python_command_shim( + env, + runtime_root=tmp_path / "runtime", + ) + first_target = shim.resolve(strict=True) + second = controller.prepare_python_command_shim( + env, + runtime_root=tmp_path / "runtime", + ) + result = subprocess.run( + ["python", "-c", "print('legacy-python-ok')"], + check=False, + capture_output=True, + text=True, + env=env, + ) + + assert shim == second + assert first_target == Path(sys.executable).resolve() + assert second.resolve(strict=True) == Path(sys.executable).resolve() + assert Path(env["PATH"].split(controller.os.pathsep, 1)[0]) == shim.parent + assert env["ENVCTL_PYTHON_EXECUTABLE"] == str(Path(sys.executable).resolve()) + assert result.returncode == 0 + assert result.stdout.strip() == "legacy-python-ok" + + +def test_preview_python_command_shims_are_isolated_by_interpreter(tmp_path): + controller = load_controller() + + def make_interpreter(index: int) -> Path: + executable = tmp_path / f"python-{index}" + executable.write_text( + "#!/bin/sh\n" + "case \"$2\" in\n" + " *sys.version_info*) printf '3.13\\n' ;;\n" + f" *) printf 'interpreter-{index}\\n' ;;\n" + "esac\n", + encoding="utf-8", + ) + executable.chmod(0o755) + return executable + + interpreters = [make_interpreter(index) for index in range(16)] + + def prepare_and_run(interpreter: Path) -> tuple[Path, str]: + env = {"PYTHON_BIN": str(interpreter), "PATH": "/usr/bin:/bin"} + shim = controller.prepare_python_command_shim( + env, + runtime_root=tmp_path / "runtime", + ) + result = subprocess.run( + ["python", "-c", "print('selected')"], + check=False, + capture_output=True, + text=True, + env=env, + ) + assert result.returncode == 0 + return shim, result.stdout.strip() + + with ThreadPoolExecutor(max_workers=len(interpreters)) as executor: + results = list(executor.map(prepare_and_run, interpreters)) + + assert len({shim.parent for shim, _output in results}) == len(interpreters) + assert {output for _shim, output in results} == { + f"interpreter-{index}" for index in range(len(interpreters)) + } + assert list((tmp_path / "runtime").rglob(".python-*")) == [] + + reused_env = {"PYTHON_BIN": str(interpreters[0]), "PATH": "/usr/bin:/bin"} + controller.prepare_python_command_shim(reused_env, runtime_root=tmp_path / "runtime") + reused_env["PYTHON_BIN"] = str(interpreters[1]) + latest = controller.prepare_python_command_shim(reused_env, runtime_root=tmp_path / "runtime") + managed_path_entries = [ + Path(part) + for part in reused_env["PATH"].split(controller.os.pathsep) + if Path(part).parent == tmp_path / "runtime" / "python-command-shims" + ] + assert managed_path_entries == [latest.parent] + + +def test_preview_python_command_shim_rejects_symlinked_runtime_directory(tmp_path): + controller = load_controller() + runtime_root = tmp_path / "runtime" + runtime_root.mkdir() + external = tmp_path / "external" + external.mkdir() + (runtime_root / "python-command-shims").symlink_to(external, target_is_directory=True) + env = {"PYTHON_BIN": sys.executable, "PATH": "/usr/bin:/bin"} + + with pytest.raises(RuntimeError, match="not a real directory"): + controller.prepare_python_command_shim(env, runtime_root=runtime_root) + + assert list(external.iterdir()) == [] + + +def test_preview_python_command_shim_rejects_non_python_executable(tmp_path): + controller = load_controller() + env = {"PYTHON_BIN": "/bin/sh", "PATH": "/usr/bin:/bin"} + + with pytest.raises(RuntimeError, match="not Python 3"): + controller.prepare_python_command_shim(env, runtime_root=tmp_path / "runtime") + + def test_unlabeled_event_stops_tracked_preview(tmp_path): controller = load_controller() config = make_config(controller, tmp_path) diff --git a/tests/python/runtime/test_prereq_policy.py b/tests/python/runtime/test_prereq_policy.py index eeab8e36a..e9bbab344 100644 --- a/tests/python/runtime/test_prereq_policy.py +++ b/tests/python/runtime/test_prereq_policy.py @@ -13,6 +13,37 @@ class PrereqPolicyTests(unittest.TestCase): + def test_docker_application_mode_requires_docker_without_dependencies(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + repo = Path(tmpdir) / "repo" + runtime = Path(tmpdir) / "runtime" + (repo / ".git").mkdir(parents=True, exist_ok=True) + config = load_config( + { + "RUN_REPO_ROOT": str(repo), + "RUN_SH_RUNTIME_DIR": str(runtime), + "POSTGRES_MAIN_ENABLE": "false", + "REDIS_ENABLE": "false", + "SUPABASE_MAIN_ENABLE": "false", + "N8N_ENABLE": "false", + } + ) + route = parse_route(["start", "--main", "--docker"], env={}) + + def fake_which(binary: str) -> str | None: + if binary == "docker": + return None + return f"/usr/bin/{binary}" + + with ( + patch("envctl_engine.runtime.cli.shutil.which", side_effect=fake_which), + patch("envctl_engine.runtime.cli._python_dependency_available", return_value=True), + ): + ok, reason = cli.check_prereqs(route, config) + + self.assertFalse(ok) + self.assertIn("docker", str(reason).lower()) + def test_main_start_without_requirements_does_not_require_docker(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo = Path(tmpdir) / "repo" diff --git a/tests/python/runtime/test_runtime_browser_diagnostics.py b/tests/python/runtime/test_runtime_browser_diagnostics.py index e001c78fe..7a80f5626 100644 --- a/tests/python/runtime/test_runtime_browser_diagnostics.py +++ b/tests/python/runtime/test_runtime_browser_diagnostics.py @@ -11,6 +11,7 @@ from envctl_engine.config import DependencyEnvTemplateEntry, load_config from envctl_engine.runtime.command_router import parse_route from envctl_engine.runtime.endpoints_command_support import build_endpoints_payload +from envctl_engine.runtime.browser_runtime_diagnostics import project_names from envctl_engine.runtime.engine_runtime import PythonEngineRuntime from envctl_engine.startup.finalization import build_success_run_state from envctl_engine.startup.session import StartupSession @@ -43,6 +44,18 @@ def pid_owns_port(self, _pid: int, _port: int) -> bool: class RuntimeBrowserDiagnosticsTests(unittest.TestCase): + def test_requirement_collision_storage_alias_is_not_exposed_as_browser_project(self) -> None: + state = RunState( + run_id="run-collision", + mode="main", + requirements={ + "Main": RequirementsResult(project="Main"), + "Main Restart Collision": RequirementsResult(project="Main"), + }, + ) + + self.assertEqual(project_names(state, runtime=None), ["Main"]) + def _runtime_with_state(self, state: RunState, *, env: dict[str, str] | None = None) -> PythonEngineRuntime: tmpdir = tempfile.TemporaryDirectory() self.addCleanup(tmpdir.cleanup) diff --git a/tests/python/runtime/test_runtime_context_protocols.py b/tests/python/runtime/test_runtime_context_protocols.py index 60ee8e62a..66a1b1961 100644 --- a/tests/python/runtime/test_runtime_context_protocols.py +++ b/tests/python/runtime/test_runtime_context_protocols.py @@ -158,7 +158,6 @@ def test_artifact_path_helpers_use_repository_when_available_and_runtime_root_fa def test_command_support_uses_runtime_context_dependency_helpers(self) -> None: support_paths = [ REPO_ROOT / "python/envctl_engine/runtime/engine_runtime_event_support.py", - REPO_ROOT / "python/envctl_engine/planning/plan_agent/workflow.py", REPO_ROOT / "python/envctl_engine/actions/action_test_interrupt_support.py", REPO_ROOT / "python/envctl_engine/runtime/playwright_command_support.py", REPO_ROOT / "python/envctl_engine/runtime/qa_user_command_support.py", diff --git a/tests/python/runtime/test_runtime_scope_isolation.py b/tests/python/runtime/test_runtime_scope_isolation.py index 503af616a..e2e3cc96c 100644 --- a/tests/python/runtime/test_runtime_scope_isolation.py +++ b/tests/python/runtime/test_runtime_scope_isolation.py @@ -13,6 +13,29 @@ class RuntimeScopeIsolationTests(unittest.TestCase): + def test_runtime_uses_repository_canonical_root_through_symlinked_parent(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + real_runtime_dir = root / "real-runtime" + runtime_alias = root / "runtime-alias" + repo = root / "repo" + real_runtime_dir.mkdir() + runtime_alias.symlink_to(real_runtime_dir, target_is_directory=True) + (repo / ".git").mkdir(parents=True) + config = load_config( + { + "RUN_REPO_ROOT": str(repo), + "RUN_SH_RUNTIME_DIR": str(runtime_alias), + } + ) + + runtime = PythonEngineRuntime(config, env={}) + + self.assertEqual(runtime.runtime_root, runtime.state_repository.runtime_root) + self.assertEqual(runtime.runtime_legacy_root, runtime.state_repository.runtime_legacy_root) + self.assertEqual(runtime.runtime_root, config.runtime_scope_dir.resolve()) + self.assertEqual(runtime.port_planner.lock_dir, runtime.runtime_root / "locks") + def test_managed_linked_worktree_uses_main_repo_runtime_scope_and_lock_dir(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -27,9 +50,7 @@ def test_managed_linked_worktree_uses_main_repo_runtime_scope_and_lock_dir(self) (worktree / ".git").write_text(f"gitdir: {gitdir}\n", encoding="utf-8") main_config = load_config({"RUN_REPO_ROOT": str(repo), "RUN_SH_RUNTIME_DIR": str(runtime_dir)}) - worktree_config = load_config( - {"RUN_REPO_ROOT": str(worktree), "RUN_SH_RUNTIME_DIR": str(runtime_dir)} - ) + worktree_config = load_config({"RUN_REPO_ROOT": str(worktree), "RUN_SH_RUNTIME_DIR": str(runtime_dir)}) main_runtime = PythonEngineRuntime(main_config, env={}) worktree_runtime = PythonEngineRuntime(worktree_config, env={"ENVCTL_INVOCATION_CWD": str(worktree)}) @@ -205,6 +226,83 @@ def test_stale_legacy_lock_symlink_is_replaced(self) -> None: (runtime.runtime_root / "locks").resolve(strict=False), ) + def test_scoped_and_read_only_construction_preserve_legacy_artifacts_and_lock_view(self) -> None: + for compat_mode in ("scoped_only", "compat_read_only"): + with self.subTest(compat_mode=compat_mode), tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime_dir = root / "runtime" + repo = root / "repo" + (repo / ".git").mkdir(parents=True) + legacy_root = runtime_dir / "python-engine" + legacy_root.mkdir(parents=True) + sentinel = legacy_root / "run_state.json" + sentinel.write_text("legacy-sentinel", encoding="utf-8") + config = load_config( + { + "RUN_REPO_ROOT": str(repo), + "RUN_SH_RUNTIME_DIR": str(runtime_dir), + } + ) + + runtime = PythonEngineRuntime( + config, + env={"ENVCTL_STATE_COMPAT_MODE": compat_mode}, + ) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "legacy-sentinel") + self.assertFalse((legacy_root / "locks").exists()) + self.assertTrue(runtime.runtime_root.is_dir()) + + def test_construction_rejects_preexisting_scoped_root_symlink_before_external_write(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime_dir = root / "runtime" + repo = root / "repo" + (repo / ".git").mkdir(parents=True) + config = load_config( + { + "RUN_REPO_ROOT": str(repo), + "RUN_SH_RUNTIME_DIR": str(runtime_dir), + } + ) + external = root / "external" + external.mkdir() + sentinel = external / "sentinel.txt" + sentinel.write_text("untouched", encoding="utf-8") + config.runtime_scope_dir.parent.mkdir(parents=True, exist_ok=True) + config.runtime_scope_dir.symlink_to(external, target_is_directory=True) + + with self.assertRaisesRegex(ValueError, "symlink"): + PythonEngineRuntime(config, env={}) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "untouched") + self.assertEqual(list(external.iterdir()), [sentinel]) + + def test_construction_rejects_preexisting_legacy_root_symlink_before_external_write(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime_dir = root / "runtime" + repo = root / "repo" + (repo / ".git").mkdir(parents=True) + external = root / "external" + external.mkdir() + sentinel = external / "sentinel.txt" + sentinel.write_text("untouched", encoding="utf-8") + runtime_dir.mkdir() + (runtime_dir / "python-engine").symlink_to(external, target_is_directory=True) + config = load_config( + { + "RUN_REPO_ROOT": str(repo), + "RUN_SH_RUNTIME_DIR": str(runtime_dir), + } + ) + + with self.assertRaisesRegex(ValueError, "symlink"): + PythonEngineRuntime(config, env={}) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "untouched") + self.assertEqual(list(external.iterdir()), [sentinel]) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/runtime/test_tests_workflow_contract.py b/tests/python/runtime/test_tests_workflow_contract.py index 7cf055902..f05688b25 100644 --- a/tests/python/runtime/test_tests_workflow_contract.py +++ b/tests/python/runtime/test_tests_workflow_contract.py @@ -32,3 +32,9 @@ def test_tests_workflow_keeps_pr_checks_split_by_signal_type() -> None: assert "uv run --extra dev pytest -q" in workflow assert "uv run --extra dev python -m build" in workflow assert "uv tool run ruff check python tests scripts" in workflow + + +def test_tests_workflow_runs_for_dev_and_main_pull_requests() -> None: + workflow = _tests_workflow() + + assert "pull_request:\n branches:\n - dev\n - main" in workflow diff --git a/tests/python/shared/test_import_audit.py b/tests/python/shared/test_import_audit.py index d2d6387d8..e34d32047 100644 --- a/tests/python/shared/test_import_audit.py +++ b/tests/python/shared/test_import_audit.py @@ -160,7 +160,6 @@ ENGINE_ROOT / "planning" / "plan_agent" / "terminal_screen.py", }, "_build_plan_agent_workflow": { - ENGINE_ROOT / "planning" / "plan_agent" / "workflow.py", ENGINE_ROOT / "planning" / "plan_agent" / "workflow_build.py", }, "_spawn_omx_session_for_worktree": { @@ -179,11 +178,9 @@ ENGINE_ROOT / "planning" / "plan_agent" / "config.py", }, "_workflow_step_prompt_text": { - ENGINE_ROOT / "planning" / "plan_agent" / "workflow.py", ENGINE_ROOT / "planning" / "plan_agent" / "workflow_prompt_support.py", }, "_resolve_preset_submission_text": { - ENGINE_ROOT / "planning" / "plan_agent" / "workflow.py", ENGINE_ROOT / "planning" / "plan_agent" / "workflow_prompt_support.py", }, "_codex_queue_screen_looks_ready": { @@ -214,13 +211,13 @@ ENGINE_ROOT / "planning" / "plan_agent" / "recovery.py", }, "_codex_goal_text_for_worktree": { - ENGINE_ROOT / "planning" / "plan_agent" / "workflow.py", + ENGINE_ROOT / "planning" / "plan_agent" / "workflow_runtime_support.py", }, "_emit_codex_goal_event": { - ENGINE_ROOT / "planning" / "plan_agent" / "workflow.py", + ENGINE_ROOT / "planning" / "plan_agent" / "workflow_runtime_support.py", }, "_wrap_omx_initial_prompt_for_workflow": { - ENGINE_ROOT / "planning" / "plan_agent" / "workflow.py", + ENGINE_ROOT / "planning" / "plan_agent" / "workflow_runtime_support.py", }, "_queue_failure_event_context": { ENGINE_ROOT / "planning" / "plan_agent" / "recovery.py", diff --git a/tests/python/shared/test_ports_lock_reclamation.py b/tests/python/shared/test_ports_lock_reclamation.py index c6fb9f436..00e9a605e 100644 --- a/tests/python/shared/test_ports_lock_reclamation.py +++ b/tests/python/shared/test_ports_lock_reclamation.py @@ -1,9 +1,13 @@ from __future__ import annotations +import os import json import tempfile +import threading import unittest +from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from unittest.mock import patch REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" @@ -35,7 +39,7 @@ def test_stale_lock_is_reclaimed(self) -> None: lock_path.write_text( json.dumps( { - "owner": "old-worker", + "owner": "worker", "session": "old-session", "pid": 999999, "created_at": "2000-01-01T00:00:00+00:00", @@ -50,11 +54,11 @@ def test_stale_lock_is_reclaimed(self) -> None: session_id="new-session", availability_checker=lambda _port: True, ) - reserved = planner.reserve_next(5432, owner="new-worker") + reserved = planner.reserve_next(5432, owner="worker") self.assertEqual(reserved, 5432) payload = json.loads(lock_path.read_text(encoding="utf-8")) - self.assertEqual(payload["owner"], "new-worker") + self.assertEqual(payload["owner"], "worker") self.assertEqual(payload["session"], "new-session") def test_stale_lock_reclaim_event_includes_previous_owner_metadata(self) -> None: @@ -63,7 +67,7 @@ def test_stale_lock_reclaim_event_includes_previous_owner_metadata(self) -> None lock_path.write_text( json.dumps( { - "owner": "old-worker", + "owner": "worker", "session": "old-session", "pid": 999999, "created_at": "2000-01-01T00:00:00+00:00", @@ -84,18 +88,66 @@ def on_event(name: str, payload: dict[str, object]) -> None: availability_checker=lambda _port: True, event_handler=on_event, ) - reserved = planner.reserve_next(5432, owner="new-worker") + reserved = planner.reserve_next(5432, owner="worker") self.assertEqual(reserved, 5432) reclaim_events = [payload for name, payload in events if name == "port.lock.reclaim"] self.assertTrue(reclaim_events) latest = reclaim_events[-1] self.assertEqual(latest.get("port"), 5432) - self.assertEqual(latest.get("owner"), "new-worker") - self.assertEqual(latest.get("reclaimed_owner"), "old-worker") + self.assertEqual(latest.get("owner"), "worker") + self.assertEqual(latest.get("reclaimed_owner"), "worker") self.assertEqual(latest.get("reclaimed_session"), "old-session") self.assertEqual(latest.get("reclaimed_pid"), 999999) + def test_dead_foreign_owner_lock_is_preserved_and_reservation_rebounds(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "5432.lock" + original = json.dumps( + { + "owner": "Other:backend", + "session": "historical-session", + "pid": 99999999, + "created_at": 0, + }, + sort_keys=True, + ) + lock_path.write_text(original, encoding="utf-8") + planner = PortPlanner( + lock_dir=tmpdir, + session_id="selected-session", + availability_checker=lambda _port: True, + ) + + reserved = planner.reserve_next(5432, owner="Selected:backend") + + self.assertEqual(reserved, 5433) + self.assertEqual(lock_path.read_text(encoding="utf-8"), original) + + def test_dead_allocator_lock_is_preserved_when_same_owner_service_still_listens(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "5432.lock" + original = json.dumps( + { + "owner": "Main:backend", + "session": "historical-session", + "pid": 99999999, + "created_at": 0, + }, + sort_keys=True, + ) + lock_path.write_text(original, encoding="utf-8") + planner = PortPlanner( + lock_dir=tmpdir, + session_id="replacement-session", + availability_checker=lambda port: port != 5432, + ) + + reserved = planner.reserve_next(5432, owner="Main:backend") + + self.assertEqual(reserved, 5433) + self.assertEqual(lock_path.read_text(encoding="utf-8"), original) + def test_active_pid_lock_is_not_reclaimed_by_age_alone(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: lock_path = Path(tmpdir) / "5432.lock" @@ -177,6 +229,308 @@ def test_release_session_isolated_to_own_session(self) -> None: self.assertFalse((Path(tmpdir) / f"{a_port}.lock").exists()) self.assertTrue((Path(tmpdir) / f"{b_port}.lock").exists()) + def test_concurrent_threads_never_publish_duplicate_reservations(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + worker_count = 64 + barrier = threading.Barrier(worker_count) + + def reserve(index: int) -> int: + planner = PortPlanner( + lock_dir=tmpdir, + session_id=f"session-{index}", + availability_checker=lambda _port: True, + ) + barrier.wait() + return planner.reserve_next(12000, owner=f"worker-{index}") + + with ThreadPoolExecutor(max_workers=worker_count) as executor: + reserved = list(executor.map(reserve, range(worker_count))) + + self.assertEqual(len(set(reserved)), worker_count) + self.assertEqual(len(list(Path(tmpdir).glob("*.lock"))), worker_count) + PortPlanner(lock_dir=tmpdir, session_id="cleanup").release_all() + + def test_concurrent_stale_reclaim_is_serialized(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "13000.lock" + lock_path.write_text( + json.dumps({"owner": "worker-0", "session": "dead", "pid": 99999999, "created_at": 0}), + encoding="utf-8", + ) + worker_count = 32 + barrier = threading.Barrier(worker_count) + + def reserve(index: int) -> int: + planner = PortPlanner( + lock_dir=tmpdir, + session_id=f"session-{index}", + availability_checker=lambda _port: True, + ) + barrier.wait() + return planner.reserve_next(13000, owner=f"worker-{index}") + + with ThreadPoolExecutor(max_workers=worker_count) as executor: + reserved = list(executor.map(reserve, range(worker_count))) + + self.assertEqual(len(set(reserved)), worker_count) + self.assertEqual(len(list(Path(tmpdir).glob("*.lock"))), worker_count) + PortPlanner(lock_dir=tmpdir, session_id="cleanup").release_all() + + def test_lock_path_is_linked_only_after_payload_is_complete(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + observed: dict[str, object] = {} + real_link = os.link + + def inspected_link(source: str | Path, destination: str | Path) -> None: + source_path = Path(source) + destination_path = Path(destination) + observed["destination_preexisted"] = destination_path.exists() + observed["payload"] = json.loads(source_path.read_text(encoding="utf-8")) + real_link(source_path, destination_path) + + planner = PortPlanner( + lock_dir=tmpdir, + session_id="complete-publication", + availability_checker=lambda _port: True, + ) + with ( + patch("envctl_engine.shared.ports.os.link", side_effect=inspected_link), + patch("envctl_engine.shared.ports.fsync_directory") as fsync_directory, + ): + reserved = planner.reserve_next(14000, owner="worker") + + self.assertEqual(reserved, 14000) + self.assertFalse(observed["destination_preexisted"]) + self.assertEqual(observed["payload"], json.loads((Path(tmpdir) / "14000.lock").read_text())) + self.assertEqual(list((Path(tmpdir) / ".port-guards").glob("*.pending")), []) + fsync_directory.assert_called_once_with(Path(tmpdir)) + + def test_corrupt_ownerless_lock_requires_explicit_reap_after_grace_period(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + now = [1000.0] + lock_path = Path(tmpdir) / "15000.lock" + lock_path.write_text("{}", encoding="utf-8") + os.utime(lock_path, (now[0], now[0])) + planner = PortPlanner( + lock_dir=tmpdir, + session_id="new", + availability_checker=lambda _port: True, + time_provider=lambda: now[0], + corrupt_lock_grace_seconds=2.0, + ) + + self.assertEqual(planner.reserve_next(15000, owner="new"), 15001) + planner.release_session() + now[0] += 3.0 + self.assertEqual(planner.reserve_next(15000, owner="new"), 15001) + planner.release_session() + self.assertTrue(planner.reap_stale(15000)) + self.assertEqual(planner.reserve_next(15000, owner="new"), 15000) + + def test_permission_error_means_lock_owner_is_still_live(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "16000.lock" + lock_path.write_text( + json.dumps({"owner": "foreign", "session": "foreign", "pid": 424242, "created_at": 0}), + encoding="utf-8", + ) + planner = PortPlanner( + lock_dir=tmpdir, + session_id="new", + availability_checker=lambda _port: True, + ) + + with patch("envctl_engine.shared.ports.os.kill", side_effect=PermissionError("denied")): + self.assertEqual(planner.reserve_next(16000, owner="new"), 16001) + + def test_boolean_pid_is_corrupt_data_not_a_live_pid_one_reservation(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "16010.lock" + lock_path.write_text( + json.dumps({"owner": "broken", "session": "broken", "pid": True, "created_at": False}), + encoding="utf-8", + ) + os.utime(lock_path, (0, 0)) + planner = PortPlanner( + lock_dir=tmpdir, + session_id="new", + availability_checker=lambda _port: True, + pid_checker=lambda _pid: self.fail("a JSON boolean must not be treated as PID 1"), + corrupt_lock_grace_seconds=0.0, + ) + + self.assertEqual(planner.reserve_next(16010, owner="broken"), 16010) + + def test_failed_pid_probe_preserves_a_foreign_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "16020.lock" + lock_path.write_text( + json.dumps({"owner": "foreign", "session": "foreign", "pid": 424242, "created_at": 0}), + encoding="utf-8", + ) + planner = PortPlanner( + lock_dir=tmpdir, + session_id="new", + availability_checker=lambda _port: True, + pid_checker=lambda _pid: (_ for _ in ()).throw(OSError("probe unavailable")), + ) + + self.assertEqual(planner.reserve_next(16020, owner="new"), 16021) + self.assertEqual(json.loads(lock_path.read_text(encoding="utf-8"))["owner"], "foreign") + + def test_non_finite_corrupt_timestamp_cannot_pin_a_port_forever(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "16030.lock" + lock_path.write_text( + '{"owner":"broken","session":"broken","pid":false,"created_at":"nan"}', + encoding="utf-8", + ) + os.utime(lock_path, (0, 0)) + planner = PortPlanner( + lock_dir=tmpdir, + session_id="new", + availability_checker=lambda _port: True, + corrupt_lock_grace_seconds=0.0, + ) + + self.assertEqual(planner.reserve_next(16030, owner="broken"), 16030) + + def test_normal_release_is_session_qualified_but_release_all_is_explicitly_scoped(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + owner = PortPlanner( + lock_dir=tmpdir, + session_id="session-a", + availability_checker=lambda _port: True, + ) + foreign = PortPlanner( + lock_dir=tmpdir, + session_id="session-b", + availability_checker=lambda _port: True, + ) + port = owner.reserve_next(17000, owner="Main:backend") + + foreign.release(port, owner="Main:backend") + self.assertTrue((Path(tmpdir) / f"{port}.lock").exists()) + foreign.release_all() + self.assertFalse((Path(tmpdir) / f"{port}.lock").exists()) + + def test_reap_stale_never_removes_a_live_foreign_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "18000.lock" + lock_path.write_text( + json.dumps({"owner": "worker", "session": "old", "pid": os.getpid(), "created_at": 0}), + encoding="utf-8", + ) + planner = PortPlanner(lock_dir=tmpdir, session_id="new", availability_checker=lambda _port: True) + + self.assertFalse(planner.reap_stale(18000, owner="worker")) + self.assertTrue(lock_path.exists()) + lock_path.write_text( + json.dumps({"owner": "worker", "session": "old", "pid": 99999999, "created_at": 0}), + encoding="utf-8", + ) + self.assertTrue(planner.reap_stale(18000, owner="worker")) + self.assertFalse(lock_path.exists()) + + def test_release_owned_requires_exact_owner_and_original_session(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + original = PortPlanner( + lock_dir=tmpdir, + session_id="old-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + port = original.reserve_next(19000, owner="Main:backend") + + self.assertFalse(cleanup.release_owned(port, "Other:backend", expected_session="old-session")) + self.assertTrue((Path(tmpdir) / f"{port}.lock").exists()) + self.assertFalse(cleanup.release_owned(port, "Main:backend", expected_session="cleanup-session")) + self.assertFalse(cleanup.release_owned(port, "Main:backend", expected_session="")) + self.assertTrue((Path(tmpdir) / f"{port}.lock").exists()) + self.assertTrue(cleanup.release_owned(port, "Main:backend", expected_session="old-session")) + self.assertFalse((Path(tmpdir) / f"{port}.lock").exists()) + + def test_bulk_stale_reap_preserves_live_and_non_numeric_locks(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + (root / "19100.lock").write_text( + json.dumps({"owner": "dead", "session": "old", "pid": 99999999, "created_at": 0}), + encoding="utf-8", + ) + (root / "19101.lock").write_text( + json.dumps({"owner": "live", "session": "live", "pid": os.getpid(), "created_at": 0}), + encoding="utf-8", + ) + (root / "19102.lock").write_text("{}", encoding="utf-8") + os.utime(root / "19102.lock", (0, 0)) + (root / "not-a-port.lock").write_text("invalid", encoding="utf-8") + planner = PortPlanner(lock_dir=tmpdir, session_id="cleanup", corrupt_lock_grace_seconds=1.0) + + self.assertEqual(planner.reap_stale_locks(), 2) + self.assertFalse((root / "19100.lock").exists()) + self.assertTrue((root / "19101.lock").exists()) + self.assertFalse((root / "19102.lock").exists()) + self.assertTrue((root / "not-a-port.lock").exists()) + + def test_event_handler_failure_cannot_hide_a_successful_acquire_or_release(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="event-failure", + availability_checker=lambda _port: True, + event_handler=lambda _name, _payload: (_ for _ in ()).throw(RuntimeError("telemetry failed")), + ) + + port = planner.reserve_next(19200, owner="worker") + + self.assertEqual(port, 19200) + self.assertTrue((Path(tmpdir) / "19200.lock").exists()) + planner.release(port, owner="worker") + self.assertFalse((Path(tmpdir) / "19200.lock").exists()) + + def test_lock_and_guard_directories_reject_symlinks(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + external = root / "external" + external.mkdir() + lock_link = root / "lock-link" + lock_link.symlink_to(external, target_is_directory=True) + + with self.assertRaisesRegex(OSError, "not a real directory"): + PortPlanner(lock_dir=str(lock_link)) + + lock_dir = root / "locks" + lock_dir.mkdir() + (lock_dir / ".port-guards").symlink_to(external, target_is_directory=True) + planner = PortPlanner( + lock_dir=str(lock_dir), + session_id="symlink-guard", + availability_checker=lambda _port: True, + ) + with self.assertRaisesRegex(OSError, "not a real directory"): + planner.reserve_next(19300, owner="worker") + self.assertEqual(list(external.iterdir()), []) + + def test_guard_artifacts_are_bounded_by_the_fixed_shard_count(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="bounded", + availability_checker=lambda _port: True, + ) + for port in range(20000, 20128): + self.assertEqual(planner.reserve_next(port, owner=f"worker-{port}"), port) + planner.release(port, owner=f"worker-{port}") + + guard_dir = Path(tmpdir) / ".port-guards" + self.assertLessEqual(len(list(guard_dir.glob("*.guard"))), 64) + self.assertEqual(list(guard_dir.glob("*.pending")), []) + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/shared/test_process_cwd.py b/tests/python/shared/test_process_cwd.py new file mode 100644 index 000000000..3843c6d14 --- /dev/null +++ b/tests/python/shared/test_process_cwd.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import subprocess +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from envctl_engine.shared.process_cwd import parse_lsof_cwd, process_cwd + + +class ProcessCwdTests(unittest.TestCase): + def test_parse_lsof_cwd_ignores_pid_record_and_returns_name_record(self) -> None: + self.assertEqual(parse_lsof_cwd("p123\nfcwd\nn/repo/backend\n"), "/repo/backend") + self.assertIsNone(parse_lsof_cwd("p123\nfcwd\n")) + + def test_process_cwd_prefers_proc_symlink_without_spawning_lsof(self) -> None: + with ( + patch("envctl_engine.shared.process_cwd.os.readlink", return_value="/repo/from-proc"), + patch("envctl_engine.shared.process_cwd.subprocess.run") as run_mock, + ): + result = process_cwd(123) + + self.assertEqual(result, "/repo/from-proc") + run_mock.assert_not_called() + + def test_process_cwd_falls_back_to_bounded_noninteractive_lsof(self) -> None: + with ( + patch("envctl_engine.shared.process_cwd.os.readlink", side_effect=OSError), + patch( + "envctl_engine.shared.process_cwd.subprocess.run", + return_value=SimpleNamespace(returncode=0, stdout="p123\nfcwd\nn/repo/from-lsof\n"), + ) as run_mock, + ): + result = process_cwd(123) + + self.assertEqual(result, "/repo/from-lsof") + self.assertEqual( + run_mock.call_args.args[0], + ["lsof", "-a", "-p", "123", "-d", "cwd", "-Fn"], + ) + self.assertEqual(run_mock.call_args.kwargs["timeout"], 1.0) + self.assertIs(run_mock.call_args.kwargs["stdin"], subprocess.DEVNULL) + + def test_process_cwd_returns_none_when_lsof_times_out(self) -> None: + with ( + patch("envctl_engine.shared.process_cwd.os.readlink", side_effect=OSError), + patch( + "envctl_engine.shared.process_cwd.subprocess.run", + side_effect=subprocess.TimeoutExpired(["lsof"], 1.0), + ), + ): + self.assertIsNone(process_cwd(123)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/shared/test_process_runner_launch_policy.py b/tests/python/shared/test_process_runner_launch_policy.py index 216cd46b9..85c50ca03 100644 --- a/tests/python/shared/test_process_runner_launch_policy.py +++ b/tests/python/shared/test_process_runner_launch_policy.py @@ -1,7 +1,11 @@ from __future__ import annotations +import os +import signal import subprocess +import sys import tempfile +import time import unittest from pathlib import Path from types import SimpleNamespace @@ -10,10 +14,313 @@ REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" -from envctl_engine.shared.process_runner import ProcessRunner +from envctl_engine.shared.process_runner import ProcessHandoffCleanupError, ProcessRunner + + +def _force_process_cleanup(process: subprocess.Popen[str] | None) -> None: + if process is None: + return + if process.poll() is None: + try: + os.killpg(process.pid, signal.SIGKILL) + except OSError: + try: + process.kill() + except OSError: + pass + try: + process.wait(timeout=5.0) + except (ChildProcessError, ProcessLookupError, subprocess.TimeoutExpired): + pass + + +def _wait_for_path(path: Path, *, timeout: float = 5.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if path.is_file(): + return + time.sleep(0.01) + raise AssertionError(f"child readiness marker was not created: {path}") + + +def _sigterm_ignoring_command(ready_path: Path) -> list[str]: + source = ( + "import pathlib, signal, time; " + "signal.signal(signal.SIGTERM, signal.SIG_IGN); " + f"pathlib.Path({str(ready_path)!r}).write_text('ready', encoding='utf-8'); " + "time.sleep(60)" + ) + return [sys.executable, "-c", source] class ProcessRunnerLaunchPolicyTests(unittest.TestCase): + def test_unconfirmed_background_cleanup_preserves_pid_authority(self) -> None: + class UnkillableProcess: + pid = 987_654 + returncode = None + + def poll(self) -> None: + return None + + def wait(self, timeout: float | None = None) -> int: + raise subprocess.TimeoutExpired(cmd=["fake-service"], timeout=timeout or 0.0) + + def terminate(self) -> None: + raise PermissionError("terminate denied") + + def kill(self) -> None: + raise PermissionError("kill denied") + + process = UnkillableProcess() + runner = ProcessRunner(emit=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("emit failed"))) + + with ( + patch( + "envctl_engine.shared.process_launch_support.subprocess.Popen", + return_value=process, + ), + patch( + "envctl_engine.shared.process_launch_support.os.killpg", + side_effect=PermissionError("killpg denied"), + ), + self.assertRaises(ProcessHandoffCleanupError) as raised, + ): + runner.start_background(["fake-service"]) + + self.assertIs(raised.exception.process, process) + self.assertEqual(raised.exception.pid, process.pid) + self.assertIsInstance(raised.exception.__cause__, RuntimeError) + self.assertTrue(runner._launch_records[-1].active) # noqa: SLF001 + with patch.object(runner, "is_pid_running", return_value=True): + self.assertEqual(runner.launch_diagnostics_summary()["active_launch_count"], 1) + + def test_unconfirmed_run_cleanup_preserves_pid_authority(self) -> None: + class UnkillableProcess: + pid = 987_655 + returncode = None + + def communicate(self, timeout: float | None = None) -> tuple[str, str]: + _ = timeout + raise KeyboardInterrupt + + def poll(self) -> None: + return None + + def wait(self, timeout: float | None = None) -> int: + raise subprocess.TimeoutExpired(cmd=["fake-command"], timeout=timeout or 0.0) + + def terminate(self) -> None: + raise PermissionError("terminate denied") + + def kill(self) -> None: + raise PermissionError("kill denied") + + process = UnkillableProcess() + runner = ProcessRunner() + + with ( + patch( + "envctl_engine.shared.process_runner.subprocess.Popen", + return_value=process, + ), + patch( + "envctl_engine.shared.process_launch_support.os.killpg", + side_effect=PermissionError("killpg denied"), + ), + self.assertRaises(ProcessHandoffCleanupError) as raised, + ): + runner.run(["fake-command"]) + + self.assertIs(raised.exception.process, process) + self.assertEqual(raised.exception.pid, process.pid) + self.assertIsInstance(raised.exception.__cause__, KeyboardInterrupt) + record = runner._launch_records[-1] # noqa: SLF001 + self.assertEqual(record.launch_intent, "foreground_cleanup_unconfirmed") + self.assertTrue(record.active) + with patch.object(runner, "is_pid_running", return_value=True): + self.assertEqual(runner.launch_diagnostics_summary()["active_launch_count"], 1) + + def test_unconfirmed_timeout_cleanup_never_returns_completed_result(self) -> None: + class UnkillableTimedOutProcess: + pid = 987_656 + returncode = None + + def communicate(self, timeout: float | None = None) -> tuple[str, str]: + raise subprocess.TimeoutExpired(cmd=["fake-command"], timeout=timeout or 0.0) + + def poll(self) -> None: + return None + + def wait(self, timeout: float | None = None) -> int: + raise subprocess.TimeoutExpired(cmd=["fake-command"], timeout=timeout or 0.0) + + def terminate(self) -> None: + raise PermissionError("terminate denied") + + def kill(self) -> None: + raise PermissionError("kill denied") + + process = UnkillableTimedOutProcess() + runner = ProcessRunner() + + with ( + patch( + "envctl_engine.shared.process_runner.subprocess.Popen", + return_value=process, + ), + patch( + "envctl_engine.shared.process_runner.os.killpg", + side_effect=PermissionError("killpg denied"), + ), + patch( + "envctl_engine.shared.process_launch_support.os.killpg", + side_effect=PermissionError("killpg denied"), + ), + self.assertRaises(ProcessHandoffCleanupError) as raised, + ): + runner.run(["fake-command"], timeout=0.01) + + self.assertIs(raised.exception.process, process) + self.assertEqual(raised.exception.pid, process.pid) + self.assertIsInstance(raised.exception.__cause__, subprocess.TimeoutExpired) + self.assertEqual( + runner._launch_records[-1].launch_intent, # noqa: SLF001 + "foreground_cleanup_unconfirmed", + ) + + @unittest.skipUnless(hasattr(os, "killpg"), "requires POSIX process groups") + def test_interactive_emit_failure_terminates_real_child_before_reraising(self) -> None: + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[str]] = [] + + with tempfile.TemporaryDirectory() as tmpdir: + ready_path = Path(tmpdir) / "ready" + command = _sigterm_ignoring_command(ready_path) + + def capture_popen(*args, **kwargs): # noqa: ANN002, ANN003, ANN202 + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + def fail_emit(_event: str, **_payload: object) -> None: + _wait_for_path(ready_path) + raise RuntimeError("emit interrupted") + + runner = ProcessRunner(emit=fail_emit) + try: + with ( + patch( + "envctl_engine.shared.process_launch_support.subprocess.Popen", + side_effect=capture_popen, + ), + self.assertRaisesRegex(RuntimeError, "emit interrupted"), + ): + runner.start_interactive_child(command) + + self.assertEqual(len(spawned), 1) + self.assertEqual(spawned[0].poll(), -signal.SIGKILL) + finally: + _force_process_cleanup(spawned[0] if spawned else None) + + @unittest.skipUnless(hasattr(os, "killpg"), "requires POSIX process groups") + def test_start_background_emit_failures_terminate_real_child_before_reraising(self) -> None: + real_popen = subprocess.Popen + + for failure_type in (RuntimeError, KeyboardInterrupt): + with self.subTest(failure_type=failure_type.__name__): + with tempfile.TemporaryDirectory() as tmpdir: + ready_path = Path(tmpdir) / "ready" + command = _sigterm_ignoring_command(ready_path) + spawned: list[subprocess.Popen[str]] = [] + + def capture_popen(*args, **kwargs): # noqa: ANN002, ANN003, ANN202 + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + def fail_emit(_event: str, **_payload: object) -> None: + _wait_for_path(ready_path) + raise failure_type("emit interrupted") + + runner = ProcessRunner(emit=fail_emit) + try: + with ( + patch( + "envctl_engine.shared.process_launch_support.subprocess.Popen", + side_effect=capture_popen, + ), + self.assertRaises(failure_type), + ): + runner.start_background(command) + + self.assertEqual(len(spawned), 1) + self.assertEqual(spawned[0].poll(), -signal.SIGKILL) + self.assertFalse(runner._launch_records[-1].active) # noqa: SLF001 + self.assertEqual(runner.launch_diagnostics_summary()["active_launch_count"], 0) + finally: + _force_process_cleanup(spawned[0] if spawned else None) + + @unittest.skipUnless(hasattr(os, "killpg"), "requires POSIX process groups") + def test_run_communicate_failures_terminate_real_child_before_reraising(self) -> None: + real_popen = subprocess.Popen + + for failure_type in (RuntimeError, KeyboardInterrupt): + with self.subTest(failure_type=failure_type.__name__): + with tempfile.TemporaryDirectory() as tmpdir: + ready_path = Path(tmpdir) / "ready" + command = _sigterm_ignoring_command(ready_path) + spawned: list[subprocess.Popen[str]] = [] + + class InterruptingProcess: + def __init__(self, process: subprocess.Popen[str]) -> None: + self.process = process + + @property + def pid(self) -> int: + return self.process.pid + + @property + def returncode(self) -> int | None: + return self.process.returncode + + def communicate(self, timeout: float | None = None) -> tuple[str, str]: + _ = timeout + _wait_for_path(ready_path) + raise failure_type("communicate interrupted") + + def poll(self) -> int | None: + return self.process.poll() + + def wait(self, timeout: float | None = None) -> int: + return self.process.wait(timeout=timeout) + + def terminate(self) -> None: + self.process.terminate() + + def kill(self) -> None: + self.process.kill() + + def capture_popen(*args, **kwargs): # noqa: ANN002, ANN003, ANN202 + process = real_popen(*args, **kwargs) + spawned.append(process) + return InterruptingProcess(process) + + runner = ProcessRunner() + try: + with ( + patch( + "envctl_engine.shared.process_runner.subprocess.Popen", + side_effect=capture_popen, + ), + self.assertRaises(failure_type), + ): + runner.run(command) + + self.assertEqual(len(spawned), 1) + self.assertEqual(spawned[0].poll(), -signal.SIGKILL) + finally: + _force_process_cleanup(spawned[0] if spawned else None) + def test_start_background_denies_controller_input_and_emits_launch_policy(self) -> None: events: list[tuple[str, dict[str, object]]] = [] runner = ProcessRunner(emit=lambda event, **payload: events.append((event, dict(payload)))) diff --git a/tests/python/shared/test_process_runner_listener_detection.py b/tests/python/shared/test_process_runner_listener_detection.py index cfa9824d7..3ba0f35c6 100644 --- a/tests/python/shared/test_process_runner_listener_detection.py +++ b/tests/python/shared/test_process_runner_listener_detection.py @@ -1,7 +1,9 @@ from __future__ import annotations +import os import signal import subprocess +import sys import unittest from pathlib import Path from unittest.mock import patch @@ -11,6 +13,11 @@ from envctl_engine.shared.process_runner import ProcessRunner +def _signal_then_report_process_group_gone(_pid: int, sig: int) -> None: + if sig == 0: + raise ProcessLookupError + + class ProcessRunnerListenerDetectionTests(unittest.TestCase): def _fake_run_for_ports(self, ports_by_pid: dict[int, list[int]]): def _run(cmd, **kwargs): # noqa: ANN001 @@ -162,9 +169,15 @@ class _FakeProcess: def communicate(self, timeout=None): # noqa: ANN001 raise subprocess.TimeoutExpired(cmd=["docker", "run"], timeout=120.0) + def wait(self, timeout=None): # noqa: ANN001 + return 0 + with ( patch("envctl_engine.shared.process_runner.subprocess.Popen", return_value=_FakeProcess()), - patch("envctl_engine.shared.process_runner.os.killpg"), + patch( + "envctl_engine.shared.process_runner.os.killpg", + side_effect=_signal_then_report_process_group_gone, + ), ): result = runner.run(["docker", "run"], timeout=120.0) @@ -191,9 +204,15 @@ def communicate(self, timeout=None): # noqa: ANN001 ) return ("tail stdout", "tail stderr") + def wait(self, timeout=None): # noqa: ANN001 + return 0 + with ( patch("envctl_engine.shared.process_runner.subprocess.Popen", return_value=_FakeProcess()), - patch("envctl_engine.shared.process_runner.os.killpg"), + patch( + "envctl_engine.shared.process_runner.os.killpg", + side_effect=_signal_then_report_process_group_gone, + ), ): result = runner.run(["docker", "run"], timeout=30.0) @@ -212,9 +231,15 @@ class _FakeProcess: def communicate(self, timeout=None): # noqa: ANN001 raise subprocess.TimeoutExpired(cmd=["docker", "run"], timeout=5.0) + def wait(self, timeout=None): # noqa: ANN001 + return 0 + with ( patch("envctl_engine.shared.process_runner.subprocess.Popen", return_value=_FakeProcess()), - patch("envctl_engine.shared.process_runner.os.killpg") as killpg, + patch( + "envctl_engine.shared.process_runner.os.killpg", + side_effect=_signal_then_report_process_group_gone, + ) as killpg, ): result = runner.run(["docker", "run"], timeout=5.0) @@ -274,7 +299,9 @@ def _fake_killpg(pid: int, sig: int) -> None: with ( patch("envctl_engine.shared.process_runner.os.killpg", side_effect=_fake_killpg), - patch.object(runner, "is_pid_running", return_value=False), + patch("envctl_engine.shared.process_runner.os.getpgid", side_effect=lambda pid: pid), + patch.object(runner, "_process_group_member_pids", return_value={4321}), + patch.object(runner, "_wait_for_process_group_exit", return_value=True), patch.object(runner, "_pid_identity", return_value="orig", create=True), ): result = runner.terminate_process_group(4321, term_timeout=0.1, kill_timeout=0.1) @@ -291,13 +318,10 @@ def _fake_killpg(pid: int, sig: int) -> None: with ( patch("envctl_engine.shared.process_runner.os.killpg", side_effect=_fake_killpg), - patch.object(runner, "is_pid_running", side_effect=[True, True, False]), + patch("envctl_engine.shared.process_runner.os.getpgid", side_effect=lambda pid: pid), + patch.object(runner, "_process_group_member_pids", return_value={5678}), + patch.object(runner, "_wait_for_process_group_exit", side_effect=[False, True]), patch.object(runner, "_pid_identity", side_effect=["orig"] * 8, create=True), - patch( - "envctl_engine.shared.process_runner.time.monotonic", - side_effect=[0.0, 0.0, 0.2, 0.2, 0.2, 0.4], - ), - patch("envctl_engine.shared.process_runner.time.sleep", return_value=None), ): result = runner.terminate_process_group(5678, term_timeout=0.1, kill_timeout=0.1) @@ -313,19 +337,120 @@ def _fake_killpg(pid: int, sig: int) -> None: with ( patch("envctl_engine.shared.process_runner.os.killpg", side_effect=_fake_killpg), - patch.object(runner, "is_pid_running", return_value=True), + patch("envctl_engine.shared.process_runner.os.getpgid", side_effect=lambda pid: pid), + patch.object(runner, "_process_group_member_pids", return_value={8765}), + patch.object(runner, "_wait_for_process_group_exit", return_value=False), patch.object(runner, "_pid_identity", side_effect=["orig", "reused"], create=True), - patch( - "envctl_engine.shared.process_runner.time.monotonic", - side_effect=[0.0, 0.0, 0.2, 0.2, 0.2, 0.4], - ), - patch("envctl_engine.shared.process_runner.time.sleep", return_value=None), ): result = runner.terminate_process_group(8765, term_timeout=0.1, kill_timeout=0.1) - self.assertTrue(result) + self.assertFalse(result) self.assertEqual(killpg_calls, [(8765, signal.SIGTERM)]) + def test_terminate_process_group_kills_term_ignoring_descendant_after_leader_exits(self) -> None: + runner = ProcessRunner() + parent = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "import subprocess,sys,time; " + "subprocess.Popen([sys.executable, '-c', " + "'import os,signal,time; signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "print(os.getpid(), flush=True); time.sleep(60)']); " + "time.sleep(60)" + ), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) + child_pid = 0 + try: + assert parent.stdout is not None + child_pid = int(parent.stdout.readline().strip()) + + self.assertTrue(runner.terminate_process_group(parent.pid, term_timeout=0.2, kill_timeout=2.0)) + parent.wait(timeout=2.0) + self.assertFalse(runner.is_pid_running(child_pid)) + finally: + try: + os.killpg(parent.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + def test_terminate_process_group_resolves_dead_leaders_group_from_live_orphan_member(self) -> None: + runner = ProcessRunner() + parent = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "import os,time; " + "child=os.fork(); " + "print(child, flush=True) if child else None; " + "os._exit(0) if child else time.sleep(60)" + ), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) + child_pid = 0 + try: + assert parent.stdout is not None + child_pid = int(parent.stdout.readline().strip()) + parent.wait(timeout=2.0) + self.assertEqual(os.getpgid(child_pid), parent.pid) + + self.assertTrue(runner.terminate_process_group(child_pid, term_timeout=0.5, kill_timeout=2.0)) + self.assertFalse(runner.is_pid_running(child_pid)) + finally: + try: + os.killpg(parent.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + def test_terminate_process_group_does_not_treat_live_eperm_as_success(self) -> None: + runner = ProcessRunner() + + with ( + patch("envctl_engine.shared.process_runner.os.killpg", side_effect=PermissionError), + patch("envctl_engine.shared.process_runner.os.getpgid", return_value=8766), + patch.object(runner, "is_pid_running", return_value=True), + patch.object(runner, "_pid_identity", return_value="orig", create=True), + ): + result = runner.terminate_process_group(8766, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + + def test_terminate_process_group_does_not_treat_missing_group_for_live_pid_as_success(self) -> None: + runner = ProcessRunner() + + with ( + patch("envctl_engine.shared.process_runner.os.killpg", side_effect=ProcessLookupError), + patch("envctl_engine.shared.process_runner.os.getpgid", return_value=8767), + patch.object(runner, "is_pid_running", return_value=True), + patch.object(runner, "_pid_identity", return_value="orig", create=True), + ): + result = runner.terminate_process_group(8767, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + + def test_terminate_does_not_treat_live_eperm_as_success(self) -> None: + runner = ProcessRunner() + + with ( + patch("envctl_engine.shared.process_runner.os.kill", side_effect=PermissionError), + patch.object(runner, "is_pid_running", return_value=True), + patch.object(runner, "_pid_identity", return_value="orig", create=True), + ): + result = runner.terminate(8768, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + def test_is_pid_running_treats_zombie_process_as_not_running(self) -> None: runner = ProcessRunner() @@ -340,6 +465,12 @@ def test_is_pid_running_treats_zombie_process_as_not_running(self) -> None: ): self.assertFalse(runner.is_pid_running(4321)) + def test_is_pid_running_treats_permission_denied_as_still_running(self) -> None: + runner = ProcessRunner() + + with patch("envctl_engine.shared.process_runner.os.kill", side_effect=PermissionError): + self.assertTrue(runner.is_pid_running(4321)) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/shared/test_process_termination.py b/tests/python/shared/test_process_termination.py new file mode 100644 index 000000000..190c4a477 --- /dev/null +++ b/tests/python/shared/test_process_termination.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import os +import signal +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from envctl_engine.shared.process_termination import terminate_pid + + +class ProcessTerminationTests(unittest.TestCase): + def test_terminate_pid_refuses_self_and_parent_without_signalling(self) -> None: + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: self.fail("protected PID must not be signalled"), + ) + + self.assertFalse(terminate_pid(os.getpid(), process_runner=runner, term_timeout=0.0, kill_timeout=0.0)) + self.assertFalse(terminate_pid(os.getppid(), process_runner=runner, term_timeout=0.0, kill_timeout=0.0)) + + def test_failed_group_termination_does_not_kill_only_the_leader(self) -> None: + runner_calls: list[str] = [] + sent: list[signal.Signals] = [] + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: runner_calls.append("group") or False, + terminate=lambda *_args, **_kwargs: runner_calls.append("single") or False, + ) + + with ( + patch( + "envctl_engine.shared.process_termination._send_signal", + side_effect=lambda _pid, requested_signal: sent.append(requested_signal) or True, + ), + patch("envctl_engine.shared.process_termination.wait_for_pid_exit", side_effect=[False, False]), + ): + result = terminate_pid(987654, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + self.assertEqual(runner_calls, ["group"]) + self.assertEqual(sent, []) + + def test_runner_without_group_support_falls_back_to_raw_signals(self) -> None: + sent: list[signal.Signals] = [] + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate=lambda *_args, **_kwargs: False, + ) + + with ( + patch( + "envctl_engine.shared.process_termination._send_signal", + side_effect=lambda _pid, requested_signal: sent.append(requested_signal) or True, + ), + patch("envctl_engine.shared.process_termination.wait_for_pid_exit", side_effect=[False, False]), + ): + result = terminate_pid(987654, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + self.assertEqual(sent, [signal.SIGTERM, signal.SIGKILL]) + + def test_runner_success_is_not_trusted_while_same_identified_pid_remains_live(self) -> None: + sent: list[signal.Signals] = [] + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + _pid_identity=lambda _pid: "original", + terminate_process_group=lambda *_args, **_kwargs: True, + ) + + with patch( + "envctl_engine.shared.process_termination._send_signal", + side_effect=lambda _pid, requested_signal: sent.append(requested_signal) or True, + ): + result = terminate_pid(987655, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + self.assertEqual(sent, []) + + def test_identity_incapable_runner_success_is_not_trusted_while_pid_remains_live(self) -> None: + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: True, + ) + + result = terminate_pid(987657, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + + def test_identity_replacement_after_failed_group_cleanup_is_not_reported_as_success(self) -> None: + identities = iter(("original", "replacement")) + sent: list[signal.Signals] = [] + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + _pid_identity=lambda _pid: next(identities), + terminate_process_group=lambda *_args, **_kwargs: False, + ) + + with patch( + "envctl_engine.shared.process_termination._send_signal", + side_effect=lambda _pid, requested_signal: sent.append(requested_signal) or True, + ): + result = terminate_pid(987656, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + self.assertEqual(sent, []) + + def test_group_cleanup_exception_is_success_after_leader_replacement(self) -> None: + identities = iter(("original", "replacement")) + sent: list[signal.Signals] = [] + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + _pid_identity=lambda _pid: next(identities), + terminate_process_group=lambda *_args, **_kwargs: (_ for _ in ()).throw( + OSError("post-kill reporting failed") + ), + ) + + with patch( + "envctl_engine.shared.process_termination._send_signal", + side_effect=lambda _pid, requested_signal: sent.append(requested_signal) or True, + ): + result = terminate_pid(987658, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertTrue(result) + self.assertEqual(sent, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/shared/test_structure_planning_worktree_layout.py b/tests/python/shared/test_structure_planning_worktree_layout.py index 36bbb178a..9fe34af50 100644 --- a/tests/python/shared/test_structure_planning_worktree_layout.py +++ b/tests/python/shared/test_structure_planning_worktree_layout.py @@ -278,16 +278,6 @@ def test_worktree_main_task_has_owned_module(self) -> None: facade.read_text(encoding="utf-8"), ) - def test_worktree_selection_memory_has_owned_module(self) -> None: - owner = REPO_ROOT / "python" / "envctl_engine" / "planning" / "worktree_selection_memory.py" - facade = REPO_ROOT / "python" / "envctl_engine" / "planning" / "worktree_domain.py" - - self.assertTrue(owner.is_file()) - self.assertIn( - "from envctl_engine.planning.worktree_selection_memory import", - facade.read_text(encoding="utf-8"), - ) - def test_worktree_project_catalog_has_owned_module(self) -> None: owner = REPO_ROOT / "python" / "envctl_engine" / "planning" / "worktree_project_catalog.py" facade = REPO_ROOT / "python" / "envctl_engine" / "planning" / "worktree_domain.py" diff --git a/tests/python/shared/test_structure_runtime_layout.py b/tests/python/shared/test_structure_runtime_layout.py index 6a62d805e..85ef02f2d 100644 --- a/tests/python/shared/test_structure_runtime_layout.py +++ b/tests/python/shared/test_structure_runtime_layout.py @@ -43,6 +43,16 @@ def test_runtime_lifecycle_cleanup_has_blast_owner(self) -> None: self.assertIn("LifecycleBlastCleanupSupport", orchestrator_text) self.assertLessEqual(len(orchestrator_text.splitlines()), 760) + def test_runtime_lifecycle_cleanup_has_full_cleanup_owner(self) -> None: + owner = REPO_ROOT / "python" / "envctl_engine" / "runtime" / "lifecycle_full_cleanup.py" + orchestrator = REPO_ROOT / "python" / "envctl_engine" / "runtime" / "lifecycle_cleanup_orchestrator.py" + + self.assertTrue(owner.is_file()) + owner_text = owner.read_text(encoding="utf-8") + orchestrator_text = orchestrator.read_text(encoding="utf-8") + self.assertIn("def clear_runtime_state", owner_text) + self.assertIn("from envctl_engine.runtime.lifecycle_full_cleanup import", orchestrator_text) + def test_runtime_command_parity_tests_are_split_by_owner(self) -> None: runtime_tests = REPO_ROOT / "tests" / "python" / "runtime" expected = [ diff --git a/tests/python/shared/test_utility_consolidation_contract.py b/tests/python/shared/test_utility_consolidation_contract.py index ad80b54ca..10c66f3eb 100644 --- a/tests/python/shared/test_utility_consolidation_contract.py +++ b/tests/python/shared/test_utility_consolidation_contract.py @@ -5,13 +5,19 @@ import tempfile import unittest from pathlib import Path +from types import SimpleNamespace from unittest.mock import patch REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" from envctl_engine.config import _default_port_value from envctl_engine.shared.node_tooling import detect_package_manager, detect_python_bin, load_package_json -from envctl_engine.shared.services import project_name_from_service_name, service_display_name +from envctl_engine.shared.services import ( + project_name_from_service_name, + resolve_service_project_name, + service_display_name, + service_project_name, +) from envctl_engine.shared.parsing import parse_bool, parse_float, parse_int, strip_quotes from envctl_engine.test_output.parser_base import strip_ansi from envctl_engine.ui.capabilities import ( @@ -87,6 +93,46 @@ def test_shared_domain_helpers_cover_services_ports_and_ansi(self) -> None: self.assertEqual(strip_ansi("\x1b]22;default\x07hello\x1b[15;72H"), "hello") self.assertEqual(strip_ansi("\\x1b[48;2;39;39;39m hello \\x1b[0m"), " hello ") + def test_service_project_resolution_prefers_record_metadata_then_legacy_parser(self) -> None: + legacy_calls: list[str] = [] + service = SimpleNamespace(name="Opaque Runtime Process", project="Customer Platform") + + self.assertEqual( + resolve_service_project_name( + service.name, + service, + project_name_from_service=lambda name: legacy_calls.append(name) or "Wrong Project", + ), + "Customer Platform", + ) + self.assertEqual(legacy_calls, []) + + legacy_additional = SimpleNamespace( + name="Customer Platform Voice Runtime", + project=None, + type="voice-runtime", + service_slug="voice-runtime", + ) + self.assertEqual(service_project_name(legacy_additional), "Customer Platform") + self.assertEqual( + resolve_service_project_name( + legacy_additional.name, + legacy_additional, + project_name_from_service=lambda _name: "Customer Platform Voice", + ), + "Customer Platform", + ) + + legacy = SimpleNamespace(name="Legacy Voice Runtime", project=None) + self.assertEqual( + resolve_service_project_name( + legacy.name, + legacy, + project_name_from_service=lambda _name: "Legacy", + ), + "Legacy", + ) + def test_service_display_name_has_one_shared_owner(self) -> None: for relative_path in ( "python/envctl_engine/dashboard_metadata.py", diff --git a/tests/python/startup/test_frontend_projection.py b/tests/python/startup/test_frontend_projection.py index fdaacf7c9..e051c73d3 100644 --- a/tests/python/startup/test_frontend_projection.py +++ b/tests/python/startup/test_frontend_projection.py @@ -62,6 +62,167 @@ def test_runtime_map_tracks_actual_frontend_rebound_port(self) -> None: self.assertEqual(runtime["service_to_actual_port"]["Tree Alpha Frontend"], 9122) self.assertEqual(runtime["port_to_service"][9122], "Tree Alpha Frontend") + def test_projection_uses_project_metadata_for_opaque_core_service_names(self) -> None: + state = RunState( + run_id="run-opaque", + mode="trees", + services={ + "Opaque API Process": ServiceRecord( + name="Opaque API Process", + type="backend", + cwd="/tmp/customer-platform/api", + actual_port=8181, + status="healthy", + project="Customer Platform", + service_slug="backend", + ), + "Opaque Web Process": ServiceRecord( + name="Opaque Web Process", + type="frontend", + cwd="/tmp/customer-platform/web", + actual_port=9191, + status="running", + project="Customer Platform", + service_slug="frontend", + ), + }, + ) + + runtime = build_runtime_map(state) + projection = runtime["projection"]["Customer Platform"] + + self.assertEqual(projection["backend_url"], "http://localhost:8181") + self.assertEqual(projection["frontend_url"], "http://localhost:9191") + self.assertEqual(projection["backend_status"], "healthy") + self.assertEqual(projection["frontend_status"], "running") + self.assertEqual( + frontend_env_for_project(state, "Customer Platform")["VITE_BACKEND_URL"], + "http://localhost:8181", + ) + + def test_collision_projection_keeps_every_instance_and_uses_one_authoritative_record(self) -> None: + collision_backend_name = "Main Backend Restart Collision 64001" + collision_frontend_name = "Main Frontend Restart Collision 64002" + services = { + collision_backend_name: ServiceRecord( + name=collision_backend_name, + type="backend", + cwd="/tmp/main/replacement-backend", + pid=64001, + actual_port=8251, + status="termination_failed", + project="Main", + service_slug="backend", + degraded=True, + ), + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + pid=63001, + actual_port=8000, + status="running", + project="Main", + service_slug="backend", + ), + collision_frontend_name: ServiceRecord( + name=collision_frontend_name, + type="frontend", + cwd="/tmp/main/replacement-frontend", + pid=64002, + actual_port=9251, + status="termination_failed", + project="Main", + service_slug="frontend", + degraded=True, + ), + "Main Frontend": ServiceRecord( + name="Main Frontend", + type="frontend", + cwd="/tmp/main/frontend", + pid=63002, + actual_port=9000, + status="healthy", + project="Main", + service_slug="frontend", + ), + } + state = RunState(run_id="run-collision", mode="main", services=services) + + runtime = build_runtime_map(state, host="runtime.test") + project = runtime["projects"]["Main"] + projected_services = project["services"] + projection = runtime["projection"]["Main"] + + self.assertEqual( + set(projected_services), + { + "backend", + f"backend@{collision_backend_name}", + "frontend", + f"frontend@{collision_frontend_name}", + }, + ) + self.assertEqual(projected_services["backend"]["name"], "Main Backend") + self.assertEqual(projected_services["backend"]["port"], 8000) + self.assertEqual(projected_services[f"backend@{collision_backend_name}"]["port"], 8251) + self.assertEqual(projected_services["frontend"]["name"], "Main Frontend") + self.assertEqual(projected_services["frontend"]["port"], 9000) + self.assertEqual(projected_services[f"frontend@{collision_frontend_name}"]["port"], 9251) + + self.assertEqual(project["backend_port"], 8000) + self.assertEqual(projection["backend_port"], 8000) + self.assertEqual(projection["backend_url"], "http://runtime.test:8000") + self.assertEqual(projection["backend_status"], "running") + self.assertEqual(project["frontend_port"], 9000) + self.assertEqual(projection["frontend_port"], 9000) + self.assertEqual(projection["frontend_url"], "http://runtime.test:9000") + self.assertEqual(projection["frontend_status"], "healthy") + + reversed_state = RunState( + run_id="run-collision-reversed", + mode="main", + services=dict(reversed(list(services.items()))), + ) + reversed_runtime = build_runtime_map(reversed_state, host="runtime.test") + + self.assertEqual( + reversed_runtime["projects"]["Main"]["services"], + projected_services, + ) + self.assertEqual(reversed_runtime["projection"]["Main"], projection) + + def test_legacy_collision_suffix_without_project_metadata_stays_under_original_project(self) -> None: + state = RunState( + run_id="run-legacy-collision", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/main", + actual_port=8000, + status="running", + ), + "Main Backend Restart Collision 2": ServiceRecord( + name="Main Backend Restart Collision 2", + type="backend", + cwd="/replacement", + actual_port=8001, + status="termination_failed", + ), + }, + ) + + runtime = build_runtime_map(state) + + self.assertEqual(set(runtime["projects"]), {"Main"}) + self.assertEqual( + set(runtime["projects"]["Main"]["services"]), + {"backend", "backend@Main Backend Restart Collision 2"}, + ) + self.assertEqual(runtime["projection"]["Main"]["backend_port"], 8000) + def test_multi_project_projection_uses_per_project_backend_port(self) -> None: state = RunState( run_id="run-2", diff --git a/tests/python/startup/test_requirements_project_startup.py b/tests/python/startup/test_requirements_project_startup.py new file mode 100644 index 000000000..5769e0f70 --- /dev/null +++ b/tests/python/startup/test_requirements_project_startup.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from envctl_engine.requirements.orchestrator import FailureClass, RequirementOutcome +from envctl_engine.shared.ports import PortPlanner +from envctl_engine.startup.requirements_component_startup import RequirementComponentStarter +from envctl_engine.startup.requirements_project_startup import RequirementProjectStarter +from envctl_engine.state.models import PortPlan + + +class RequirementProjectStarterTests(unittest.TestCase): + @staticmethod + def _plan(port: int) -> PortPlan: + return PortPlan( + project="Main", + requested=port, + assigned=port, + final=port, + source="test", + ) + + def _starter(self, *, parallel: bool) -> RequirementProjectStarter: + postgres = SimpleNamespace( + id="postgres", + resources=(SimpleNamespace(name="db", legacy_port_key="db"),), + ) + redis = SimpleNamespace( + id="redis", + resources=(SimpleNamespace(name="redis", legacy_port_key="redis"),), + ) + + def start_component(_context, component, plan, _reserve_next, **_kwargs): # noqa: ANN001, ANN202 + if component == "redis": + raise RuntimeError("redis native startup crashed") + return RequirementOutcome( + service_name=component, + success=True, + requested_port=plan.requested, + final_port=plan.final, + retries=0, + container_name="postgres-main", + ) + + runtime = SimpleNamespace( + env={"ENVCTL_REQUIREMENTS_PARALLEL": "true" if parallel else "false"}, + config=SimpleNamespace(raw={}, strict_n8n_bootstrap=False), + port_planner=SimpleNamespace(session_id="requirements-session"), + _emit=lambda *_args, **_kwargs: None, + _start_requirement_component=start_component, + ) + return RequirementProjectStarter( + orchestrator=SimpleNamespace(runtime=runtime), + context=SimpleNamespace( + name="Main", + ports={"db": self._plan(5432), "redis": self._plan(6379)}, + ), + mode="main", + definitions=[postgres, redis], + ) + + def test_sequential_component_exception_becomes_partial_failed_outcome(self) -> None: + starter = self._starter(parallel=False) + + starter._run_enabled_components(starter.definitions) + + self.assertTrue(starter.outcomes["postgres"].success) + self.assertFalse(starter.outcomes["redis"].success) + self.assertEqual(starter.outcomes["redis"].final_port, 6379) + self.assertIn("native startup crashed", str(starter.outcomes["redis"].error)) + + def test_parallel_component_exception_waits_for_and_preserves_other_outcomes(self) -> None: + starter = self._starter(parallel=True) + + with patch( + "envctl_engine.startup.requirements_project_startup.requirements_parallel_enabled", + return_value=True, + ): + starter._run_enabled_components(starter.definitions) + + self.assertEqual(set(starter.outcomes), {"postgres", "redis"}) + self.assertTrue(starter.outcomes["postgres"].success) + self.assertFalse(starter.outcomes["redis"].success) + + def test_managed_components_record_exact_port_planner_session(self) -> None: + starter = self._starter(parallel=False) + starter._run_enabled_components(starter.definitions) + + components = starter._build_components( + {"postgres": True, "redis": True}, + {"postgres": False, "redis": False}, + ) + + self.assertEqual( + components["postgres"]["port_lock_session"], + "requirements-session", + ) + self.assertEqual( + components["redis"]["port_lock_session"], + "requirements-session", + ) + + def test_multiple_requirement_rebinds_release_every_failed_port_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="startup-session", + availability_checker=lambda _port: True, + ) + planner.reserve_next(5432, owner="Main:db") + starter = object.__new__(RequirementComponentStarter) + starter.runtime = SimpleNamespace( + port_planner=planner, + requirements=SimpleNamespace(reason_code_for_failure=lambda *_args, **_kwargs: "port_in_use"), + _emit=lambda *_args, **_kwargs: None, + ) + starter.context = SimpleNamespace(name="Main") + starter.name = "postgres" + + retry_one = planner.reserve_next(5433, owner="Main:requirements") + starter._on_retry( + "postgres", + 5432, + retry_one, + 1, + FailureClass.BIND_CONFLICT_RETRYABLE, + "address already in use", + ) + retry_two = planner.reserve_next(5434, owner="Main:requirements") + starter._on_retry( + "postgres", + retry_one, + retry_two, + 2, + FailureClass.BIND_CONFLICT_RETRYABLE, + "address already in use", + ) + + locks = list(Path(tmpdir).glob("*.lock")) + self.assertEqual([path.name for path in locks], ["5434.lock"]) + payload = json.loads(locks[0].read_text(encoding="utf-8")) + self.assertEqual(payload["owner"], "Main:requirements") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/startup/test_restart_prestop_support.py b/tests/python/startup/test_restart_prestop_support.py index 49bc46997..ff706cafb 100644 --- a/tests/python/startup/test_restart_prestop_support.py +++ b/tests/python/startup/test_restart_prestop_support.py @@ -1,11 +1,16 @@ from __future__ import annotations import unittest +import tempfile +from pathlib import Path from types import SimpleNamespace -from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.command_router import Route, parse_route +from envctl_engine.shared.ports import PortPlanner +from envctl_engine.state.models import RunState, ServiceRecord from envctl_engine.startup.restart_prestop_support import ( RestartOrphanListenerScan, + _validate_restart_selection, apply_restart_port_assignments, apply_restart_ports_to_contexts, handle_restart_prestop, @@ -19,6 +24,10 @@ restart_prestop_preservation, restart_start_route, ) +from envctl_engine.startup.startup_selection_support import ( + _restart_service_types_for_project, + trees_start_selection_required, +) class _NullContext: @@ -49,6 +58,389 @@ def fail(self, message: str) -> None: class RestartPrestopSupportTests(unittest.TestCase): + def test_restart_cli_parses_ignore_service_dependencies(self) -> None: + route = parse_route( + ["restart", "--service", "voice-runtime", "--ignore-service-deps"], + env={}, + ) + + self.assertEqual(route.flags["services"], ["voice-runtime"]) + self.assertTrue(route.flags["ignore_service_deps"]) + + def test_restart_prestop_respects_runtime_scope_and_project_intersection(self) -> None: + services = { + f"{project} {display}": ServiceRecord( + name=f"{project} {display}", + type=service_type, + cwd=".", + project=project, + service_slug=service_type, + ) + for project in ("Alpha", "Beta") + for service_type, display in ( + ("backend", "Backend"), + ("frontend", "Frontend"), + ("voice-runtime", "Voice Runtime"), + ) + } + state = SimpleNamespace(services=services) + + cases = ( + ({"runtime_scope": "backend"}, [], {"Alpha Backend", "Beta Backend"}), + ( + {"runtime_scope": "fullstack"}, + [], + {"Alpha Backend", "Alpha Frontend", "Beta Backend", "Beta Frontend"}, + ), + ({"runtime_scope": "dependencies"}, [], set()), + ( + {"services": ["voice-runtime"], "ignore_service_deps": True}, + ["Alpha"], + {"Alpha Voice Runtime"}, + ), + ) + for flags, projects, expected in cases: + with self.subTest(flags=flags, projects=projects): + selection = restart_prestop_selection( + state=state, + route=Route(command="restart", mode="trees", projects=projects, flags=flags), + runtime=SimpleNamespace(_project_name_from_service=lambda _name: "wrong"), + ) + self.assertEqual(selection.selected_services, expected) + + def test_restart_only_and_no_modifiers_keep_prestop_and_launch_in_lockstep(self) -> None: + services = { + f"Main {display}": ServiceRecord( + name=f"Main {display}", + type=service_type, + cwd=".", + project="Main", + service_slug=service_type, + ) + for service_type, display in ( + ("backend", "Backend"), + ("frontend", "Frontend"), + ("voice-runtime", "Voice Runtime"), + ) + } + state = RunState( + run_id="run-1", + mode="main", + services=services, + requirements={"Main": object()}, # type: ignore[dict-item] + metadata={ + "project_roots": {"Main": "/repo"}, + "dashboard_project_configured_services": { + "Main": ["backend", "frontend", "voice-runtime"] + }, + }, + ) + additional = (SimpleNamespace(name="voice-runtime", depends_on=("backend",)),) + runtime = SimpleNamespace( + config=SimpleNamespace(additional_services=additional), + _project_name_from_service=lambda _name: "Main", + ) + cases = ( + ("--only-backend", {"backend"}), + ("--only-frontend", {"frontend"}), + ("--no-infra", set()), + ("--no-dependencies", {"backend", "frontend", "voice-runtime"}), + ) + + for modifier, expected_types in cases: + with self.subTest(modifier=modifier): + route = parse_route(["restart", modifier], env={}) + selection = restart_prestop_selection(state=state, route=route, runtime=runtime) + expected_names = { + name for name, service in services.items() if service.type in expected_types + } + self.assertEqual(selection.selected_services, expected_names) + self.assertEqual(selection.service_types_by_project, {"Main": expected_types}) + self.assertFalse(selection.include_requirements) + + start_route = restart_start_route( + route, + restart_lookup_mode="main", + selected_services=selection.selected_services, + target_projects=selection.target_projects, + include_requirements=selection.include_requirements, + service_types_by_project=selection.service_types_by_project, + ) + self.assertEqual( + _restart_service_types_for_project( + route=start_route, + project_name="Main", + default_service_types={"backend", "frontend", "voice-runtime"}, + additional_services=additional, + ), + expected_types, + ) + + def test_restart_selection_preserves_heterogeneous_opaque_service_targets(self) -> None: + services = { + "alpha-api": ServiceRecord( + name="alpha-api", type="backend", service_slug="backend", project="Alpha", cwd="." + ), + "alpha-agent": ServiceRecord( + name="alpha-agent", + type="voice-runtime", + service_slug="voice-runtime", + project="Alpha", + cwd=".", + ), + "beta-api": ServiceRecord( + name="beta-api", type="backend", service_slug="backend", project="Beta", cwd="." + ), + "beta-agent": ServiceRecord( + name="beta-agent", + type="voice-runtime", + service_slug="voice-runtime", + project="Beta", + cwd=".", + ), + } + state = RunState( + run_id="run-1", + mode="trees", + services=services, + metadata={ + "project_roots": {"Alpha": "/alpha", "Beta": "/beta"}, + "dashboard_project_configured_services": { + "Alpha": ["backend", "voice-runtime"], + "Beta": ["backend", "voice-runtime", "worker"], + }, + "dashboard_stopped_services": [ + {"project": "Beta", "type": "worker", "name": "beta-job"} + ], + }, + ) + additional = ( + SimpleNamespace(name="voice-runtime", depends_on=("backend",)), + SimpleNamespace(name="worker", depends_on=("voice-runtime",)), + ) + runtime = SimpleNamespace( + config=SimpleNamespace(additional_services=additional), + _project_name_from_service=lambda _name: "wrong", + ) + route = Route( + command="restart", + mode="trees", + flags={"services": ["alpha-agent", "beta-job"]}, + ) + + selection = restart_prestop_selection(state=state, route=route, runtime=runtime) + _validate_restart_selection( + state=state, + route=route, + runtime=runtime, + target_projects=selection.target_projects, + service_types_by_project=selection.service_types_by_project, + ) + + self.assertEqual(selection.target_projects, {"Alpha", "Beta"}) + self.assertEqual( + selection.service_types_by_project, + { + "Alpha": {"backend", "voice-runtime"}, + "Beta": {"backend", "voice-runtime", "worker"}, + }, + ) + self.assertEqual(selection.selected_services, set(services)) + + def test_restart_validation_rejects_cross_project_unconfigured_and_empty_tree_targets(self) -> None: + state = RunState( + run_id="run-1", + mode="trees", + services={ + "alpha-agent": ServiceRecord( + name="alpha-agent", + type="voice-runtime", + service_slug="voice-runtime", + project="Alpha", + cwd=".", + ) + }, + metadata={ + "project_roots": {"Alpha": "/alpha", "Beta": "/beta"}, + "dashboard_project_configured_services": { + "Alpha": ["backend", "voice-runtime"], + "Beta": ["frontend"], + }, + }, + ) + runtime = SimpleNamespace( + config=SimpleNamespace( + additional_services=(SimpleNamespace(name="voice-runtime", depends_on=("backend",)),) + ), + _project_name_from_service=lambda _name: "wrong", + ) + for selector in ("alpha-agent", "backend"): + with self.subTest(selector=selector): + route = Route( + command="restart", + mode="trees", + projects=["Beta"], + flags={"services": [selector]}, + ) + selection = restart_prestop_selection(state=state, route=route, runtime=runtime) + with self.assertRaisesRegex(RuntimeError, "No matching services"): + _validate_restart_selection( + state=state, + route=route, + runtime=runtime, + target_projects=selection.target_projects, + service_types_by_project=selection.service_types_by_project, + ) + + conflicting_route = Route( + command="restart", + mode="trees", + projects=["Alpha"], + flags={ + "services": ["alpha-agent"], + "launch_backend": True, + "launch_frontend": False, + "launch_dependencies": False, + }, + ) + conflicting_selection = restart_prestop_selection( + state=state, + route=conflicting_route, + runtime=runtime, + ) + with self.assertRaisesRegex(RuntimeError, "conflicts with the requested launch scope"): + _validate_restart_selection( + state=state, + route=conflicting_route, + runtime=runtime, + target_projects=conflicting_selection.target_projects, + service_types_by_project=conflicting_selection.service_types_by_project, + ) + + empty_state = RunState(run_id="run-empty", mode="trees") + empty_route = Route( + command="restart", + mode="trees", + flags={"services": ["backend"], "batch": True}, + ) + empty_selection = restart_prestop_selection( + state=empty_state, + route=empty_route, + runtime=runtime, + ) + with self.assertRaisesRegex(RuntimeError, "No matching services|No active restart target"): + _validate_restart_selection( + state=empty_state, + route=empty_route, + runtime=runtime, + target_projects=empty_selection.target_projects, + service_types_by_project=empty_selection.service_types_by_project, + ) + + def test_global_dependency_restart_targets_requirements_only_projects(self) -> None: + state = RunState( + run_id="run-1", + mode="trees", + services={ + "Alpha Backend": ServiceRecord( + name="Alpha Backend", type="backend", project="Alpha", cwd="." + ) + }, + requirements={"Alpha": object(), "Beta": object()}, # type: ignore[dict-item] + metadata={"project_roots": {"Alpha": "/alpha", "Beta": "/beta"}}, + ) + runtime = SimpleNamespace( + config=SimpleNamespace(additional_services=()), + _project_name_from_service=lambda name: str(name).removesuffix(" Backend"), + ) + route = parse_route(["restart", "--trees", "--dependencies"], env={}) + + selection = restart_prestop_selection(state=state, route=route, runtime=runtime) + preservation = restart_prestop_preservation( + state, + selected_services=selection.selected_services, + include_requirements=selection.include_requirements, + target_projects=selection.target_projects, + ) + + self.assertEqual(selection.target_projects, {"Alpha", "Beta"}) + self.assertEqual(selection.selected_services, set()) + self.assertEqual(set(preservation.requirements_to_release), {"Alpha", "Beta"}) + + def test_explicit_service_restart_prestops_tracked_dependency_before_relaunch(self) -> None: + state = SimpleNamespace( + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + project="Main", + ), + "Main Voice Runtime": ServiceRecord( + name="Main Voice Runtime", + type="voice-runtime", + cwd="/repo/voice-runtime", + project="Main", + service_slug="voice-runtime", + ), + } + ) + runtime = SimpleNamespace( + config=SimpleNamespace( + additional_services=( + SimpleNamespace(name="voice-runtime", depends_on=("backend",)), + ) + ), + _project_name_from_service=lambda _name: "wrong-fallback", + ) + + selection = restart_prestop_selection( + state=state, + route=Route( + command="restart", + mode="main", + flags={"services": ["voice-runtime"]}, + ), + runtime=runtime, + ) + + self.assertEqual(selection.target_projects, {"Main"}) + self.assertEqual(selection.selected_services, {"Main Backend", "Main Voice Runtime"}) + + def test_explicit_service_restart_ignore_deps_does_not_prestop_dependency(self) -> None: + state = SimpleNamespace( + services={ + "Main Backend": ServiceRecord( + name="Main Backend", type="backend", cwd="/repo/backend", project="Main" + ), + "Main Voice Runtime": ServiceRecord( + name="Main Voice Runtime", + type="voice-runtime", + cwd="/repo/voice-runtime", + project="Main", + service_slug="voice-runtime", + ), + } + ) + runtime = SimpleNamespace( + config=SimpleNamespace( + additional_services=(SimpleNamespace(name="voice-runtime", depends_on=("backend",)),) + ), + _project_name_from_service=lambda _name: "Main", + ) + + selection = restart_prestop_selection( + state=state, + route=Route( + command="restart", + mode="main", + flags={"services": ["voice-runtime"], "ignore_service_deps": True}, + ), + runtime=runtime, + ) + + self.assertEqual(selection.selected_services, {"Main Voice Runtime"}) + def test_handle_restart_prestop_noops_for_non_restart_routes(self) -> None: route = Route(command="start", mode="main", raw_args=["start"], flags={}) session = SimpleNamespace(effective_route=route, runtime_mode="main") @@ -105,11 +497,14 @@ def test_handle_restart_prestop_stops_selected_services_and_preserves_remaining_ restart_state=None, preserved_requirements={}, preserved_services={}, + base_metadata={"new": True}, ) state = SimpleNamespace( + run_id="run-old", mode="main", services={"Main Backend": object(), "Other Backend": object()}, requirements={"Main": object(), "Other": object()}, + metadata={"state_source_run_ids": ["run-ancestor"], "old": True}, ) events: list[tuple[str, dict[str, object]]] = [] released: list[object] = [] @@ -122,8 +517,8 @@ def test_handle_restart_prestop_stops_selected_services_and_preserves_remaining_ _effective_start_mode=lambda route: "main", _try_load_existing_state=lambda *, mode: state, _project_name_from_service=lambda name: str(name).removesuffix(" Backend"), - _terminate_services_from_state=lambda state, *, selected_services, aggressive, verify_ownership: terminated.append( - (state, set(selected_services)) + _terminate_services_from_state=lambda state, *, selected_services, aggressive, verify_ownership: ( + terminated.append((state, set(selected_services))) or set() ), _release_requirement_ports=released.append, _emit=lambda event, **payload: events.append((event, payload)), @@ -133,7 +528,7 @@ def test_handle_restart_prestop_stops_selected_services_and_preserves_remaining_ runtime=runtime, session=session, suppress_progress_output=lambda route: False, - terminate_restart_orphan_listeners=lambda **kwargs: orphan_calls.append(dict(kwargs)), + terminate_restart_orphan_listeners=lambda **kwargs: orphan_calls.append(dict(kwargs)) or set(), spinner_factory=lambda *args, **kwargs: spinner, use_spinner_policy_fn=lambda policy: _NullContext(), resolve_spinner_policy_fn=lambda env: SimpleNamespace(enabled=True), @@ -148,10 +543,258 @@ def test_handle_restart_prestop_stops_selected_services_and_preserves_remaining_ self.assertEqual(released, [state.requirements["Main"]]) self.assertEqual(session.preserved_services, {"Other Backend": state.services["Other Backend"]}) self.assertEqual(session.preserved_requirements, {"Other": state.requirements["Other"]}) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-ancestor", "run-old"]) + self.assertTrue(session.base_metadata["old"]) + self.assertTrue(session.base_metadata["new"]) self.assertEqual(session.effective_route.command, "start") self.assertEqual(session.effective_route.flags["_restart_selected_services"], ["Main Backend"]) self.assertEqual(spinner.events, ["enter", "succeed:Restart pre-stop complete", "exit"]) - self.assertIn(("restart.selection", {"include_requirements": True, "target_projects": ["Main"], "selected_services": ["Main Backend"]}), events) + self.assertIn( + ( + "restart.selection", + {"include_requirements": True, "target_projects": ["Main"], "selected_services": ["Main Backend"]}, + ), + events, + ) + + def test_handle_restart_prestop_aborts_without_mutating_session_when_termination_fails(self) -> None: + route = Route( + command="restart", + mode="main", + flags={"services": ["Main Backend"]}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="main", + restart_state=None, + preserved_requirements={}, + preserved_services={}, + base_metadata={}, + ) + state = SimpleNamespace( + run_id="run-old", + mode="main", + services={"Main Backend": object()}, + requirements={"Main": object()}, + metadata={}, + ) + runtime = SimpleNamespace( + env={}, + _effective_start_mode=lambda _route: "main", + _try_load_existing_state=lambda *, mode: state, + _project_name_from_service=lambda _name: "Main", + _terminate_services_from_state=lambda *_args, **_kwargs: {"Main Backend"}, + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(RuntimeError, "Main Backend"): + handle_restart_prestop( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: True, + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("orphans must not be reaped"), + spinner_factory=lambda *_args, **_kwargs: _RecordingSpinner(), + use_spinner_policy_fn=lambda _policy: _NullContext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + ) + + self.assertEqual(session.preserved_services, state.services) + self.assertEqual(session.preserved_requirements, state.requirements) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-old"]) + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertIs(session.restart_state, state) + self.assertIs(session.effective_route, route) + + def test_handle_restart_prestop_treats_missing_termination_result_as_unconfirmed(self) -> None: + route = Route(command="restart", mode="main", flags={"services": ["Main Backend"]}) + session = SimpleNamespace( + effective_route=route, + runtime_mode="main", + restart_state=None, + preserved_requirements={}, + preserved_services={}, + base_metadata={}, + ) + state = SimpleNamespace( + run_id="run-old", + mode="main", + services={"Main Backend": object()}, + requirements={"Main": object()}, + metadata={}, + ) + runtime = SimpleNamespace( + env={}, + _effective_start_mode=lambda _route: "main", + _try_load_existing_state=lambda *, mode: state, + _project_name_from_service=lambda _name: "Main", + _terminate_services_from_state=lambda *_args, **_kwargs: None, + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(RuntimeError, "Main Backend"): + handle_restart_prestop( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: True, + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("orphans must not be reaped"), + spinner_factory=lambda *_args, **_kwargs: _RecordingSpinner(), + use_spinner_policy_fn=lambda _policy: _NullContext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, state.services) + self.assertEqual(session.preserved_requirements, state.requirements) + self.assertIs(session.effective_route, route) + + def test_handle_restart_prestop_preserves_source_authority_when_termination_raises(self) -> None: + route = Route(command="restart", mode="main", flags={"services": ["Main Backend"]}) + state = SimpleNamespace( + run_id="run-old", + mode="main", + services={"Main Backend": object()}, + requirements={"Main": object()}, + metadata={"authority": "old"}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="main", + restart_state=None, + preserved_requirements={}, + preserved_services={}, + base_metadata={}, + ) + + def fail_termination(*_args, **_kwargs): # noqa: ANN202 + raise OSError("service lock unlink failed") + + runtime = SimpleNamespace( + env={}, + _effective_start_mode=lambda _route: "main", + _try_load_existing_state=lambda *, mode: state, + _project_name_from_service=lambda _name: "Main", + _terminate_services_from_state=fail_termination, + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(OSError, "service lock unlink failed"): + handle_restart_prestop( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: True, + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("orphans must not be reaped"), + spinner_factory=lambda *_args, **_kwargs: _RecordingSpinner(), + use_spinner_policy_fn=lambda _policy: _NullContext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, state.services) + self.assertEqual(session.preserved_requirements, state.requirements) + self.assertEqual(session.base_metadata["authority"], "old") + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-old"]) + + def test_handle_restart_prestop_preserves_source_authority_when_requirement_release_raises(self) -> None: + route = Route( + command="restart", + mode="main", + flags={"services": ["Main Backend"], "restart_include_requirements": True}, + ) + state = SimpleNamespace( + run_id="run-old", + mode="main", + services={"Main Backend": object()}, + requirements={"Main": object()}, + metadata={"authority": "old"}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="main", + restart_state=None, + preserved_requirements={}, + preserved_services={}, + base_metadata={}, + ) + + def fail_release(_requirements): # noqa: ANN001, ANN202 + raise OSError("requirement lock unlink failed") + + runtime = SimpleNamespace( + env={}, + _effective_start_mode=lambda _route: "main", + _try_load_existing_state=lambda *, mode: state, + _project_name_from_service=lambda _name: "Main", + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + _release_requirement_ports=fail_release, + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(OSError, "requirement lock unlink failed"): + handle_restart_prestop( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: True, + terminate_restart_orphan_listeners=lambda **_kwargs: set(), + spinner_factory=lambda *_args, **_kwargs: _RecordingSpinner(), + use_spinner_policy_fn=lambda _policy: _NullContext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, state.services) + self.assertEqual(session.preserved_requirements, state.requirements) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-old"]) + + def test_handle_restart_prestop_aborts_when_orphan_exit_is_unconfirmed(self) -> None: + route = Route(command="restart", mode="main", flags={"services": ["Main Backend"]}) + session = SimpleNamespace( + effective_route=route, + runtime_mode="main", + restart_state=None, + preserved_requirements={}, + preserved_services={}, + base_metadata={}, + ) + state = SimpleNamespace( + run_id="run-old", + mode="main", + services={"Main Backend": object()}, + requirements={"Main": object()}, + metadata={}, + ) + runtime = SimpleNamespace( + env={}, + _effective_start_mode=lambda _route: "main", + _try_load_existing_state=lambda *, mode: state, + _project_name_from_service=lambda _name: "Main", + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(RuntimeError, "orphan listeners.*4242"): + handle_restart_prestop( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: True, + terminate_restart_orphan_listeners=lambda **_kwargs: {4242}, + spinner_factory=lambda *_args, **_kwargs: _RecordingSpinner(), + use_spinner_policy_fn=lambda _policy: _NullContext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, state.services) + self.assertEqual(session.preserved_requirements, state.requirements) + self.assertIs(session.effective_route, route) def test_restart_fallback_start_route_preserves_request_context_and_marks_restart(self) -> None: route = Route( @@ -185,12 +828,120 @@ def test_restart_start_route_records_sorted_selection_policy(self) -> None: ) self.assertEqual(updated.command, "start") + self.assertEqual(updated.projects, ["Main"]) self.assertEqual(updated.flags["_restart_selected_services"], ["Main Backend", "Main Frontend"]) self.assertEqual(updated.flags["_restart_target_projects"], ["Main"]) self.assertFalse(updated.flags["_restart_include_requirements"]) self.assertTrue(updated.flags["_restart_request"]) self.assertTrue(updated.flags["force"]) + def test_restart_start_route_carries_resolved_tree_project_without_prompt(self) -> None: + updated = restart_start_route( + Route(command="restart", mode="trees", flags={"services": ["voice-runtime"]}), + restart_lookup_mode="trees", + selected_services=set(), + target_projects={"Customer Platform"}, + include_requirements=False, + ) + + self.assertEqual(updated.projects, ["Customer Platform"]) + self.assertFalse(trees_start_selection_required(route=updated, runtime_mode="trees")) + + def test_unmatched_service_selector_never_expands_to_all_projects(self) -> None: + state = SimpleNamespace( + services={ + "Alpha Backend": ServiceRecord( + name="Alpha Backend", type="backend", cwd=".", project="Alpha" + ), + "Alpha Voice Runtime": ServiceRecord( + name="Alpha Voice Runtime", + type="voice-runtime", + cwd=".", + project="Alpha", + service_slug="voice-runtime", + ), + "Beta Backend": ServiceRecord( + name="Beta Backend", type="backend", cwd=".", project="Beta" + ), + "Beta Voice Runtime": ServiceRecord( + name="Beta Voice Runtime", + type="voice-runtime", + cwd=".", + project="Beta", + service_slug="voice-runtime", + ), + }, + requirements={}, + metadata={"project_roots": {"Alpha": "/repo/alpha", "Beta": "/repo/beta"}}, + ) + route = Route(command="restart", mode="trees", flags={"services": ["typo-service"]}) + runtime = SimpleNamespace( + config=SimpleNamespace( + additional_services=(SimpleNamespace(name="voice-runtime", depends_on=("backend",)),) + ), + _project_name_from_service=lambda name: str(name).rsplit(" ", 1)[0], + ) + + selection = restart_prestop_selection(state=state, route=route, runtime=runtime) + + self.assertEqual(selection.selected_services, set()) + self.assertEqual(selection.target_projects, set()) + with self.assertRaisesRegex(RuntimeError, "No matching services found.*typo-service"): + _validate_restart_selection( + state=state, + route=route, + runtime=runtime, + target_projects=selection.target_projects, + ) + + project_route = Route( + command="restart", + mode="trees", + projects=["Alpha"], + flags={"services": ["typo-service"]}, + ) + project_selection = restart_prestop_selection( + state=state, + route=project_route, + runtime=runtime, + ) + self.assertEqual(project_selection.selected_services, set()) + with self.assertRaisesRegex(RuntimeError, "No matching services found.*typo-service"): + _validate_restart_selection( + state=state, + route=project_route, + runtime=runtime, + target_projects=project_selection.target_projects, + ) + + def test_unmatched_project_selector_never_selects_all_services(self) -> None: + state = SimpleNamespace( + services={ + "Alpha Backend": ServiceRecord( + name="Alpha Backend", type="backend", cwd=".", project="Alpha" + ), + "Beta Backend": ServiceRecord( + name="Beta Backend", type="backend", cwd=".", project="Beta" + ), + }, + requirements={}, + metadata={"project_roots": {"Alpha": "/repo/alpha", "Beta": "/repo/beta"}}, + ) + route = Route(command="restart", mode="trees", projects=["Typo"], flags={}) + runtime = SimpleNamespace(_project_name_from_service=lambda name: str(name).removesuffix(" Backend")) + + selection = restart_prestop_selection(state=state, route=route, runtime=runtime) + + self.assertEqual(selection.selected_services, set()) + self.assertEqual(selection.target_projects, {"Typo"}) + with self.assertRaisesRegex(RuntimeError, "No active restart target.*Typo"): + _validate_restart_selection( + state=state, + route=route, + runtime=runtime, + target_projects=selection.target_projects, + ) + def test_restart_prestop_state_rejects_mode_mismatch_and_builds_fallback_route(self) -> None: route = Route(command="restart", mode="trees", raw_args=["restart", "--trees"], flags={}) events: list[tuple[str, dict[str, object]]] = [] @@ -218,7 +969,28 @@ def test_restart_prestop_state_rejects_mode_mismatch_and_builds_fallback_route(s ], ) - def test_restart_prestop_preservation_splits_services_and_requirements(self) -> None: + def test_restart_prestop_state_inherits_docker_runtime_from_saved_state(self) -> None: + route = Route(command="restart", mode="main", raw_args=["restart"], flags={}) + state = SimpleNamespace( + mode="main", + run_id="run-docker", + metadata={"application_runtime": "docker"}, + services={}, + ) + runtime = SimpleNamespace( + env={}, + _effective_start_mode=lambda route: "main", + _try_load_existing_state=lambda *, mode: state, + _emit=lambda event, **payload: None, + ) + + result = restart_prestop_state(route=route, runtime=runtime) + + self.assertIs(result.state, state) + self.assertTrue(route.flags["docker"]) + self.assertEqual(runtime.env["DOCKER_MODE"], "true") + + def test_restart_prestop_preservation_matches_requirement_projects_case_insensitively(self) -> None: state = SimpleNamespace( services={ "Main Backend": object(), @@ -227,6 +999,10 @@ def test_restart_prestop_preservation_splits_services_and_requirements(self) -> }, requirements={ "Main": SimpleNamespace(name="main requirements"), + "Main Restart Collision": SimpleNamespace( + name="replacement requirements", + project="Main", + ), "Other": SimpleNamespace(name="other requirements"), }, ) @@ -235,11 +1011,14 @@ def test_restart_prestop_preservation_splits_services_and_requirements(self) -> state, selected_services={"Main Backend", "Main Frontend"}, include_requirements=True, - target_projects={"Main"}, + target_projects={"main"}, ) self.assertEqual(set(result.preserved_services), {"Other Backend"}) - self.assertEqual(set(result.requirements_to_release), {"Main"}) + self.assertEqual( + set(result.requirements_to_release), + {"Main", "Main Restart Collision"}, + ) self.assertEqual(set(result.preserved_requirements), {"Other"}) def test_restart_prestop_preservation_preserves_requirements_when_not_included(self) -> None: @@ -282,6 +1061,36 @@ def test_restart_prestop_selection_falls_back_to_selected_service_projects_for_r self.assertEqual(selection.target_projects, {"Main"}) self.assertTrue(selection.include_requirements) + def test_restart_prestop_selection_includes_project_additional_services(self) -> None: + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace(project="Main", type="backend"), + "Main Frontend": SimpleNamespace(project="Main", type="frontend"), + "Main Optional Bad": SimpleNamespace( + project="Main", + type="optional-bad", + service_slug="optional-bad", + ), + } + ) + route = Route( + command="restart", + mode="main", + raw_args=["restart", "--project", "Main", "--entire-system"], + projects=["Main"], + flags={"runtime_scope": "entire-system"}, + ) + runtime = SimpleNamespace(_project_name_from_service=lambda name: str(name).split(" ", 1)[0]) + + selection = restart_prestop_selection(state=state, route=route, runtime=runtime) + + self.assertEqual( + selection.selected_services, + {"Main Backend", "Main Frontend", "Main Optional Bad"}, + ) + self.assertEqual(selection.target_projects, {"Main"}) + self.assertTrue(selection.include_requirements) + def test_restart_port_assignments_use_actual_port_then_requested_port_for_selected_services(self) -> None: state = SimpleNamespace( services={ @@ -502,7 +1311,7 @@ def test_terminate_restart_orphan_listeners_releases_successfully_terminated_por released: list[int] = [] terminated: list[tuple[int, float, float]] = [] - terminate_restart_orphan_listeners( + failed = terminate_restart_orphan_listeners( state=state, selected_services={"Main Backend"}, aggressive=True, @@ -511,15 +1320,121 @@ def test_terminate_restart_orphan_listeners_releases_successfully_terminated_por port_spacing=0, listener_pids_for_port=lambda port: [42] if port == 8000 else [], process_cwd=lambda pid: "/repo/backend" if pid == 42 else None, - terminate_pid=lambda pid, *, term_timeout, kill_timeout: terminated.append( - (pid, term_timeout, kill_timeout) - ) - or True, - release_port=released.append, + terminate_pid=lambda pid, *, term_timeout, kill_timeout: ( + terminated.append((pid, term_timeout, kill_timeout)) or True + ), + port_planner=SimpleNamespace(release=released.append), ) self.assertEqual(terminated, [(42, 0.5, 1.0)]) self.assertEqual(released, [8000]) + self.assertEqual(failed, set()) + + def test_restart_orphan_cleanup_releases_verified_prior_session_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(8000, owner="Main:backend") + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace( + project="Main", + type="backend", + cwd="/repo/backend", + actual_port=8000, + port_lock_session="prior-session", + ) + } + ) + + failed = terminate_restart_orphan_listeners( + state=state, + selected_services={"Main Backend"}, + aggressive=True, + backend_port_base=8000, + frontend_port_base=3000, + port_spacing=0, + listener_pids_for_port=lambda port: [42] if port == 8000 else [], + process_cwd=lambda pid: "/repo/backend" if pid == 42 else None, + terminate_pid=lambda *_args, **_kwargs: True, + port_planner=current, + ) + + self.assertEqual(failed, set()) + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + + def test_stale_restart_state_cannot_release_newer_same_owner_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + newer = PortPlanner( + lock_dir=tmpdir, + session_id="newer-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + newer.reserve_next(8000, owner="Main:backend") + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace( + project="Main", + type="backend", + cwd="/repo/backend", + actual_port=8000, + port_lock_session="stale-session", + ) + } + ) + + failed = terminate_restart_orphan_listeners( + state=state, + selected_services={"Main Backend"}, + aggressive=True, + backend_port_base=8000, + frontend_port_base=3000, + port_spacing=0, + listener_pids_for_port=lambda port: [42] if port == 8000 else [], + process_cwd=lambda pid: "/repo/backend" if pid == 42 else None, + terminate_pid=lambda *_args, **_kwargs: True, + port_planner=cleanup, + ) + + self.assertEqual(failed, set()) + self.assertTrue((Path(tmpdir) / "8000.lock").exists()) + + def test_terminate_restart_orphan_listeners_reports_unconfirmed_exits_without_releasing_ports(self) -> None: + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace(type="backend", cwd="/repo/backend", actual_port=8000), + } + ) + released: list[int] = [] + + failed = terminate_restart_orphan_listeners( + state=state, + selected_services={"Main Backend"}, + aggressive=True, + backend_port_base=8000, + frontend_port_base=3000, + port_spacing=0, + listener_pids_for_port=lambda port: [42] if port == 8000 else [], + process_cwd=lambda pid: "/repo/backend" if pid == 42 else None, + terminate_pid=lambda *_args, **_kwargs: False, + port_planner=SimpleNamespace(release=released.append), + ) + + self.assertEqual(failed, {42}) + self.assertEqual(released, []) if __name__ == "__main__": diff --git a/tests/python/startup/test_resume_restore_policy.py b/tests/python/startup/test_resume_restore_policy.py index a5fa2b510..848e8610c 100644 --- a/tests/python/startup/test_resume_restore_policy.py +++ b/tests/python/startup/test_resume_restore_policy.py @@ -6,6 +6,11 @@ from types import SimpleNamespace from envctl_engine.runtime.command_router import Route, parse_route +from envctl_engine.startup.resume_restore_execution import ( + _apply_restore_runtime_mode, + _projects_to_restore, + _route_for_resume_restore, +) from envctl_engine.startup.resume_restore_policy import ( _configured_restore_service_types, _requirements_reuse_decision, @@ -16,10 +21,40 @@ apply_ports_to_context, project_root, ) -from envctl_engine.state.models import RequirementsResult, ServiceRecord +from envctl_engine.startup.resume_restore_results import apply_restore_result +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord class ResumeRestorePolicyTests(unittest.TestCase): + def test_docker_state_forces_docker_mode_for_direct_resume_restore(self) -> None: + state = RunState( + run_id="run-docker", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + runtime_kind="docker", + container_id="stale-container", + ) + }, + ) + route = Route(command="resume", mode="main", flags={"batch": True}) + + restored_route = _route_for_resume_restore( + route, + state=state, + suppress_timing_output=False, + ) + + self.assertTrue(restored_route.flags["docker"]) + self.assertTrue(restored_route.flags["_resume_restore"]) + self.assertTrue(restored_route.flags["batch"]) + runtime = SimpleNamespace(env={}) + _apply_restore_runtime_mode(runtime, restored_route) + self.assertEqual(runtime.env["DOCKER_MODE"], "true") + def test_plan_resume_defaults_to_parallel_tree_restore(self) -> None: captured_routes: list[Route] = [] @@ -151,6 +186,60 @@ def test_project_root_uses_metadata_then_service_cwd_fallback(self) -> None: self.assertEqual(project_root(SimpleNamespace(runtime=runtime), state, "Feature"), repo) + def test_resume_project_discovery_and_root_prefer_service_project_metadata(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + service_root = Path(tmpdir) / "customer-platform" / "voice" + service_root.mkdir(parents=True) + service = ServiceRecord( + name="Opaque Runtime Process", + type="voice-runtime", + cwd=str(service_root), + project="Customer Platform", + ) + state = RunState( + run_id="run-extra", + mode="trees", + services={service.name: service}, + ) + runtime = SimpleNamespace( + config=SimpleNamespace(base_dir=Path(tmpdir)), + _project_name_from_service=lambda _name: "Opaque Runtime", + _requirements_ready=lambda _requirements: True, + ) + + self.assertEqual( + _projects_to_restore(runtime, state=state, missing_services=[service.name]), + ["Customer Platform"], + ) + self.assertEqual( + project_root(SimpleNamespace(runtime=runtime), state, "Customer Platform"), + service_root, + ) + + def test_projects_to_restore_deduplicates_service_and_requirement_project_case(self) -> None: + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/repo/backend", + project="Main", + ) + requirements = RequirementsResult(project="main", health="degraded") + state = RunState( + run_id="run-casefold", + mode="main", + services={service.name: service}, + requirements={"main": requirements}, + ) + runtime = SimpleNamespace( + _project_name_from_service=lambda _name: "Main", + _requirements_ready=lambda value: value is not requirements, + ) + + self.assertEqual( + _projects_to_restore(runtime, state=state, missing_services=[service.name]), + ["Main"], + ) + def test_apply_ports_to_context_projects_service_ports(self) -> None: backend_plan = SimpleNamespace(port=None) frontend_plan = SimpleNamespace(port=None) @@ -173,6 +262,125 @@ def test_apply_ports_to_context_projects_service_ports(self) -> None: self.assertEqual(calls, [(backend_plan, 8000), (frontend_plan, 9000)]) + def test_apply_ports_to_context_uses_record_project_metadata(self) -> None: + backend_plan = SimpleNamespace(port=None) + calls: list[tuple[object, int]] = [] + runtime = SimpleNamespace( + _project_name_from_service=lambda _name: "Opaque Runtime", + _set_plan_port=lambda plan, port: calls.append((plan, port)), + _set_plan_port_from_component=lambda plan, component: None, + ) + context = SimpleNamespace( + name="Customer Platform", + ports={"backend": backend_plan, "frontend": SimpleNamespace(port=None)}, + ) + service = ServiceRecord( + name="Opaque API Process", + type="backend", + cwd="/tmp/customer-platform/api", + actual_port=8100, + project="Customer Platform", + ) + state = RunState(run_id="run-ports", mode="trees", services={service.name: service}) + + apply_ports_to_context(SimpleNamespace(runtime=runtime), context, state) + + self.assertEqual(calls, [(backend_plan, 8100)]) + + def test_apply_ports_to_context_finds_case_insensitive_requirements_key(self) -> None: + db_plan = SimpleNamespace(port=None) + projected: list[tuple[object, dict[str, object]]] = [] + runtime = SimpleNamespace( + _project_name_from_service=lambda _name: "Main", + _set_plan_port=lambda _plan, _port: None, + _set_plan_port_from_component=lambda plan, component: projected.append((plan, component)), + ) + context = SimpleNamespace(name="Main", ports={"db": db_plan}) + requirements = RequirementsResult( + project="main", + db={"enabled": True, "success": True, "final": 5544}, + ) + state = RunState( + run_id="run-requirement-case", + mode="main", + requirements={"main": requirements}, + ) + + apply_ports_to_context(SimpleNamespace(runtime=runtime), context, state) + + self.assertIn((db_plan, requirements.db), projected) + + def test_failed_restore_retains_unterminated_service_under_collision_safe_key(self) -> None: + old = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/repo/backend", + project="Main", + pid=1111, + status="running", + ) + replacement = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/repo/backend", + project="Main", + pid=2222, + status="starting", + ) + state = RunState(run_id="run-collision", mode="main", services={old.name: old}) + errors: list[str] = [] + + apply_restore_result( + state, + project="Main", + result={ + "error": "replacement cleanup was unconfirmed", + "unterminated_services": {replacement.name: replacement}, + }, + errors=errors, + timing_enabled=False, + timing_lines=[], + project_totals_ms={}, + ) + + self.assertEqual({service.pid for service in state.services.values()}, {1111, 2222}) + collision_name = next(name for name in state.services if "Resume Collision" in name) + self.assertEqual(state.services[collision_name].status, "termination_failed") + self.assertTrue(state.services[collision_name].degraded) + self.assertEqual(state.metadata["termination_failed_services"], [collision_name]) + self.assertEqual(errors, ["Main: replacement cleanup was unconfirmed"]) + + def test_successful_restore_retains_cross_project_service_name_collisions(self) -> None: + state = RunState(run_id="run-success-collision", mode="trees") + + for project, pid in (("Alpha", 101), ("Beta", 202)): + service = ServiceRecord( + name="Shared Worker", + type="worker", + cwd=f"/{project.casefold()}", + project=project, + pid=pid, + status="running", + ) + apply_restore_result( + state, + project=project, + result={ + "error": None, + "remove_service_names": ["Shared Worker"], + "project_services": {"Shared Worker": service}, + }, + errors=[], + timing_enabled=False, + timing_lines=[], + project_totals_ms={}, + ) + + self.assertEqual({service.pid for service in state.services.values()}, {101, 202}) + self.assertEqual({service.project for service in state.services.values()}, {"Alpha", "Beta"}) + collision_name = next(name for name in state.services if "Resume Collision" in name) + self.assertEqual(state.services[collision_name].name, collision_name) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/startup/test_run_reuse_identity.py b/tests/python/startup/test_run_reuse_identity.py index 15cd8531d..5cb50cd30 100644 --- a/tests/python/startup/test_run_reuse_identity.py +++ b/tests/python/startup/test_run_reuse_identity.py @@ -1,5 +1,7 @@ from __future__ import annotations +import hashlib +import json import unittest from pathlib import Path from types import SimpleNamespace @@ -16,6 +18,7 @@ project_identities_from_contexts, project_identities_from_state, ) +from envctl_engine.startup.run_reuse_decision import state_has_resumable_services from envctl_engine.state.models import RunState, ServiceRecord @@ -45,10 +48,57 @@ def test_build_startup_identity_metadata_records_roots_and_fingerprint_payload(s self.assertEqual(metadata["project_roots"], {"Main": normalize_project_root("/tmp/repo")}) identity = metadata["startup_identity"] self.assertEqual(identity["mode"], "main") + self.assertEqual(identity["application_runtime"], "process") self.assertEqual(identity["projects"], [{"name": "Main", "root": normalize_project_root("/tmp/repo")}]) self.assertEqual(identity["services"], {"backend": True, "frontend": False}) self.assertIn("fingerprint", identity) + def test_build_startup_identity_metadata_preserves_unselected_roots_without_case_duplicates(self) -> None: + runtime = SimpleNamespace( + config=SimpleNamespace( + backend_dir_name="api", + frontend_dir_name="web", + additional_services=(), + startup_enabled_for_mode=lambda _mode: True, + service_enabled_for_mode=lambda _mode, service: service == "backend", + requirement_enabled_for_mode=lambda _mode, _requirement: False, + ) + ) + + metadata = build_startup_identity_metadata( + runtime, + runtime_mode="trees", + project_contexts=[SimpleNamespace(name="featurea", root=Path("/tmp/new-a"))], + base_metadata={ + "project_roots": { + "FEATUREA": "/tmp/old-a", + "FeatureB": "/tmp/feature-b", + } + }, + route=Route(command="start", mode="trees", flags={}), + ) + + self.assertEqual( + metadata["project_roots"], + { + "featurea": normalize_project_root("/tmp/new-a"), + "FeatureB": normalize_project_root("/tmp/feature-b"), + }, + ) + identity = metadata["startup_identity"] + self.assertEqual( + identity["projects"], + [ + {"name": "featurea", "root": normalize_project_root("/tmp/new-a")}, + {"name": "FeatureB", "root": normalize_project_root("/tmp/feature-b")}, + ], + ) + fingerprint_payload = {key: value for key, value in identity.items() if key != "fingerprint"} + self.assertEqual( + identity["fingerprint"], + hashlib.sha256(json.dumps(fingerprint_payload, sort_keys=True).encode("utf-8")).hexdigest(), + ) + def test_project_identities_from_state_uses_metadata_requirements_and_services(self) -> None: state = RunState( run_id="run-1", @@ -69,6 +119,32 @@ def test_project_identities_from_state_uses_metadata_requirements_and_services(s ], ) + def test_state_identity_and_resumability_prefer_multiword_record_project_metadata(self) -> None: + service = ServiceRecord( + name="Opaque API Runtime", + type="backend", + cwd="/tmp/customer-platform/runtime", + project="Customer Platform", + ) + state = RunState( + run_id="run-metadata", + mode="trees", + services={service.name: service}, + metadata={"project_roots": {"Customer Platform": "/tmp/customer-platform"}}, + ) + runtime = SimpleNamespace(_project_name_from_service=lambda _name: "Opaque API") + + self.assertEqual( + identities_to_payload(project_identities_from_state(runtime, state)), + [ + { + "name": "Customer Platform", + "root": normalize_project_root("/tmp/customer-platform"), + } + ], + ) + self.assertTrue(state_has_resumable_services(runtime, state)) + def test_context_identity_sorting_and_weak_keys_are_stable(self) -> None: identities = project_identities_from_contexts( [ @@ -83,7 +159,9 @@ def test_context_identity_sorting_and_weak_keys_are_stable(self) -> None: def test_startup_identity_mismatch_ignores_matching_fingerprints_and_reports_changed_fields(self) -> None: self.assertEqual( - _startup_identity_mismatch({"fingerprint": "same", "mode": "main"}, {"fingerprint": "same", "mode": "trees"}), + _startup_identity_mismatch( + {"fingerprint": "same", "mode": "main"}, {"fingerprint": "same", "mode": "trees"} + ), {}, ) @@ -94,6 +172,21 @@ def test_startup_identity_mismatch_ignores_matching_fingerprints_and_reports_cha self.assertEqual(mismatch, {"fields": ["mode"]}) + self.assertEqual( + _startup_identity_mismatch( + {"mode": "main", "services": {"backend": True}}, + {"mode": "main", "services": {"backend": True}, "application_runtime": "process"}, + ), + {}, + ) + self.assertEqual( + _startup_identity_mismatch( + {"mode": "main", "application_runtime": "process"}, + {"mode": "main", "application_runtime": "docker"}, + ), + {"fields": ["application_runtime"]}, + ) + def test_root_mismatches_compare_only_strong_roots(self) -> None: selected = [ProjectIdentity("Main", "/repo"), ProjectIdentity("Weak", None)] state = [ProjectIdentity("Main", "/other"), ProjectIdentity("Weak", "/weak")] diff --git a/tests/python/startup/test_run_reuse_support.py b/tests/python/startup/test_run_reuse_support.py index 566c0b45e..8aa9a1255 100644 --- a/tests/python/startup/test_run_reuse_support.py +++ b/tests/python/startup/test_run_reuse_support.py @@ -7,6 +7,7 @@ from envctl_engine.startup.run_reuse_support import ( dashboard_stopped_service_entries, fresh_start_replacement_services, + mark_run_reused, prepare_dashboard_stopped_service_restore, replace_existing_project_services_for_fresh_start, metadata_without_dashboard_stopped_services, @@ -15,13 +16,19 @@ class RunReuseSupportTests(unittest.TestCase): + def test_mark_run_reused_recovers_from_malformed_counter(self) -> None: + updated = mark_run_reused({"run_reuse_count": "not-a-number"}, reason="exact_match") + + self.assertEqual(updated["run_reuse_count"], 1) + self.assertEqual(updated["last_reuse_reason"], "exact_match") + def test_run_reuse_debug_orch_groups_only_apply_to_plan_commands(self) -> None: runtime = SimpleNamespace(env={"ENVCTL_DEBUG_PLAN_ORCH_GROUP": "alpha+ beta,gamma ,,"}) self.assertEqual(run_reuse_debug_orch_groups(runtime, requested_command="plan"), {"alpha", "beta", "gamma"}) self.assertEqual(run_reuse_debug_orch_groups(runtime, requested_command="start"), set()) - def test_dashboard_stopped_service_entries_normalizes_valid_backend_frontend_entries(self) -> None: + def test_dashboard_stopped_service_entries_normalizes_all_valid_service_types(self) -> None: state = SimpleNamespace( metadata={ "dashboard_stopped_services": [ @@ -39,12 +46,15 @@ def test_dashboard_stopped_service_entries_normalizes_valid_backend_frontend_ent [ {"project": "Main", "type": "frontend", "name": "Main Frontend"}, {"project": "Main", "type": "backend", "name": "Main Backend"}, + {"project": "Main", "type": "worker", "name": "Main Worker"}, ], ) def test_dashboard_stopped_service_entries_ignores_missing_or_invalid_metadata(self) -> None: self.assertEqual(dashboard_stopped_service_entries(SimpleNamespace(metadata={})), []) - self.assertEqual(dashboard_stopped_service_entries(SimpleNamespace(metadata={"dashboard_stopped_services": {}})), []) + self.assertEqual( + dashboard_stopped_service_entries(SimpleNamespace(metadata={"dashboard_stopped_services": {}})), [] + ) def test_metadata_without_dashboard_stopped_services_removes_restored_entries_only(self) -> None: metadata = { @@ -86,6 +96,24 @@ def test_metadata_without_dashboard_stopped_services_drops_key_when_all_entries_ {"keep": True}, ) + def test_metadata_without_dashboard_stopped_services_is_project_scoped(self) -> None: + entries = [ + { + "name": f"{project} Voice Runtime", + "project": project, + "type": "voice-runtime", + } + for project in ("Customer Platform", "Other") + ] + + updated = metadata_without_dashboard_stopped_services( + {"dashboard_stopped_services": entries, "keep": True}, + restored_service_types_by_project={"Customer Platform": {"voice-runtime"}}, + ) + + self.assertEqual(updated["dashboard_stopped_services"], [entries[1]]) + self.assertTrue(updated["keep"]) + def test_prepare_dashboard_stopped_service_restore_rewrites_route_and_preserves_active_state(self) -> None: route = Route(command="start", mode="trees", raw_args=["start"], flags={}) session = SimpleNamespace( @@ -104,6 +132,7 @@ def test_prepare_dashboard_stopped_service_restore_rewrites_route_and_preserves_ metadata={ "dashboard_stopped_services": [ {"project": "Main", "type": "frontend", "name": "Main Frontend"}, + {"project": "Main", "type": "voice-runtime", "name": "Main Voice Runtime"}, {"project": "Other", "type": "backend", "name": "Other Backend"}, ], "keep": True, @@ -127,12 +156,28 @@ def test_prepare_dashboard_stopped_service_restore_rewrites_route_and_preserves_ self.assertEqual(session.preserved_services, candidate_state.services) self.assertEqual(session.preserved_requirements, candidate_state.requirements) self.assertEqual(session.base_metadata["keep"], True) - self.assertNotIn("dashboard_stopped_services", session.base_metadata) + self.assertEqual( + session.base_metadata["dashboard_stopped_services"], + candidate_state.metadata["dashboard_stopped_services"], + ) self.assertEqual(session.effective_route.projects, ["Main", "Other"]) self.assertEqual(session.effective_route.flags["_restart_request"], True) self.assertEqual(session.effective_route.flags["_restore_dashboard_stopped_services"], True) - self.assertEqual(session.effective_route.flags["services"], ["Main Frontend", "Other Backend"]) - self.assertEqual(session.effective_route.flags["restart_service_types"], ["backend", "frontend"]) + self.assertEqual( + session.effective_route.flags["services"], + ["Main Frontend", "Main Voice Runtime", "Other Backend"], + ) + self.assertEqual( + session.effective_route.flags["restart_service_types"], + ["backend", "frontend", "voice-runtime"], + ) + self.assertEqual( + session.effective_route.flags["_restart_service_types_by_project"], + { + "Main": ["frontend", "voice-runtime"], + "Other": ["backend"], + }, + ) self.assertEqual(phases[0][0], "auto_resume_evaluate") self.assertEqual(phases[0][1]["status"], "restore_stopped_services") self.assertEqual(phases[0][1]["match_mode"], "subset") @@ -172,8 +217,7 @@ def test_prepare_dashboard_stopped_service_restore_ignores_active_or_unselected_ ) ) - def test_fresh_start_replacement_services_selects_configured_service_types_for_target_projects(self) -> None: - route = Route(command="start", mode="main", raw_args=["start"], flags={}) + def test_fresh_start_replacement_services_selects_every_service_for_target_projects(self) -> None: candidate_state = SimpleNamespace( services={ "Main Backend": SimpleNamespace(name="Main Backend", project="Main", type="backend"), @@ -184,20 +228,104 @@ def test_fresh_start_replacement_services_selects_configured_service_types_for_t self.assertEqual( fresh_start_replacement_services( - route=route, selected_contexts=[SimpleNamespace(name="Main")], candidate_state=candidate_state, - configured_service_types={"backend"}, - additional_services=(), project_name_from_service=lambda name: str(name).removesuffix(" Backend").removesuffix(" Frontend"), ), - {"Main Backend"}, + {"Main Backend", "Main Frontend"}, + ) + + def test_unfiltered_explicit_no_resume_replaces_tracked_removed_additional_services(self) -> None: + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace(name="Main Backend", project="Main", type="backend"), + "Main Old Worker": SimpleNamespace( + name="Main Old Worker", + project="Main", + type="old-worker", + service_slug="old-worker", + ), + } + ) + + selected = fresh_start_replacement_services( + selected_contexts=[SimpleNamespace(name="Main")], + candidate_state=state, + project_name_from_service=lambda _name: "Main", ) + self.assertEqual(selected, {"Main Backend", "Main Old Worker"}) + + def test_fresh_start_replacement_services_respects_launch_only_filter(self) -> None: + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace(project="Main", type="backend"), + "Main Frontend": SimpleNamespace(project="Main", type="frontend"), + "Main Old Worker": SimpleNamespace(project="Main", type="old-worker"), + } + ) + + cases = ( + ({"no_resume": True, "launch_backend": True, "launch_frontend": False}, {"Main Backend"}), + ({"no_resume": True, "launch_backend": False, "launch_frontend": True}, {"Main Frontend"}), + ({"no_resume": True, "launch_backend": False, "launch_frontend": False}, set()), + ) + for flags, expected in cases: + with self.subTest(flags=flags): + selected = fresh_start_replacement_services( + route=Route(command="start", mode="trees", raw_args=["start"], flags=flags), + selected_contexts=[SimpleNamespace(name="Main")], + candidate_state=state, + project_name_from_service=lambda _name: "Main", + ) + + self.assertEqual(selected, expected) + + def test_fresh_start_no_dependencies_flag_keeps_unfiltered_removed_service_cleanup(self) -> None: + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace(project="Main", type="backend"), + "Main Old Worker": SimpleNamespace(project="Main", type="old-worker"), + } + ) + + selected = fresh_start_replacement_services( + route=Route( + command="start", + mode="trees", + raw_args=["start", "--no-resume", "--no-dependencies"], + flags={"no_resume": True, "launch_dependencies": False}, + ), + selected_contexts=[SimpleNamespace(name="Main")], + candidate_state=state, + project_name_from_service=lambda _name: "Main", + ) + + self.assertEqual(selected, {"Main Backend", "Main Old Worker"}) + def test_replace_existing_project_services_for_fresh_start_terminates_selected_services_and_orphans(self) -> None: route = Route(command="start", mode="trees", raw_args=["start"], flags={}) - session = SimpleNamespace(effective_route=route, runtime_mode="trees") - candidate_state = SimpleNamespace(run_id="run-existing") + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="Main")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + candidate_state = SimpleNamespace( + run_id="run-existing", + services={ + "Main Backend": SimpleNamespace(project="Main"), + "Main Frontend": SimpleNamespace(project="Main"), + "Other Backend": SimpleNamespace(project="Other"), + }, + requirements={ + "Main": SimpleNamespace(project="Main"), + "Other": SimpleNamespace(project="Other"), + }, + metadata={"state_source_run_ids": ["run-ancestor"], "keep": True}, + ) events: list[tuple[str, dict[str, object]]] = [] announced: list[object] = [] progress: list[tuple[Route, str]] = [] @@ -205,22 +333,24 @@ def test_replace_existing_project_services_for_fresh_start_terminates_selected_s orphaned: list[tuple[object, set[str], bool]] = [] runtime = SimpleNamespace( _emit=lambda event, **payload: events.append((event, payload)), - _terminate_services_from_state=lambda state, *, selected_services, aggressive, verify_ownership: terminated.append( - (state, set(selected_services), aggressive, verify_ownership) + _terminate_services_from_state=lambda state, *, selected_services, aggressive, verify_ownership: ( + terminated.append((state, set(selected_services), aggressive, verify_ownership)) or set() ), + _project_name_from_service=lambda name: str(name).removesuffix(" Backend").removesuffix(" Frontend"), + _release_requirement_ports=lambda requirements: released.append(requirements), ) + released: list[object] = [] replace_existing_project_services_for_fresh_start( runtime=runtime, session=session, candidate_state=candidate_state, reason="startup_fingerprint_mismatch", - fresh_start_replacement_services=lambda session, *, candidate_state: {"Main Backend", "Main Frontend"}, announce_session_identifiers=announced.append, report_progress=lambda route, message: progress.append((route, message)), terminate_restart_orphan_listeners=lambda *, state, selected_services, aggressive: orphaned.append( (state, set(selected_services), aggressive) - ), + ) or set(), ) self.assertEqual(announced, [session]) @@ -241,35 +371,486 @@ def test_replace_existing_project_services_for_fresh_start_terminates_selected_s ) self.assertEqual(terminated, [(candidate_state, {"Main Backend", "Main Frontend"}, False, True)]) self.assertEqual(orphaned, [(candidate_state, {"Main Backend", "Main Frontend"}, True)]) + self.assertEqual(released, [candidate_state.requirements["Main"]]) + self.assertEqual(session.preserved_services, {"Other Backend": candidate_state.services["Other Backend"]}) + self.assertEqual(session.preserved_requirements, {"Other": candidate_state.requirements["Other"]}) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-ancestor", "run-existing"]) + self.assertTrue(session.base_metadata["keep"]) - def test_replace_existing_project_services_for_fresh_start_skips_non_mismatch_or_dependencies_scope(self) -> None: - runtime = SimpleNamespace(_emit=lambda *args, **kwargs: self.fail("replace should not emit")) - candidate_state = SimpleNamespace(run_id="run-existing") + def test_explicit_no_resume_replaces_matching_services_instead_of_leaking_a_second_copy(self) -> None: + route = Route(command="start", mode="trees", flags={"no_resume": True}) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": SimpleNamespace(project="feature-a")}, + requirements={}, + metadata={}, + ) + terminated: list[set[str]] = [] + progress: list[str] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _terminate_services_from_state=lambda _state, *, selected_services, **_kwargs: terminated.append( + set(selected_services) + ) or set(), + _project_name_from_service=lambda name: str(name).removesuffix(" Backend"), + _release_requirement_ports=lambda _requirements: None, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda _route, message: progress.append(message), + terminate_restart_orphan_listeners=lambda **_kwargs: set(), + ) + + self.assertEqual(terminated, [{"feature-a Backend"}]) + self.assertEqual(progress, ["Auto-resume disabled; replacing 1 existing service(s)..."]) + + def test_explicit_no_resume_rebinds_reserved_plan_to_released_service_port(self) -> None: + route = Route(command="start", mode="main", flags={"no_resume": True}) + backend_plan = SimpleNamespace(assigned=8001, final=8001, source="retry") + context = SimpleNamespace(name="Main", ports={"backend": backend_plan}) + service = SimpleNamespace( + name="Main Backend", + project="Main", + type="backend", + service_slug="backend", + listener_expected=True, + actual_port=8000, + requested_port=8000, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="main", + selected_contexts=[context], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"Main Backend": service}, + requirements={}, + metadata={}, + ) + calls: list[tuple[str, int, str]] = [] + runtime = SimpleNamespace( + port_planner=SimpleNamespace( + release=lambda port, *, owner: calls.append(("release", port, owner)), + ), + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "Main", + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + _release_requirement_ports=lambda _requirements: None, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda _route, _message: None, + terminate_restart_orphan_listeners=lambda **_kwargs: set(), + ) + + self.assertEqual(calls, [("release", 8001, "Main:backend")]) + self.assertEqual((backend_plan.assigned, backend_plan.final), (8000, 8000)) + self.assertEqual(backend_plan.source, "fresh_start_replacement") + + def test_backend_scoped_fresh_start_preserves_frontend_and_existing_requirements(self) -> None: + route = Route( + command="start", + mode="trees", + flags={"no_resume": True, "runtime_scope": "backend"}, + ) + backend = SimpleNamespace(project="feature-a", type="backend") + frontend = SimpleNamespace(project="feature-a", type="frontend") + requirements = SimpleNamespace(project="feature-a") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": backend, "feature-a Frontend": frontend}, + requirements={"feature-a": requirements}, + metadata={}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + terminated: list[set[str]] = [] + released: list[object] = [] + runtime = SimpleNamespace( + config=SimpleNamespace(additional_services=()), + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda name: str(name).rsplit(" ", 1)[0], + _terminate_services_from_state=lambda _state, *, selected_services, **_kwargs: ( + terminated.append(set(selected_services)) or set() + ), + _release_requirement_ports=released.append, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda *_args: None, + terminate_restart_orphan_listeners=lambda **_kwargs: set(), + ) + + self.assertEqual(terminated, [{"feature-a Backend"}]) + self.assertEqual(session.preserved_services, {"feature-a Frontend": frontend}) + self.assertEqual(session.preserved_requirements, {"feature-a": requirements}) + self.assertEqual(released, []) + + def test_explicit_no_resume_failure_keeps_previous_state_authoritative(self) -> None: + route = Route(command="start", mode="trees", flags={"no_resume": True}) + original_service = SimpleNamespace(project="feature-a") + original_requirements = SimpleNamespace(project="feature-a") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": original_service}, + requirements={"feature-a": original_requirements}, + metadata={"keep": True}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={"sentinel": object()}, + preserved_requirements={"sentinel": object()}, + base_metadata={"sentinel": True}, + ) + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "feature-a", + _terminate_services_from_state=lambda *_args, **_kwargs: {"feature-a Backend"}, + _release_requirement_ports=lambda _requirements: self.fail("failed replacement must not release ports"), + ) - for reason, flags in ( - ("no_matching_state", {}), - ("startup_fingerprint_mismatch", {"runtime_scope": "dependencies"}), - ): - route = Route(command="start", mode="trees", raw_args=["start"], flags=flags) - session = SimpleNamespace(effective_route=route, runtime_mode="trees") + with self.assertRaisesRegex(RuntimeError, "feature-a Backend"): replace_existing_project_services_for_fresh_start( runtime=runtime, session=session, candidate_state=candidate_state, - reason=reason, - fresh_start_replacement_services=lambda session, *, candidate_state: self.fail("selection should skip"), - announce_session_identifiers=lambda session: self.fail("announce should skip"), - report_progress=lambda route, message: self.fail("progress should skip"), - terminate_restart_orphan_listeners=lambda **kwargs: self.fail("orphan cleanup should skip"), + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda _route, _message: None, + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("must not reap after failure"), ) - def test_fresh_start_replacement_services_honors_restart_service_type_filters(self) -> None: + self.assertEqual(set(session.preserved_services), {"sentinel", "feature-a Backend"}) + self.assertEqual(set(session.preserved_requirements), {"sentinel", "feature-a"}) + self.assertTrue(session.base_metadata["sentinel"]) + self.assertTrue(session.base_metadata["keep"]) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-existing"]) + self.assertTrue(session.preserve_existing_state_on_failure) + + def test_explicit_no_resume_missing_termination_result_keeps_previous_state_authoritative(self) -> None: + route = Route(command="start", mode="trees", flags={"no_resume": True}) + service = SimpleNamespace(project="feature-a") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": service}, + requirements={"feature-a": SimpleNamespace(project="feature-a")}, + metadata={}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "feature-a", + _terminate_services_from_state=lambda *_args, **_kwargs: None, + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + ) + + with self.assertRaisesRegex(RuntimeError, "feature-a Backend"): + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda *_args: None, + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("orphans must not be reaped"), + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, candidate_state.services) + self.assertEqual(session.preserved_requirements, candidate_state.requirements) + + def test_fresh_start_preserves_source_authority_when_termination_raises(self) -> None: + service = SimpleNamespace(project="feature-a") + requirements = SimpleNamespace(project="feature-a") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": service}, + requirements={"feature-a": requirements}, + metadata={"authority": "old"}, + ) + session = SimpleNamespace( + effective_route=Route(command="start", mode="trees", flags={"no_resume": True}), + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + + def fail_termination(*_args, **_kwargs): # noqa: ANN202 + raise OSError("service lock unlink failed") + + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "feature-a", + _terminate_services_from_state=fail_termination, + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + ) + + with self.assertRaisesRegex(OSError, "service lock unlink failed"): + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda *_args: None, + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("orphans must not be reaped"), + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, candidate_state.services) + self.assertEqual(session.preserved_requirements, candidate_state.requirements) + self.assertEqual(session.base_metadata["authority"], "old") + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-existing"]) + + def test_fresh_start_preserves_source_authority_when_requirement_release_raises(self) -> None: + requirements = SimpleNamespace(project="feature-a") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={}, + requirements={"feature-a": requirements}, + metadata={"authority": "old"}, + ) + session = SimpleNamespace( + effective_route=Route(command="start", mode="trees", flags={"no_resume": True}), + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + + def fail_release(_requirements): # noqa: ANN001, ANN202 + raise OSError("requirement lock unlink failed") + + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "", + _terminate_services_from_state=lambda *_args, **_kwargs: self.fail("no services to terminate"), + _release_requirement_ports=fail_release, + ) + + with self.assertRaisesRegex(OSError, "requirement lock unlink failed"): + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: self.fail("no announcement expected"), + report_progress=lambda *_args: self.fail("no progress expected"), + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("no orphan scan expected"), + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_requirements, candidate_state.requirements) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-existing"]) + + def test_fresh_start_aborts_when_orphan_exit_is_unconfirmed(self) -> None: + route = Route(command="start", mode="trees", flags={"no_resume": True}) + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": SimpleNamespace(project="feature-a")}, + requirements={"feature-a": SimpleNamespace(project="feature-a")}, + metadata={}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "feature-a", + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + ) + + with self.assertRaisesRegex(RuntimeError, "orphan listeners.*4242"): + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda *_args: None, + terminate_restart_orphan_listeners=lambda **_kwargs: {4242}, + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, candidate_state.services) + self.assertEqual(session.preserved_requirements, candidate_state.requirements) + + def test_explicit_no_resume_supersedes_requirement_only_state(self) -> None: route = Route( - command="restart", - mode="main", - raw_args=["restart"], - flags={"_restart_request": True, "restart_service_types": ["frontend"]}, + command="start", + mode="trees", + projects=["feature-a"], + flags={"no_resume": True}, + ) + target_requirements = SimpleNamespace(project="feature-a") + other_requirements = SimpleNamespace(project="feature-b") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={}, + requirements={"feature-a": target_requirements, "feature-b": other_requirements}, + metadata={}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + released: list[object] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "", + _terminate_services_from_state=lambda *_args, **_kwargs: self.fail("no service termination expected"), + _release_requirement_ports=released.append, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: self.fail("empty replacement needs no announcement"), + report_progress=lambda _route, _message: self.fail("empty replacement needs no progress"), + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("no orphan scan expected"), + ) + + self.assertEqual(released, [target_requirements]) + self.assertEqual(session.preserved_requirements, {"feature-b": other_requirements}) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-existing"]) + + def test_no_reusable_runtime_still_supersedes_requirement_only_state(self) -> None: + target_requirements = SimpleNamespace(project="feature-a") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={}, + requirements={"feature-a": target_requirements}, + metadata={"existing": True}, + ) + session = SimpleNamespace( + effective_route=Route(command="start", mode="trees", flags={}), + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + released: list[object] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "", + _terminate_services_from_state=lambda *_args, **_kwargs: self.fail("no services should be stopped"), + _release_requirement_ports=released.append, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="no_reusable_runtime", + announce_session_identifiers=lambda _session: self.fail("no services need an announcement"), + report_progress=lambda _route, _message: self.fail("no services need progress"), + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("no orphan scan expected"), + ) + + self.assertEqual(released, [target_requirements]) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-existing"]) + + def test_dependencies_only_fresh_start_preserves_services_but_replaces_selected_requirements(self) -> None: + target_service = SimpleNamespace(project="feature-a") + other_service = SimpleNamespace(project="feature-b") + target_requirements = SimpleNamespace(project="feature-a") + other_requirements = SimpleNamespace(project="feature-b") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": target_service, "feature-b Backend": other_service}, + requirements={"feature-a": target_requirements, "feature-b": other_requirements}, + metadata={}, ) + session = SimpleNamespace( + effective_route=Route(command="start", mode="trees", flags={"runtime_scope": "dependencies"}), + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + released: list[object] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "feature-a", + _terminate_services_from_state=lambda *_args, **_kwargs: self.fail("app services must be preserved"), + _release_requirement_ports=released.append, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="startup_fingerprint_mismatch", + announce_session_identifiers=lambda _session: self.fail("app services must be preserved"), + report_progress=lambda _route, _message: self.fail("app services must be preserved"), + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("no orphan scan expected"), + ) + + self.assertEqual(session.preserved_services, candidate_state.services) + self.assertEqual(session.preserved_requirements, {"feature-b": other_requirements}) + self.assertEqual(released, [target_requirements]) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-existing"]) + + def test_fresh_start_replacement_services_does_not_leave_disabled_service_types_running(self) -> None: candidate_state = SimpleNamespace( services={ "Main Backend": SimpleNamespace(name="Main Backend", project="Main", type="backend"), @@ -279,14 +860,11 @@ def test_fresh_start_replacement_services_honors_restart_service_type_filters(se self.assertEqual( fresh_start_replacement_services( - route=route, selected_contexts=[SimpleNamespace(name="Main")], candidate_state=candidate_state, - configured_service_types={"backend", "frontend"}, - additional_services=(), project_name_from_service=lambda name: str(name).removesuffix(" Backend").removesuffix(" Frontend"), ), - {"Main Frontend"}, + {"Main Backend", "Main Frontend"}, ) diff --git a/tests/python/startup/test_selected_context_startup.py b/tests/python/startup/test_selected_context_startup.py index a752e410e..e6bbc79bb 100644 --- a/tests/python/startup/test_selected_context_startup.py +++ b/tests/python/startup/test_selected_context_startup.py @@ -1,18 +1,27 @@ from __future__ import annotations from contextlib import nullcontext +import concurrent.futures from pathlib import Path from types import SimpleNamespace import unittest +from unittest.mock import patch from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.service_manager import ServiceCleanupError +from envctl_engine.startup.startup_execution_support import ProjectStartupFailure from envctl_engine.startup.selected_context_startup import ( record_project_startup, start_selected_contexts, start_selected_contexts_with_runtime, ) -from envctl_engine.startup.session import ProjectStartupResult, StartupSession -from envctl_engine.state.models import RequirementsResult +from envctl_engine.startup.session import ( + ProjectStartupResult, + StartupSession, + track_unterminated_services, + unconfirmed_service_names, +) +from envctl_engine.state.models import RequirementsResult, ServiceRecord class _SpinnerStub: @@ -77,6 +86,427 @@ def _start_project_context(self, *, context, mode: str, route: Route, run_id: st class SelectedContextStartupTests(unittest.TestCase): + def test_blank_termination_identity_fails_closed_for_all_services(self) -> None: + services = {"Main Backend": object(), "Main Frontend": object()} + + self.assertEqual(unconfirmed_service_names([" "], services), set(services)) + self.assertEqual(unconfirmed_service_names(["", "Main Backend"], services), set(services)) + + def test_parallel_renderer_error_occurs_only_after_successful_resources_are_recorded(self) -> None: + runtime = _RuntimeStub() + runtime._tree_parallel_startup_config = ( # type: ignore[method-assign] + lambda **_kwargs: (True, 2) + ) + route = Route(command="start", mode="trees", flags={}) + contexts = [self._context("Alpha"), self._context("Beta")] + session = self._session(route=route, contexts=contexts) + + with self.assertRaisesRegex(OSError, "renderer failed"): + start_selected_contexts( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: False, + resolved_run_id=lambda _session: "run-1", + record_project_startup=record_project_startup, + render_project_startup_warnings=lambda **_kwargs: (_ for _ in ()).throw(OSError("renderer failed")), + should_degrade_to_plan_agent_handoff=lambda _session, _error: False, + record_plan_agent_handoff_local_startup_failure=lambda **_kwargs: None, + spinner_factory=_SpinnerFactory(), + use_spinner_policy_fn=lambda _policy: nullcontext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False, backend="rich"), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + project_spinner_group_factory=lambda **_kwargs: SimpleNamespace(), + ) + + self.assertEqual(set(session.services_by_project), {"Alpha", "Beta"}) + self.assertEqual(set(session.requirements_by_project), {"Alpha", "Beta"}) + self.assertEqual(set(session.started_context_names), {"Alpha", "Beta"}) + + def test_sequential_record_callback_interrupt_retains_returned_resources(self) -> None: + runtime = _RuntimeStub() + context = self._context("Alpha") + requirements = RequirementsResult(project=context.name) + service = ServiceRecord( + name="Alpha Backend", + type="backend", + cwd="/alpha", + pid=8101, + ) + runtime._start_project_context = lambda **_kwargs: ProjectStartupResult( # type: ignore[method-assign] + requirements=requirements, + services={service.name: service}, + ) + route = Route(command="start", mode="trees", flags={}) + session = self._session(route=route, contexts=[context]) + cancellation = KeyboardInterrupt("cancel during result handoff") + + with self.assertRaises(KeyboardInterrupt) as raised: + start_selected_contexts( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: False, + resolved_run_id=lambda _session: "run-1", + record_project_startup=lambda *_args: (_ for _ in ()).throw(cancellation), + render_project_startup_warnings=lambda **_kwargs: None, + should_degrade_to_plan_agent_handoff=lambda _session, _error: False, + record_plan_agent_handoff_local_startup_failure=lambda **_kwargs: None, + spinner_factory=_SpinnerFactory(), + use_spinner_policy_fn=lambda _policy: nullcontext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False, backend="rich"), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + project_spinner_group_factory=lambda **_kwargs: SimpleNamespace(), + ) + + self.assertIs(raised.exception, cancellation) + self.assertIs(session.requirements_by_project[context.name], requirements) + self.assertEqual( + {tracked.pid for tracked in session.unterminated_services.values()}, + {service.pid}, + ) + + def test_parallel_main_thread_interrupt_drains_every_successful_future(self) -> None: + runtime = _RuntimeStub() + runtime._tree_parallel_startup_config = lambda **_kwargs: (True, 3) # type: ignore[method-assign] + contexts = [self._context(name) for name in ("Alpha", "Beta", "Gamma")] + expected_pids = {context.name: 8200 + index for index, context in enumerate(contexts, start=1)} + + def start_context(*, context, **_kwargs): # noqa: ANN001, ANN202 + service = ServiceRecord( + name=f"{context.name} Backend", + type="backend", + cwd=f"/{context.name.lower()}", + pid=expected_pids[context.name], + ) + return ProjectStartupResult( + requirements=RequirementsResult(project=context.name), + services={service.name: service}, + ) + + runtime._start_project_context = start_context # type: ignore[method-assign] + route = Route(command="start", mode="trees", flags={}) + session = self._session(route=route, contexts=contexts) + cancellation = KeyboardInterrupt("cancel while collecting parallel results") + + def interrupting_as_completed(futures): # noqa: ANN001, ANN202 + done, _ = concurrent.futures.wait( + futures, + return_when=concurrent.futures.FIRST_COMPLETED, + ) + yield next(iter(done)) + raise cancellation + + with ( + patch( + "envctl_engine.startup.selected_context_startup.concurrent.futures.as_completed", + side_effect=interrupting_as_completed, + ), + self.assertRaises(KeyboardInterrupt) as raised, + ): + start_selected_contexts( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: False, + resolved_run_id=lambda _session: "run-1", + record_project_startup=record_project_startup, + render_project_startup_warnings=lambda **_kwargs: None, + should_degrade_to_plan_agent_handoff=lambda _session, _error: False, + record_plan_agent_handoff_local_startup_failure=lambda **_kwargs: None, + spinner_factory=_SpinnerFactory(), + use_spinner_policy_fn=lambda _policy: nullcontext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False, backend="rich"), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + project_spinner_group_factory=lambda **_kwargs: SimpleNamespace(), + ) + + self.assertIs(raised.exception, cancellation) + tracked_pids = { + service.pid + for project_services in session.services_by_project.values() + for service in project_services.values() + }.union(service.pid for service in session.unterminated_services.values()) + self.assertEqual(tracked_pids, set(expected_pids.values())) + self.assertEqual( + {requirements.project for requirements in session.requirements_by_project.values()}, + set(expected_pids), + ) + + def test_parallel_failure_callback_error_does_not_hide_later_success(self) -> None: + runtime = _RuntimeStub() + runtime._tree_parallel_startup_config = lambda **_kwargs: (True, 2) # type: ignore[method-assign] + + def start_context(*, context, **_kwargs): # noqa: ANN001, ANN202 + if context.name == "Alpha": + raise RuntimeError("alpha failed") + return ProjectStartupResult( + requirements=RequirementsResult(project=context.name), + services={"Beta Backend": object()}, + ) + + runtime._start_project_context = start_context # type: ignore[method-assign] + original_emit = runtime._emit + + def emit(event: str, **payload: object) -> None: + if event == "startup.project.failed": + raise OSError("event sink failed") + original_emit(event, **payload) + + runtime._emit = emit # type: ignore[method-assign] + route = Route(command="start", mode="trees", flags={}) + session = self._session(route=route, contexts=[self._context("Alpha"), self._context("Beta")]) + + with self.assertRaisesRegex(RuntimeError, "alpha failed"): + start_selected_contexts( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: False, + resolved_run_id=lambda _session: "run-1", + record_project_startup=record_project_startup, + render_project_startup_warnings=lambda **_kwargs: None, + should_degrade_to_plan_agent_handoff=lambda _session, _error: False, + record_plan_agent_handoff_local_startup_failure=lambda **_kwargs: None, + spinner_factory=_SpinnerFactory(), + use_spinner_policy_fn=lambda _policy: nullcontext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False, backend="rich"), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + project_spinner_group_factory=lambda **_kwargs: SimpleNamespace(), + ) + + self.assertIn("Beta", session.services_by_project) + self.assertIn("Beta", session.requirements_by_project) + + def test_parallel_name_collision_drains_and_records_every_completed_future(self) -> None: + runtime = _RuntimeStub() + runtime._tree_parallel_startup_config = lambda **_kwargs: (True, 3) # type: ignore[method-assign] + + def start_context(*, context, **_kwargs): # noqa: ANN001, ANN202 + if context.name == "Gamma": + service = ServiceRecord( + name="Gamma Worker", + type="worker", + cwd="/gamma", + pid=303, + project="Gamma", + service_slug="worker", + ) + else: + service = ServiceRecord( + name="Opaque Shared Runtime", + type="worker", + cwd=f"/{context.name.lower()}", + pid=101 if context.name == "Alpha" else 202, + project=context.name, + service_slug="worker", + ) + return ProjectStartupResult( + requirements=RequirementsResult(project=context.name), + services={service.name: service}, + ) + + runtime._start_project_context = start_context # type: ignore[method-assign] + route = Route(command="start", mode="trees", flags={}) + contexts = [self._context(name) for name in ("Alpha", "Beta", "Gamma")] + session = self._session(route=route, contexts=contexts) + + with self.assertRaisesRegex(RuntimeError, "tracked startup state"): + start_selected_contexts( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: False, + resolved_run_id=lambda _session: "run-1", + record_project_startup=record_project_startup, + render_project_startup_warnings=lambda **_kwargs: None, + should_degrade_to_plan_agent_handoff=lambda _session, _error: False, + record_plan_agent_handoff_local_startup_failure=lambda **_kwargs: None, + spinner_factory=_SpinnerFactory(), + use_spinner_policy_fn=lambda _policy: nullcontext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False, backend="rich"), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + project_spinner_group_factory=lambda **_kwargs: SimpleNamespace(), + ) + + tracked = { + service.pid + for services in session.services_by_project.values() + for service in services.values() + }.union(service.pid for service in session.unterminated_services.values()) + self.assertEqual(tracked, {101, 202, 303}) + self.assertIn("Gamma", session.requirements_by_project) + + def test_unterminated_service_error_is_carried_into_session_state(self) -> None: + route = Route(command="start", mode="main", flags={}) + session = self._session(route=route, contexts=[]) + service = ServiceRecord(name="Main Backend", type="backend", cwd="/repo", pid=63001) + failure = ServiceCleanupError("cleanup failed", {service.name: service}) + + track_unterminated_services(session, failure) + + self.assertIs(session.unterminated_services[service.name], service) + self.assertIs(session.merged_services[service.name], service) + + def test_parallel_failure_records_with_duplicate_names_keep_both_processes(self) -> None: + route = Route(command="start", mode="trees", flags={}) + session = self._session(route=route, contexts=[]) + alpha = ServiceRecord( + name="Opaque Worker", + type="worker", + cwd="/alpha", + pid=111, + project="Alpha", + service_slug="worker", + ) + beta = ServiceRecord( + name="Opaque Worker", + type="worker", + cwd="/beta", + pid=222, + project="Beta", + service_slug="worker", + ) + + track_unterminated_services(session, ServiceCleanupError("alpha cleanup failed", {alpha.name: alpha})) + track_unterminated_services(session, ServiceCleanupError("beta cleanup failed", {beta.name: beta})) + + self.assertEqual({service.pid for service in session.unterminated_services.values()}, {111, 222}) + self.assertEqual(len(session.unterminated_services), 2) + self.assertTrue(any("Restart Collision" in name for name in session.unterminated_services)) + + def test_merged_services_rejects_preserved_new_name_collision(self) -> None: + route = Route(command="restart", mode="main", flags={}) + session = self._session(route=route, contexts=[]) + preserved = ServiceRecord(name="Main Backend", type="backend", cwd="/repo/old", pid=63001) + replacement = ServiceRecord(name="Main Backend", type="backend", cwd="/repo/new", pid=63002) + session.preserved_services[preserved.name] = preserved + session.services_by_project["Main"] = {replacement.name: replacement} + + with self.assertRaisesRegex( + RuntimeError, + "Refusing to overwrite preserved service state.*Main Backend", + ): + _ = session.merged_services + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.service_state_collisions, {"Main Backend"}) + + def test_merged_requirements_rejects_preserved_new_owner_collision(self) -> None: + route = Route(command="restart", mode="main", flags={}) + session = self._session(route=route, contexts=[]) + session.preserved_requirements["Main"] = RequirementsResult( + project="Main", + redis={"enabled": True, "final": 6380}, + ) + session.requirements_by_project["Main"] = RequirementsResult( + project="Main", + redis={"enabled": True, "final": 6381}, + ) + + with self.assertRaisesRegex(RuntimeError, "preserved requirement state.*Main"): + _ = session.merged_requirements + + self.assertTrue(session.preserve_existing_state_on_failure) + + def test_merged_requirements_deduplicates_reused_alias_authority_by_identity(self) -> None: + route = Route(command="restart", mode="main", flags={}) + session = self._session(route=route, contexts=[]) + requirements = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6381}, + ) + session.preserved_requirements["Main Restart Collision"] = requirements + session.requirements_by_project["Main"] = requirements + + merged = session.merged_requirements + + self.assertEqual(list(merged), ["Main Restart Collision"]) + self.assertIs(merged["Main Restart Collision"], requirements) + + def test_merged_requirements_ignores_disabled_projection_when_dependencies_are_skipped(self) -> None: + route = Route(command="start", mode="main", flags={"launch_dependencies": False}) + session = self._session(route=route, contexts=[]) + preserved = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6379}, + ) + session.preserved_requirements["Main Historical Authority"] = preserved + session.requirements_by_project["Main"] = RequirementsResult( + project="Main", + redis={"enabled": False, "success": True, "final": 6482}, + ) + + merged = session.merged_requirements + + self.assertEqual(list(merged), ["Main Historical Authority"]) + self.assertIs(merged["Main Historical Authority"], preserved) + + def test_record_project_startup_tracks_collision_for_failure_cleanup(self) -> None: + route = Route(command="restart", mode="main", flags={}) + context = self._context("Main") + session = self._session(route=route, contexts=[context]) + preserved = ServiceRecord(name="Main Backend", type="backend", cwd="/repo/old", pid=63001) + replacement = ServiceRecord(name="Main Backend", type="backend", cwd="/repo/new", pid=63002) + session.preserved_services[preserved.name] = preserved + requirements = RequirementsResult(project="Main") + + with self.assertRaisesRegex(RuntimeError, "Refusing to overwrite preserved service state"): + record_project_startup( + session, + context, + ProjectStartupResult(requirements=requirements, services={replacement.name: replacement}), + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + replacement_names = [ + name for name in session.unterminated_services if name.startswith("Main Backend Restart Collision") + ] + self.assertEqual(len(replacement_names), 1) + self.assertEqual(session.unterminated_services[replacement_names[0]].pid, replacement.pid) + self.assertEqual(session.unterminated_services[replacement_names[0]].name, replacement_names[0]) + self.assertIs(session.requirements_by_project["Main"], requirements) + self.assertEqual(session.services_by_project, {}) + + def test_duplicate_context_name_with_disjoint_services_retains_both_startups(self) -> None: + route = Route(command="start", mode="trees", flags={}) + context = self._context("Same") + session = self._session(route=route, contexts=[context, context]) + first = ServiceRecord(name="Opaque A", type="worker", cwd="/one", pid=1, project="Same") + second = ServiceRecord(name="Opaque B", type="worker", cwd="/two", pid=2, project="Same") + first_requirements = RequirementsResult(project="Same", redis={"enabled": True, "final": 6379}) + second_requirements = RequirementsResult(project="Same", redis={"enabled": True, "final": 6381}) + + record_project_startup( + session, + context, + ProjectStartupResult(requirements=first_requirements, services={first.name: first}), + ) + with self.assertRaisesRegex(RuntimeError, "project:Same"): + record_project_startup( + session, + context, + ProjectStartupResult(requirements=second_requirements, services={second.name: second}), + ) + + self.assertEqual({service.pid for service in session.services_by_project["Same"].values()}, {1}) + self.assertEqual({service.pid for service in session.unterminated_services.values()}, {2}) + self.assertEqual(set(session.requirements_by_project), {"Same", "Same Restart Collision"}) + self.assertIs(session.requirements_by_project["Same"], first_requirements) + self.assertIs(session.requirements_by_project["Same Restart Collision"], second_requirements) + + def test_project_startup_failure_carries_requirements_into_failure_state(self) -> None: + route = Route(command="start", mode="main", flags={}) + session = self._session(route=route, contexts=[]) + requirements = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6379}, + ) + failure = ProjectStartupFailure( + "backend failed", + project="Main", + requirements=requirements, + ) + + track_unterminated_services(session, failure) + + self.assertIs(session.requirements_by_project["Main"], requirements) + self.assertIs(session.merged_requirements["Main"], requirements) + @staticmethod def _context(name: str) -> SimpleNamespace: return SimpleNamespace(name=name, root=Path("/tmp") / name, ports={}) @@ -110,7 +540,9 @@ def test_start_selected_contexts_builds_execution_route_and_records_sequential_r session.services_by_project.__setitem__(context.name, result.services), session.started_context_names.append(context.name), ), - render_project_startup_warnings=lambda *, context, warnings, route, project_spinner_group: warnings and None, + render_project_startup_warnings=lambda *, context, warnings, route, project_spinner_group: ( + warnings and None + ), should_degrade_to_plan_agent_handoff=lambda session, error: False, record_plan_agent_handoff_local_startup_failure=lambda session, project_name, error: None, spinner_factory=_SpinnerFactory(), @@ -133,7 +565,9 @@ def test_start_selected_contexts_builds_execution_route_and_records_sequential_r self.assertIsNone(execution_route.flags["_spinner_update_project"]) self.assertTrue(execution_route.flags["debug_suppress_progress_output"]) self.assertEqual(runtime.events[0][0], "spinner.policy") - self.assertIn(("startup.execution", {"mode": "sequential", "workers": 1, "projects": ["Alpha"]}), runtime.events) + self.assertIn( + ("startup.execution", {"mode": "sequential", "workers": 1, "projects": ["Alpha"]}), runtime.events + ) def test_record_project_startup_updates_session_result_maps_and_started_names(self) -> None: route = Route(command="start", mode="trees", raw_args=["start"], passthrough_args=[], projects=[], flags={}) @@ -194,13 +628,12 @@ def test_runtime_bound_selected_context_startup_records_results_and_uses_owner_d runtime, session, resolved_run_id=lambda session: "run-1", - render_project_startup_warnings=lambda *, context, warnings, route, project_spinner_group: rendered_warnings.append( - (context.name, list(warnings)) + render_project_startup_warnings=lambda *, context, warnings, route, project_spinner_group: ( + rendered_warnings.append((context.name, list(warnings))) + ), + should_degrade_to_plan_agent_handoff=lambda session, error: ( + degradation_checks.append((session.requested_command, error)) or False ), - should_degrade_to_plan_agent_handoff=lambda session, error: degradation_checks.append( - (session.requested_command, error) - ) - or False, record_plan_agent_handoff_local_startup_failure=lambda session, project_name, error: local_failures.append( (project_name, error) ), diff --git a/tests/python/startup/test_service_attach_execution.py b/tests/python/startup/test_service_attach_execution.py index 0c61b31f4..dcbdf37b0 100644 --- a/tests/python/startup/test_service_attach_execution.py +++ b/tests/python/startup/test_service_attach_execution.py @@ -1,12 +1,18 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Callable, Iterable +import json from pathlib import Path +import tempfile from types import SimpleNamespace from typing import cast import unittest +from unittest.mock import patch -from envctl_engine.runtime.service_manager import ServiceStartDescriptor +from envctl_engine.runtime.docker_service_runtime import DockerServiceLaunch +from envctl_engine.runtime.service_manager import ServiceCleanupError, ServiceManager, ServiceStartDescriptor +from envctl_engine.shared.ports import PortPlanner +from envctl_engine.shared.process_runner import ProcessHandoffCleanupError from envctl_engine.startup.service_attach_execution import ServiceAttachRunner from envctl_engine.startup.service_execution_records import PreparedServiceLaunch from envctl_engine.state.models import PortPlan, ServiceRecord @@ -17,6 +23,579 @@ def port_plan(port: int) -> PortPlan: class ServiceAttachExecutionTests(unittest.TestCase): + def test_later_layer_failure_rolls_back_and_tracks_prior_layer_records(self) -> None: + project_root = Path("/tmp/envctl-project") + voice = SimpleNamespace( + name="voice-runtime", + start_cmd="python voice.py", + depends_on=("backend",), + start_order=25, + expect_listener=True, + critical=True, + env_suffix="VOICE_RUNTIME", + ) + worker = SimpleNamespace( + name="worker", + start_cmd="python worker.py", + depends_on=("voice-runtime",), + start_order=50, + expect_listener=False, + critical=True, + env_suffix="WORKER", + ) + attach_layers: list[tuple[str, ...]] = [] + backend_record = ServiceRecord( + name="Main Backend", + type="backend", + cwd=str(project_root / "backend"), + pid=61001, + actual_port=8000, + ) + + def generic_attach(**kwargs: object) -> dict[str, ServiceRecord]: + descriptors = tuple(cast(Iterable[ServiceStartDescriptor], kwargs["descriptors"])) + layer = tuple(descriptor.service_type for descriptor in descriptors) + attach_layers.append(layer) + if "worker" not in layer: + return {backend_record.name: backend_record} + raise RuntimeError("worker startup failed") + + runtime = SimpleNamespace( + services=SimpleNamespace(start_services_with_attach=generic_attach), + _terminate_started_services=lambda _services: (_ for _ in ()).throw(OSError("layer terminator crashed")), + config=SimpleNamespace( + app_service_by_name=lambda name: {"voice-runtime": voice, "worker": worker}.get(name) + ), + _conflict_remaining={}, + _emit=lambda *_args, **_kwargs: None, + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(), + port_allocator=SimpleNamespace( + reserve_next=lambda port, owner: port, + session_id="attach-session", + ), + project_name="Main", + project_root=project_root, + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=project_root / "backend", + frontend_cwd=project_root / "frontend", + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=project_root / "backend", + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ), + "worker": PreparedServiceLaunch( + service_name="worker", + cwd=project_root / "worker", + log_path="/logs/worker.txt", + requested_port=0, + env={}, + command_source="configured", + listener_expected=False, + ), + "voice-runtime": PreparedServiceLaunch( + service_name="voice-runtime", + cwd=project_root / "voice-runtime", + log_path="/logs/voice.txt", + requested_port=8010, + env={}, + command_source="configured", + ), + }, + selected_service_types={"backend", "voice-runtime", "worker"}, + additional_services=(voice, worker), + backend_listener_expected=True, + rebound_delta=0, + ) + + with self.assertRaises(ServiceCleanupError) as raised: + runner.start(attach_parallel=False, on_service_retry=lambda *_args: None) + + self.assertEqual(attach_layers, [("backend", "voice-runtime"), ("worker",)]) + self.assertIs(raised.exception.unterminated_services["Main Backend"], backend_record) + self.assertEqual(backend_record.port_lock_session, "attach-session") + self.assertIn("cleanup error: layer terminator crashed", str(raised.exception)) + + def test_process_launch_without_pid_never_invents_envctl_or_adjacent_pid(self) -> None: + project_root = Path("/tmp/envctl-project") + runtime = SimpleNamespace( + _conflict_remaining={}, + _emit=lambda *_args, **_kwargs: None, + _service_start_command_resolved=lambda **_kwargs: (["backend"], "configured"), + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(start_background=lambda *_args, **_kwargs: object()), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="Main", + project_root=project_root, + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=project_root / "backend", + frontend_cwd=project_root / "frontend", + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=project_root / "backend", + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + success, error, pid = runner.start_backend(8000) + + self.assertTrue(success) + self.assertIsNone(error) + self.assertIsNone(pid) + + def test_post_spawn_telemetry_failure_still_returns_real_pid_to_manager(self) -> None: + project_root = Path("/tmp/envctl-project") + + def emit(event: str, **payload: object) -> None: + if event == "service.start" or payload.get("phase") == "process_launch": + raise OSError("event sink failed") + + runtime = SimpleNamespace( + _conflict_remaining={}, + _emit=emit, + _service_start_command_resolved=lambda **_kwargs: (["backend"], "configured"), + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(start_background=lambda *_args, **_kwargs: SimpleNamespace(pid=62001)), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="Main", + project_root=project_root, + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=project_root / "backend", + frontend_cwd=project_root / "frontend", + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=project_root / "backend", + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + success, error, pid = runner.start_backend(8000) + + self.assertTrue(success) + self.assertIsNone(error) + self.assertEqual(pid, 62001) + + def test_process_spawn_error_is_structured_without_listener_wait(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + runtime = SimpleNamespace( + _conflict_remaining={}, + _emit=lambda event, **payload: events.append((event, payload)), + _service_start_command_resolved=lambda **_kwargs: (["missing-backend"], "configured"), + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace( + start_background=lambda *_args, **_kwargs: (_ for _ in ()).throw( + FileNotFoundError("missing interpreter") + ) + ), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="Main", + project_root=Path("/repo"), + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=Path("/repo/backend"), + frontend_cwd=Path("/repo/frontend"), + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=Path("/repo/backend"), + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + success, error, pid = runner.start_backend(8000) + + self.assertFalse(success) + self.assertIn("process spawn failed for missing-backend", error) + self.assertIsNone(pid) + failure_events = [payload for event, payload in events if event == "service.failure"] + self.assertEqual(failure_events[0]["failure_class"], "process_spawn_failed") + + def test_unconfirmed_process_handoff_returns_pid_authority_to_service_manager(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + process = SimpleNamespace(pid=62002) + runtime = SimpleNamespace( + _conflict_remaining={}, + _emit=lambda event, **payload: events.append((event, payload)), + _service_start_command_resolved=lambda **_kwargs: (["backend"], "configured"), + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace( + start_background=lambda *_args, **_kwargs: (_ for _ in ()).throw( + ProcessHandoffCleanupError( + process=process, + launch_intent="background_service", + ) + ) + ), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="Main", + project_root=Path("/repo"), + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=Path("/repo/backend"), + frontend_cwd=Path("/repo/frontend"), + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=Path("/repo/backend"), + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + success, error, pid = runner.start_backend(8000) + + self.assertFalse(success) + self.assertIn("cleanup could not confirm exit", error) + self.assertEqual(pid, 62002) + failure_events = [payload for event, payload in events if event == "service.failure"] + self.assertEqual(failure_events[0]["failure_class"], "process_cleanup_unconfirmed") + + def test_explicit_docker_mode_launches_and_annotates_container_service(self) -> None: + project_root = Path("/tmp/envctl-project") + runtime = SimpleNamespace( + env={}, + config=SimpleNamespace(raw={}), + services=ServiceManager(), + _conflict_remaining={}, + _emit=lambda _event, **_payload: None, + _service_start_command_resolved=lambda **kwargs: self.fail( + f"host command resolution must be skipped: {kwargs}" + ), + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="Main", + project_root=project_root, + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=project_root / "backend", + frontend_cwd=project_root / "frontend", + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={"PORT": "8000"}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: {**extra, "PORT": str(port)}, + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=project_root / "backend", + log_path="/logs/backend.txt", + requested_port=8000, + env={"PORT": "8000"}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + docker_mode=True, + ) + launch = DockerServiceLaunch( + container_id="container-id", + container_name="envctl-app-main-backend", + image="example/backend:dev", + log_path="/logs/backend.txt", + ) + + with ( + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.launch", + return_value=launch, + ) as docker_launch, + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.wait_until_ready", + return_value=True, + ), + ): + records = runner.start(attach_parallel=False, on_service_retry=lambda *_args: None) + + record = records["Main Backend"] + self.assertEqual(record.runtime_kind, "docker") + self.assertEqual(record.container_id, "container-id") + self.assertEqual(record.container_name, "envctl-app-main-backend") + self.assertEqual(record.container_image, "example/backend:dev") + docker_launch.assert_called_once() + self.assertEqual(docker_launch.call_args.kwargs["command"], []) + + runtime.config.raw["ENVCTL_BACKEND_DOCKER_COMMAND"] = "uvicorn app:api --port 8000" + self.assertEqual(runner._core_service_command("backend", 8001), []) + + runtime.config.raw.pop("ENVCTL_BACKEND_DOCKER_COMMAND") + runtime.config.raw["ENVCTL_BACKEND_DOCKER_COMMAND_MODE"] = "service" + resolved_calls: list[dict[str, object]] = [] + runtime._service_start_command_resolved = lambda **kwargs: ( + resolved_calls.append(kwargs) or ["container-service", "--port", str(kwargs["port"])], + "configured", + ) + self.assertEqual( + runner._core_service_command("backend", 8001), + ["container-service", "--port", "8001"], + ) + self.assertEqual(resolved_calls, [{"service_name": "backend", "project_root": project_root, "port": 8001}]) + + def test_failed_optional_docker_readiness_removes_container_before_degrading(self) -> None: + project_root = Path("/tmp/envctl-project") + optional_service = SimpleNamespace( + name="voice-runtime", + start_cmd="python voice.py --port {port}", + depends_on=(), + start_order=25, + expect_listener=True, + critical=False, + env_suffix="VOICE_RUNTIME", + ) + runtime = SimpleNamespace( + env={"ENVCTL_VOICE_RUNTIME_DOCKER_IMAGE": "example/voice:dev"}, + config=SimpleNamespace( + raw={}, + app_service_by_name=lambda name: optional_service if name == "voice-runtime" else None, + ), + services=ServiceManager(), + _conflict_remaining={}, + _emit=lambda event, **payload: None, + _split_command=lambda *args, **kwargs: self.fail("host command resolution must be skipped"), + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="Main", + project_root=project_root, + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=project_root / "backend", + frontend_cwd=project_root / "frontend", + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: {**extra, "PORT": str(port)}, + prepared_launches={ + "voice-runtime": PreparedServiceLaunch( + service_name="voice-runtime", + cwd=project_root / "voice-runtime", + log_path="/logs/voice.txt", + requested_port=8010, + env={"PORT": "8010"}, + command_source="configured", + ) + }, + selected_service_types={"voice-runtime"}, + additional_services=(optional_service,), + backend_listener_expected=True, + rebound_delta=0, + docker_mode=True, + ) + launch = DockerServiceLaunch( + container_id="failed-container-id", + container_name="envctl-app-main-voice-runtime", + image="example/voice:dev", + log_path="/logs/voice.txt", + ) + + with ( + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.launch", + return_value=launch, + ), + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.wait_until_ready", + return_value=False, + ), + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.stop", + return_value=True, + ) as stop_container, + ): + records = runner.start(attach_parallel=False, on_service_retry=lambda *args: None) + + record = records["Main Voice Runtime"] + self.assertEqual(record.status, "degraded") + self.assertTrue(record.degraded) + self.assertIsNone(record.container_id) + self.assertEqual(runner._container_launches, {}) + stop_container.assert_called_once_with("failed-container-id") + + runner._container_launches["voice-runtime"] = launch + with patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.stop", + return_value=False, + ): + self.assertEqual( + runner._remove_failed_container_launch("voice-runtime", launch), + "failed to remove container", + ) + self.assertEqual(runner._container_launches, {"voice-runtime": launch}) + + def test_failed_critical_docker_cleanup_retains_container_identity_and_port_lock(self) -> None: + project_root = Path("/tmp/envctl-project") + releases: list[tuple[int, str]] = [] + runtime = SimpleNamespace( + env={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}, + config=SimpleNamespace(raw={}), + services=ServiceManager(), + _conflict_remaining={}, + _emit=lambda *_args, **_kwargs: None, + _service_start_command_resolved=lambda **_kwargs: [], + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(), + port_allocator=SimpleNamespace( + session_id="planner-session-docker", + reserve_next=lambda port, owner: port, + release=lambda port, *, owner: releases.append((port, owner)), + ), + project_name="Main", + project_root=project_root, + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=project_root / "backend", + frontend_cwd=project_root / "frontend", + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: {**extra, "PORT": str(port)}, + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=project_root / "backend", + log_path="/logs/backend.txt", + requested_port=8000, + env={"PORT": "8000"}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + docker_mode=True, + ) + launch = DockerServiceLaunch( + container_id="live-container-id", + container_name="envctl-app-main-backend", + image="example/backend:dev", + log_path="/logs/backend.txt", + launch_token="launch-token-live", + cleanup_pending_since=1_700_000_000.25, + ) + + with ( + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.launch", + return_value=launch, + ), + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.wait_until_ready", + return_value=False, + ), + patch( + "envctl_engine.startup.service_attach_execution.DockerServiceRuntime.stop", + return_value=False, + ) as stop_container, + self.assertRaises(ServiceCleanupError) as raised, + ): + runner.start(attach_parallel=False, on_service_retry=lambda *_args: None) + + self.assertEqual(stop_container.call_count, 2) + self.assertTrue( + all( + call.kwargs["pending_cleanup_since"] == 1_700_000_000.25 + for call in stop_container.call_args_list + ) + ) + self.assertEqual(runner._container_launches, {"backend": launch}) + self.assertEqual(releases, []) + record = raised.exception.unterminated_services["Main Backend"] + self.assertEqual(record.runtime_kind, "docker") + self.assertEqual(record.container_id, "live-container-id") + self.assertEqual(record.container_name, "envctl-app-main-backend") + self.assertEqual(record.container_image, "example/backend:dev") + self.assertEqual(record.container_launch_token, "launch-token-live") + self.assertEqual(record.container_cleanup_pending_since, 1_700_000_000.25) + self.assertEqual(record.actual_port, 8000) + self.assertEqual(record.port_lock_session, "planner-session-docker") + def test_runner_builds_layered_descriptors_and_preserves_additional_urls(self) -> None: project_root = Path("/tmp/envctl-project") events: list[tuple[str, dict[str, object]]] = [] @@ -92,14 +671,16 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: ) process_runtime = SimpleNamespace( start_background=lambda command, cwd, env, stdout_path, stderr_path: ( - process_commands.append((list(command), str(cwd), int(env["PORT"]))) - or SimpleNamespace(pid=4321) + process_commands.append((list(command), str(cwd), int(env["PORT"]))) or SimpleNamespace(pid=4321) ) ) runner = ServiceAttachRunner( runtime=runtime, process_runtime=process_runtime, - port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + port_allocator=SimpleNamespace( + reserve_next=lambda port, owner: port, + session_id="layered-session", + ), project_name="Main", project_root=project_root, backend_plan=port_plan(8000), @@ -152,12 +733,26 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: self.assertEqual(layers, [("backend", "voice-runtime"), ("worker",)]) self.assertEqual(records["Main Backend"].actual_port, 8101) + self.assertEqual( + {record.port_lock_session for record in records.values()}, + {"layered-session"}, + ) self.assertEqual(records["Main Voice Runtime"].actual_port, 8022) self.assertIsNone(records["Main Worker"].actual_port) self.assertEqual(records["Main Voice Runtime"].public_url, "http://localhost:8010") self.assertIn((["backend", "8000"], str(project_root / "backend"), 8000), process_commands) - self.assertIn((["voice.py", "8010", "voice-runtime", str(project_root / "voice-runtime")], str(project_root / "voice-runtime"), 8010), process_commands) - self.assertIn(("service.bind.skipped", {"project": "Main", "service": "worker", "reason": "listener_not_expected"}), events) + self.assertIn( + ( + ["voice.py", "8010", "voice-runtime", str(project_root / "voice-runtime")], + str(project_root / "voice-runtime"), + 8010, + ), + process_commands, + ) + self.assertIn( + ("service.bind.skipped", {"project": "Main", "service": "worker", "reason": "listener_not_expected"}), + events, + ) self.assertTrue( any( event == "service.attach.phase" @@ -182,7 +777,10 @@ def test_additional_listener_truth_failure_emits_failure_event(self) -> None: services=SimpleNamespace(), _conflict_remaining={}, _emit=lambda event, **payload: events.append((event, payload)), - _service_start_command_resolved=lambda service_name, project_root, port: ([service_name, str(port)], "configured"), + _service_start_command_resolved=lambda service_name, project_root, port: ( + [service_name, str(port)], + "configured", + ), _detect_service_actual_port=lambda **kwargs: None, _listener_truth_enforced=lambda: True, _service_listener_failure_detail=lambda **kwargs: "process exited", @@ -220,7 +818,7 @@ def test_additional_listener_truth_failure_emits_failure_event(self) -> None: with self.assertRaisesRegex( RuntimeError, - r"voice-runtime listener not detected for Main on port 8010 \(process exited\)", + r"voice-runtime process exited before opening its listener for Main on port 8010 \(process exited\)", ): runner.detect_additional_actual("voice-runtime", pid=4321, requested=8010) @@ -230,7 +828,7 @@ def test_additional_listener_truth_failure_emits_failure_event(self) -> None: { "project": "Main", "service": "voice-runtime", - "failure_class": "listener_not_detected", + "failure_class": "process_exited", "requested_port": 8010, "detail": "process exited", }, @@ -238,16 +836,76 @@ def test_additional_listener_truth_failure_emits_failure_event(self) -> None: events, ) + def test_missing_nested_executable_is_reported_as_dependency_failure(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + detail = ( + "process 2304354 exited; log_path: /tmp/voice-runtime.txt; " + "log: scripts/envctl/start-voice-runtime.sh: line 20: exec: python: not found" + ) + runtime = SimpleNamespace( + services=SimpleNamespace(), + _conflict_remaining={}, + _emit=lambda event, **payload: events.append((event, payload)), + _detect_service_actual_port=lambda **_kwargs: None, + _listener_truth_enforced=lambda: True, + _service_listener_failure_detail=lambda **_kwargs: detail, + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="refactoring-elevenlabs-agent-runtime-automation-and-pipecat-strangler-3", + project_root=Path("/repo"), + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=Path("/repo/backend"), + frontend_cwd=Path("/repo/frontend"), + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "voice-runtime": PreparedServiceLaunch( + service_name="voice-runtime", + cwd=Path("/repo/voice-runtime"), + log_path="/tmp/voice-runtime.txt", + requested_port=8251, + env={"PORT": "8251"}, + command_source="configured", + ) + }, + selected_service_types={"voice-runtime"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + with self.assertRaisesRegex( + RuntimeError, + "voice-runtime is missing a required executable or module", + ): + runner.detect_additional_actual("voice-runtime", pid=2304354, requested=8251) + + failure_events = [payload for event, payload in events if event == "service.failure"] + self.assertEqual(failure_events[0]["failure_class"], "dependency_missing") + self.assertEqual(failure_events[0]["detail"], detail) + def test_core_only_runner_uses_legacy_service_manager_when_generic_attach_is_missing(self) -> None: attach_kwargs: dict[str, object] = {} runtime = SimpleNamespace( services=SimpleNamespace( - start_project_with_attach=lambda **kwargs: attach_kwargs.update(kwargs) - or {"Main Backend": ServiceRecord(name="Main Backend", type="backend", cwd="/repo")} + start_project_with_attach=lambda **kwargs: ( + attach_kwargs.update(kwargs) + or {"Main Backend": ServiceRecord(name="Main Backend", type="backend", cwd="/repo")} + ) ), _conflict_remaining={}, _emit=lambda event, **payload: None, - _service_start_command_resolved=lambda service_name, project_root, port: ([service_name, str(port)], "configured"), + _service_start_command_resolved=lambda service_name, project_root, port: ( + [service_name, str(port)], + "configured", + ), _detect_service_actual_port=lambda **kwargs: kwargs["requested_port"], _listener_truth_enforced=lambda: True, _service_listener_failure_detail=lambda **kwargs: "", @@ -255,7 +913,10 @@ def test_core_only_runner_uses_legacy_service_manager_when_generic_attach_is_mis runner = ServiceAttachRunner( runtime=runtime, process_runtime=SimpleNamespace(start=lambda *args, **kwargs: SimpleNamespace(pid=123)), - port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + port_allocator=SimpleNamespace( + reserve_next=lambda port, owner: port, + session_id="legacy-attach-session", + ), project_name="Main", project_root=Path("/repo"), backend_plan=port_plan(8000), @@ -286,9 +947,135 @@ def test_core_only_runner_uses_legacy_service_manager_when_generic_attach_is_mis records = runner.start(attach_parallel=False, on_service_retry=lambda *args: None) self.assertIn("Main Backend", records) + self.assertEqual( + records["Main Backend"].port_lock_session, + "legacy-attach-session", + ) self.assertTrue(callable(attach_kwargs["start_backend"])) self.assertFalse(attach_kwargs["parallel_start"]) + def test_multiple_service_retries_release_every_failed_port_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="startup-session", + availability_checker=lambda _port: True, + ) + planner.reserve_next(8000, owner="Main:backend") + + def generic_attach(**kwargs: object) -> dict[str, ServiceRecord]: + reserve_next = cast(Callable[[int], int], kwargs["reserve_next"]) + on_retry = cast(Callable[[str, int, int, int, str | None], None], kwargs["on_retry"]) + retry_one = reserve_next(8001) + on_retry("backend", 8000, retry_one, 1, "address already in use") + retry_two = reserve_next(retry_one + 1) + on_retry("backend", retry_one, retry_two, 2, "address already in use") + return {} + + runtime = SimpleNamespace( + services=SimpleNamespace(start_services_with_attach=generic_attach), + _conflict_remaining={}, + _emit=lambda *_args, **_kwargs: None, + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(), + port_allocator=planner, + project_name="Main", + project_root=Path("/repo"), + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=Path("/repo/backend"), + frontend_cwd=Path("/repo/frontend"), + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=Path("/repo/backend"), + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + runner.start(attach_parallel=False, on_service_retry=lambda *_args: None) + + locks = list(Path(tmpdir).glob("*.lock")) + self.assertEqual([path.name for path in locks], ["8002.lock"]) + payload = json.loads(locks[0].read_text(encoding="utf-8")) + self.assertEqual(payload["owner"], "Main:services") + + def test_failed_rebound_frontend_releases_attempt_locks_without_bulk_session_release(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="startup-session", + availability_checker=lambda _port: True, + ) + planner.reserve_next(5173, owner="Main:frontend") + live_pids = {123} + process_runtime = SimpleNamespace( + start_background=lambda *_args, **_kwargs: SimpleNamespace(pid=123), + is_pid_running=lambda pid: pid in live_pids, + terminate_process_group=lambda pid, **_kwargs: live_pids.discard(pid) or True, + ) + runtime = SimpleNamespace( + services=ServiceManager(), + _conflict_remaining={}, + _emit=lambda *_args, **_kwargs: None, + _service_start_command_resolved=lambda **_kwargs: (["frontend"], "configured"), + _detect_service_actual_port=lambda **_kwargs: None, + _listener_truth_enforced=lambda: True, + _service_listener_failure_detail=lambda **_kwargs: "listener missing", + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=process_runtime, + port_allocator=planner, + project_name="Main", + project_root=Path("/repo"), + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=Path("/repo/backend"), + frontend_cwd=Path("/repo/frontend"), + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "frontend": PreparedServiceLaunch( + service_name="frontend", + cwd=Path("/repo/frontend"), + log_path="/logs/frontend.txt", + requested_port=5173, + env={}, + command_source="configured", + ) + }, + selected_service_types={"frontend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=2, + ) + + with self.assertRaisesRegex(RuntimeError, "listener not detected"): + runner.start(attach_parallel=False, on_service_retry=lambda *_args: None) + + # Failure finalization keeps the whole session when managed + # requirements remain, so per-attempt cleanup must be sufficient. + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/startup/test_service_manager.py b/tests/python/startup/test_service_manager.py index 06186f094..c21eb108d 100644 --- a/tests/python/startup/test_service_manager.py +++ b/tests/python/startup/test_service_manager.py @@ -1,15 +1,56 @@ from __future__ import annotations +import concurrent.futures +import threading import time import unittest from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" -from envctl_engine.runtime.service_manager import ServiceManager, ServiceStartDescriptor +from envctl_engine.runtime.service_manager import ( + ServiceCleanupError, + ServiceManager, + ServiceStartDescriptor, + ServiceStartError, +) class ServiceManagerTests(unittest.TestCase): + @staticmethod + def _tracking_manager(live_pids: set[int], terminated: list[int]) -> ServiceManager: + return ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + terminate_process_group=lambda pid, **_kwargs: ( + terminated.append(pid), + live_pids.discard(pid), + True, + )[-1], + ) + ) + + def test_pidless_runtime_descriptor_records_success_without_inventing_a_process(self) -> None: + records = ServiceManager().start_services_with_attach( + project="Main", + descriptors=( + ServiceStartDescriptor( + service_type="backend", + cwd="/tmp/main/backend", + requested_port=8000, + start=lambda _port: (True, None, None), + detect_actual=lambda pid, requested: requested if pid is None else self.fail("unexpected PID"), + pid_required=False, + ), + ), + reserve_next=lambda port: port, + ) + + self.assertIsNone(records["Main Backend"].pid) + self.assertEqual(records["Main Backend"].actual_port, 8000) + def test_backend_retries_bind_conflict_and_updates_final_port(self) -> None: manager = ServiceManager() attempts: list[int] = [] @@ -149,11 +190,14 @@ def fail_backend(_pid: int | None, _requested: int) -> int: ) def test_listener_detection_failure_is_not_retried_as_port_conflict(self) -> None: - manager = ServiceManager() attempts: list[int] = [] terminated: list[int] = [] - - manager.terminate_process_group = lambda pid, **kwargs: terminated.append(pid) or True # type: ignore[attr-defined] + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda pid, **_kwargs: terminated.append(pid) or True, + ) + ) def start_backend(port: int) -> tuple[bool, str | None, int | None]: attempts.append(port) @@ -226,7 +270,6 @@ def test_non_listener_service_can_start_without_persisted_ports(self) -> None: self.assertIsNone(service.requested_port) self.assertIsNone(service.actual_port) - def test_start_services_with_attach_returns_degraded_record_for_noncritical_failure(self) -> None: manager = ServiceManager() descriptors = ( @@ -330,9 +373,13 @@ def test_start_services_with_attach_starts_arbitrary_services_and_non_listeners( self.assertIsNone(records["Main Worker"].actual_port) def test_start_services_with_attach_cleans_partial_records_on_failure(self) -> None: - manager = ServiceManager() terminated: list[int] = [] - manager.terminate_process_group = lambda pid, **kwargs: terminated.append(pid) or True # type: ignore[attr-defined] + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda pid, **_kwargs: terminated.append(pid) or True, + ) + ) descriptors = ( ServiceStartDescriptor( @@ -361,6 +408,358 @@ def test_start_services_with_attach_cleans_partial_records_on_failure(self) -> N self.assertEqual(terminated, [41001]) + def test_detection_interrupt_terminates_launched_pid_and_preserves_cancellation(self) -> None: + live_pids: set[int] = set() + terminated: list[int] = [] + manager = self._tracking_manager(live_pids, terminated) + cancellation = KeyboardInterrupt("listener probe interrupted") + + def start(_port: int) -> tuple[bool, str | None, int | None]: + live_pids.add(61001) + return True, None, 61001 + + with self.assertRaises(KeyboardInterrupt) as raised: + manager.start_service_with_retry( + project="Main", + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=start, + reserve_next=lambda port: port, + detect_actual=lambda *_args: (_ for _ in ()).throw(cancellation), + ) + + self.assertIs(raised.exception, cancellation) + self.assertEqual(terminated, [61001]) + self.assertEqual(live_pids, set()) + + def test_sequential_interrupt_cleans_prior_service_and_preserves_cancellation(self) -> None: + live_pids: set[int] = set() + terminated: list[int] = [] + manager = self._tracking_manager(live_pids, terminated) + cancellation = KeyboardInterrupt("second descriptor interrupted") + + def start_backend(_port: int) -> tuple[bool, str | None, int | None]: + live_pids.add(62001) + return True, None, 62001 + + descriptors = ( + ServiceStartDescriptor( + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=start_backend, + detect_actual=lambda _pid, requested: requested, + ), + ServiceStartDescriptor( + service_type="frontend", + cwd="/repo/frontend", + requested_port=3000, + start=lambda _port: (_ for _ in ()).throw(cancellation), + ), + ) + + with self.assertRaises(KeyboardInterrupt) as raised: + manager.start_services_with_attach( + project="Main", + descriptors=descriptors, + reserve_next=lambda port: port, + ) + + self.assertIs(raised.exception, cancellation) + self.assertEqual(terminated, [62001]) + self.assertEqual(live_pids, set()) + + def test_parallel_interrupt_drains_futures_and_cleans_every_launched_service(self) -> None: + live_pids: set[int] = set() + terminated: list[int] = [] + manager = self._tracking_manager(live_pids, terminated) + cancellation = KeyboardInterrupt("parallel descriptor interrupted") + interrupt_started = threading.Event() + + def start_service(pid: int) -> tuple[bool, str | None, int | None]: + live_pids.add(pid) + return True, None, pid + + def interrupt_detection(_pid: int | None, _requested: int) -> int: + interrupt_started.set() + raise cancellation + + def start_slow_sibling(_port: int) -> tuple[bool, str | None, int | None]: + self.assertTrue(interrupt_started.wait(timeout=1.0)) + time.sleep(0.02) + return start_service(63003) + + descriptors = ( + ServiceStartDescriptor( + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=lambda _port: start_service(63001), + detect_actual=lambda _pid, requested: requested, + ), + ServiceStartDescriptor( + service_type="voice-runtime", + cwd="/repo/voice-runtime", + requested_port=8010, + start=lambda _port: start_service(63002), + detect_actual=interrupt_detection, + ), + ServiceStartDescriptor( + service_type="frontend", + cwd="/repo/frontend", + requested_port=3000, + start=start_slow_sibling, + detect_actual=lambda _pid, requested: requested, + ), + ) + + with self.assertRaises(KeyboardInterrupt) as raised: + manager.start_services_with_attach( + project="Main", + descriptors=descriptors, + reserve_next=lambda port: port, + parallel_start=True, + max_workers=3, + ) + + self.assertIs(raised.exception, cancellation) + self.assertEqual(set(terminated), {63001, 63002, 63003}) + self.assertEqual(live_pids, set()) + + def test_main_thread_parallel_interrupt_drains_late_success_before_cleanup(self) -> None: + live_pids: set[int] = set() + terminated: list[int] = [] + manager = self._tracking_manager(live_pids, terminated) + cancellation = KeyboardInterrupt("main thread interrupted") + launch_barrier = threading.Barrier(2) + + def start_service(pid: int, *, delay: float = 0.0) -> tuple[bool, str | None, int | None]: + live_pids.add(pid) + launch_barrier.wait(timeout=1.0) + if delay: + time.sleep(delay) + return True, None, pid + + descriptors = ( + ServiceStartDescriptor( + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=lambda _port: start_service(64001), + detect_actual=lambda _pid, requested: requested, + ), + ServiceStartDescriptor( + service_type="frontend", + cwd="/repo/frontend", + requested_port=3000, + start=lambda _port: start_service(64002, delay=0.03), + detect_actual=lambda _pid, requested: requested, + ), + ) + real_as_completed = concurrent.futures.as_completed + + def interrupted_as_completed(futures): # noqa: ANN001, ANN202 + completed = iter(real_as_completed(futures)) + yield next(completed) + raise cancellation + + with ( + patch("envctl_engine.runtime.service_manager.concurrent.futures.as_completed", interrupted_as_completed), + self.assertRaises(KeyboardInterrupt) as raised, + ): + manager.start_services_with_attach( + project="Main", + descriptors=descriptors, + reserve_next=lambda port: port, + parallel_start=True, + max_workers=2, + ) + + self.assertIs(raised.exception, cancellation) + self.assertEqual(set(terminated), {64001, 64002}) + self.assertEqual(live_pids, set()) + + def test_listener_failure_fails_closed_when_group_and_raw_cleanup_cannot_prove_exit(self) -> None: + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: False, + terminate=lambda *_args, **_kwargs: False, + ) + manager = ServiceManager(process_runner=runner) + + with ( + patch("envctl_engine.shared.process_termination._send_signal", return_value=True), + patch("envctl_engine.shared.process_termination.wait_for_pid_exit", side_effect=[False, False]), + self.assertRaises(ServiceCleanupError) as raised, + ): + manager.start_service_with_retry( + project="Main", + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=lambda _port: (True, None, 51001), + reserve_next=lambda port: port, + detect_actual=lambda *_args: (_ for _ in ()).throw(RuntimeError("backend listener not detected")), + ) + + record = raised.exception.unterminated_services["Main Backend"] + self.assertEqual(record.pid, 51001) + self.assertEqual(record.status, "termination_failed") + + def test_non_runtime_detection_exception_still_terminates_started_pid(self) -> None: + live_pids = {51501} + terminated: list[int] = [] + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + terminate_process_group=lambda pid, **_kwargs: (terminated.append(pid), live_pids.discard(pid), True)[ + -1 + ], + ) + ) + + with self.assertRaisesRegex(ServiceStartError, "listener probe crashed"): + manager.start_service_with_retry( + project="Main", + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=lambda _port: (True, None, 51501), + reserve_next=lambda port: port, + detect_actual=lambda *_args: (_ for _ in ()).throw(OSError("listener probe crashed")), + ) + + self.assertEqual(terminated, [51501]) + + def test_failed_launcher_pid_is_terminated_before_bind_retry(self) -> None: + attempts: list[int] = [] + terminated: list[int] = [] + live_pids = {52001} + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + terminate_process_group=lambda pid, **_kwargs: (terminated.append(pid), live_pids.discard(pid), True)[ + -1 + ], + ) + ) + + def start(port: int) -> tuple[bool, str | None, int | None]: + attempts.append(port) + if len(attempts) == 1: + return False, "address already in use", 52001 + return True, None, 52002 + + record = manager.start_service_with_retry( + project="Main", + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=start, + reserve_next=lambda port: port, + detect_actual=lambda _pid, requested: requested, + ) + + self.assertEqual(attempts, [8000, 8001]) + self.assertEqual(terminated, [52001]) + self.assertEqual(record.pid, 52002) + + def test_parallel_rollback_reports_every_service_whose_exit_is_unconfirmed(self) -> None: + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: False, + terminate=lambda *_args, **_kwargs: False, + ) + ) + descriptors = ( + ServiceStartDescriptor( + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=lambda _port: (True, None, 53001), + detect_actual=lambda _pid, requested: requested, + ), + ServiceStartDescriptor( + service_type="frontend", + cwd="/repo/frontend", + requested_port=3000, + start=lambda _port: (False, "boot failed", None), + max_retries=0, + ), + ) + + with ( + patch("envctl_engine.shared.process_termination._send_signal", return_value=True), + patch("envctl_engine.shared.process_termination.wait_for_pid_exit", side_effect=[False, False]), + self.assertRaises(ServiceCleanupError) as raised, + ): + manager.start_services_with_attach( + project="Main", + descriptors=descriptors, + reserve_next=lambda port: port, + parallel_start=True, + ) + + self.assertEqual(set(raised.exception.unterminated_services), {"Main Backend"}) + + def test_noncritical_listener_cleanup_failure_is_never_downgraded_to_pidless_record(self) -> None: + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: False, + terminate=lambda *_args, **_kwargs: False, + ) + ) + descriptor = ServiceStartDescriptor( + service_type="worker", + cwd="/repo/worker", + requested_port=0, + start=lambda _port: (True, None, 54001), + detect_actual=lambda *_args: (_ for _ in ()).throw(RuntimeError("worker attach failed")), + listener_expected=False, + critical=False, + ) + + with ( + patch("envctl_engine.shared.process_termination._send_signal", return_value=True), + patch("envctl_engine.shared.process_termination.wait_for_pid_exit", side_effect=[False, False]), + self.assertRaises(ServiceCleanupError) as raised, + ): + manager.start_services_with_attach( + project="Main", + descriptors=(descriptor,), + reserve_next=lambda port: port, + ) + + record = raised.exception.unterminated_services["Main Worker"] + self.assertEqual(record.pid, 54001) + self.assertEqual(record.status, "termination_failed") + + def test_success_without_valid_pid_is_tracked_as_unconfirmed_launch_even_for_non_listener(self) -> None: + manager = ServiceManager() + descriptor = ServiceStartDescriptor( + service_type="worker", + cwd="/repo/worker", + requested_port=0, + start=lambda _port: (True, None, None), + listener_expected=False, + critical=False, + ) + + with self.assertRaises(ServiceCleanupError) as raised: + manager.start_services_with_attach( + project="Main", + descriptors=(descriptor,), + reserve_next=lambda port: port, + ) + + record = raised.exception.unterminated_services["Main Worker"] + self.assertIsNone(record.pid) + self.assertEqual(record.status, "termination_failed") + self.assertFalse(record.listener_expected) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/startup/test_startup_dashboard_stopped_restore.py b/tests/python/startup/test_startup_dashboard_stopped_restore.py index af1dea6ad..ad6401b81 100644 --- a/tests/python/startup/test_startup_dashboard_stopped_restore.py +++ b/tests/python/startup/test_startup_dashboard_stopped_restore.py @@ -77,7 +77,9 @@ def test_startup_restores_dashboard_stopped_services_instead_of_auto_resuming_pa ) captured_state: dict[str, RunState] = {} - engine._try_load_existing_state = lambda mode=None, strict_mode_match=False: previous_state # type: ignore[method-assign] + engine._try_load_existing_state = ( # type: ignore[method-assign] + lambda mode=None, strict_mode_match=False, project_names=None: previous_state + ) engine._discover_projects = lambda mode: [context] # type: ignore[method-assign] engine._reserve_project_ports = lambda _context, route=None: None # type: ignore[method-assign] engine._resume = lambda route: self.fail("startup should start stopped services, not auto-resume") # type: ignore[method-assign] diff --git a/tests/python/startup/test_startup_finalization.py b/tests/python/startup/test_startup_finalization.py index 72cc316b0..3b2b34f0f 100644 --- a/tests/python/startup/test_startup_finalization.py +++ b/tests/python/startup/test_startup_finalization.py @@ -4,10 +4,12 @@ from types import SimpleNamespace import tempfile import unittest +from unittest.mock import patch from envctl_engine.config import load_config from envctl_engine.runtime.command_router import parse_route from envctl_engine.startup.finalization import ( + _write_artifacts_at_commit_boundary, emit_preserved_service_merge, failure_context_label, finalize_failed_startup, @@ -33,8 +35,10 @@ ) from envctl_engine.planning.plan_agent.models import CreatedPlanWorktree, PlanAgentLaunchOutcome, PlanAgentLaunchResult from envctl_engine.startup.finalization_failure import StartupFailureFinalizer -from envctl_engine.startup.session import LocalStartupFailure, StartupSession -from envctl_engine.state.models import RunState +from envctl_engine.startup.selected_context_startup import record_project_startup +from envctl_engine.startup.session import LocalStartupFailure, ProjectStartupResult, StartupSession +from envctl_engine.startup.lifecycle import _finalize_startup_exception, execute_startup_lifecycle +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord def _session(*, contexts: list[object], contexts_to_start: list[object] | None = None) -> StartupSession: @@ -72,6 +76,412 @@ def _runtime_stub(**overrides: object) -> SimpleNamespace: class StartupFinalizationTests(unittest.TestCase): + def test_direct_collision_is_disambiguated_before_cleanup_without_prior_merge_access(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + preserved = ServiceRecord(name="Main Backend", type="backend", cwd="/old", pid=41, project="Main") + replacement = ServiceRecord(name="Main Backend", type="backend", cwd="/new", pid=42, project="Main") + occupied = ServiceRecord( + name="Main Backend Restart Collision 42", + type="worker", + cwd="/other", + pid=43, + project="Aux", + service_slug="worker", + ) + session.preserved_services[preserved.name] = preserved + session.services_by_project["Main"] = {replacement.name: replacement} + session.unterminated_services[occupied.name] = occupied + writes: list[RunState] = [] + cleanup_requests: list[dict[str, ServiceRecord]] = [] + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda services: ( + cleanup_requests.append(dict(services)), + set(services), + )[1], + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="collision", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual(len(cleanup_requests), 1) + self.assertEqual({service.pid for service in cleanup_requests[0].values()}, {42, 43}) + self.assertEqual({service.pid for service in writes[0].services.values()}, {41, 42, 43}) + replacement_names = [name for name, service in writes[0].services.items() if service.pid == 42] + self.assertEqual(replacement_names, ["Main Backend Restart Collision 42 2"]) + self.assertTrue( + any( + row.get("replacement_name") == replacement_names[0] + and row.get("replacement_exit_unconfirmed") is True + for row in writes[0].metadata["startup_state_collisions"] + ) + ) + + def test_duplicate_new_service_names_both_survive_unconfirmed_cleanup(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Alpha"), SimpleNamespace(name="Beta")]) + alpha = ServiceRecord(name="Shared Runtime", type="worker", cwd="/alpha", pid=101, project="Alpha") + beta = ServiceRecord(name="Shared Runtime", type="worker", cwd="/beta", pid=202, project="Beta") + record_project_startup( + session, + SimpleNamespace(name="Alpha"), + ProjectStartupResult(requirements=RequirementsResult(project="Alpha"), services={alpha.name: alpha}), + ) + with self.assertRaisesRegex(RuntimeError, "tracked startup state"): + record_project_startup( + session, + SimpleNamespace(name="Beta"), + ProjectStartupResult(requirements=RequirementsResult(project="Beta"), services={beta.name: beta}), + ) + writes: list[RunState] = [] + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda services: set(services), + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="duplicate hook identity", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual({service.pid for service in writes[0].services.values()}, {101, 202}) + self.assertEqual(len(writes[0].services), 2) + self.assertTrue(any("Restart Collision" in name for name in writes[0].services)) + + def test_collision_failure_persists_unconfirmed_replacement_without_overwriting_authority(self) -> None: + context = SimpleNamespace(name="Main") + session = _session(contexts=[context]) + preserved = ServiceRecord(name="Main Backend", type="backend", cwd="/repo/old", pid=64000) + replacement = ServiceRecord(name="Main Backend", type="backend", cwd="/repo/new", pid=64001) + session.preserved_services[preserved.name] = preserved + preserved_requirements = RequirementsResult( + project="Main", + db={"enabled": True, "success": True, "final": 5432}, + ) + replacement_requirements = RequirementsResult( + project="Main", + db={"enabled": True, "success": True, "final": 5544}, + ) + session.preserved_requirements["Main"] = preserved_requirements + with self.assertRaisesRegex(RuntimeError, "Refusing to overwrite preserved service state") as raised: + record_project_startup( + session, + context, + ProjectStartupResult( + requirements=replacement_requirements, + services={replacement.name: replacement}, + ), + ) + + terminated: list[dict[str, ServiceRecord]] = [] + writes: list[RunState] = [] + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda services: ( + terminated.append(dict(services)), + {replacement.name}, + )[1], + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + result = finalize_failed_startup( + runtime=runtime, + session=session, + error=str(raised.exception), + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual(result, 1) + self.assertEqual(len(terminated), 1) + terminated_names = list(terminated[0]) + self.assertEqual(len(terminated_names), 1) + self.assertTrue(terminated_names[0].startswith("Main Backend Restart Collision")) + self.assertEqual(terminated[0][terminated_names[0]].pid, replacement.pid) + self.assertEqual(len(writes), 1) + collision_names = [name for name in writes[0].services if "Restart Collision" in name] + self.assertEqual(len(collision_names), 1) + self.assertEqual(writes[0].services[preserved.name].pid, preserved.pid) + self.assertEqual(writes[0].services[collision_names[0]].pid, replacement.pid) + collision_projects = [project for project in writes[0].requirements if "Restart Collision" in project] + self.assertEqual(len(collision_projects), 1) + self.assertEqual(writes[0].requirements["Main"].db["final"], 5432) + self.assertEqual(writes[0].requirements[collision_projects[0]].db["final"], 5544) + self.assertEqual(writes[0].requirements[collision_projects[0]].project, "Main") + self.assertEqual( + writes[0].metadata["startup_state_collisions"][0]["replacement_name"], + collision_names[0], + ) + self.assertIs(session.preserved_services[preserved.name], preserved) + self.assertEqual(set(session.unterminated_services), set(collision_names)) + + def test_failure_finalization_persists_service_when_exit_remains_unconfirmed(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=64001, + actual_port=8000, + ) + session.unterminated_services[service.name] = service + writes: list[RunState] = [] + released = {"value": False} + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("event sink failed")), + _terminate_started_services=lambda _services: {service.name}, + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + result = finalize_failed_startup( + runtime=runtime, + session=session, + error="listener truth failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace( + release_session=lambda: released.__setitem__("value", True) + ), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual(result, 1) + self.assertEqual(writes[0].metadata["termination_failed_services"], [service.name]) + self.assertIs(writes[0].services[service.name], service) + self.assertEqual(service.status, "termination_failed") + self.assertFalse(released["value"]) + + def test_unknown_termination_identity_fails_closed_for_every_started_service(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + backend = ServiceRecord(name="Main Backend", type="backend", cwd="/repo", pid=1) + frontend = ServiceRecord(name="Main Frontend", type="frontend", cwd="/repo", pid=2) + session.services_by_project["Main"] = {backend.name: backend, frontend.name: frontend} + writes: list[RunState] = [] + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: {"main backend"}, + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="cleanup adapter identity mismatch", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual(set(writes[0].services), {backend.name, frontend.name}) + self.assertEqual({service.status for service in writes[0].services.values()}, {"termination_failed"}) + + def test_failure_finalization_preserves_port_session_for_started_requirements(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + requirements = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6379}, + ) + session.requirements_by_project["Main"] = requirements + writes: list[RunState] = [] + released = {"value": False} + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: set(), + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="backend launch failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace( + release_session=lambda: released.__setitem__("value", True) + ), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertIs(writes[0].requirements["Main"], requirements) + self.assertFalse(released["value"]) + + def test_successful_rollback_is_removed_from_failure_state_when_requirements_remain(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=64002, + requested_port=8000, + actual_port=8000, + ) + session.started_context_names.append("Main") + session.services_by_project["Main"] = {service.name: service} + session.requirements_by_project["Main"] = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6379}, + ) + writes: list[RunState] = [] + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: set(), + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="frontend launch failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual(writes[0].services, {}) + self.assertEqual(session.services_by_project, {}) + self.assertEqual(session.unterminated_services, {}) + + def test_missing_termination_result_retains_service_and_port_session(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=64003, + requested_port=8000, + actual_port=8000, + ) + session.started_context_names.append("Main") + session.services_by_project["Main"] = {service.name: service} + writes: list[RunState] = [] + released = {"value": False} + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: None, + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="frontend launch failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace( + release_session=lambda: released.__setitem__("value", True) + ), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertIs(writes[0].services[service.name], service) + self.assertEqual(writes[0].metadata["termination_failed_services"], [service.name]) + self.assertEqual(service.status, "termination_failed") + self.assertFalse(released["value"]) + + def test_failure_finalization_releases_port_session_for_external_only_requirements(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + requirements = RequirementsResult( + project="Main", + redis={ + "enabled": True, + "success": True, + "external": True, + "final": 6379, + }, + ) + session.requirements_by_project["Main"] = requirements + released = {"value": False} + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: set(), + _write_artifacts=lambda *_args, **_kwargs: None, + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="backend launch failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace( + release_session=lambda: released.__setitem__("value", True) + ), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertTrue(released["value"]) + + def test_failure_finalization_releases_port_session_for_clean_internal_start_failure(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + session.requirements_by_project["Main"] = RequirementsResult( + project="Main", + redis={"enabled": True, "success": False, "final": 6379, "error": "command missing"}, + ) + released = {"value": False} + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: set(), + _write_artifacts=lambda *_args, **_kwargs: None, + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="requirements failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace( + release_session=lambda: released.__setitem__("value", True) + ), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertTrue(released["value"]) + + def test_failure_event_exception_does_not_skip_cleanup_or_failure_state_write(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + service = ServiceRecord(name="Main Backend", type="backend", cwd="/repo", pid=65001) + session.started_context_names.append("Main") + session.services_by_project["Main"] = {service.name: service} + terminated: list[dict[str, ServiceRecord]] = [] + writes: list[RunState] = [] + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("event sink failed")), + _terminate_started_services=lambda services: terminated.append(dict(services)) or set(), + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + result = finalize_failed_startup( + runtime=runtime, + session=session, + error="backend failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual(result, 1) + self.assertEqual(terminated, [{service.name: service}]) + self.assertEqual(len(writes), 1) + def test_failure_finalization_uses_named_owner(self) -> None: self.assertTrue(callable(StartupFailureFinalizer.finalize)) @@ -91,7 +501,7 @@ def test_finalize_failed_startup_writes_error_artifacts_and_releases_ports(self) env={}, _emit=lambda event, **payload: events.append((event, payload)), _write_artifacts=lambda state, contexts, *, errors: writes.append((state, list(contexts), list(errors))), - _terminate_started_services=lambda services: terminated.append(dict(services)), + _terminate_started_services=lambda services: terminated.append(dict(services)) or set(), ) result = finalize_failed_startup( @@ -101,8 +511,9 @@ def test_finalize_failed_startup_writes_error_artifacts_and_releases_ports(self) ensure_run_id=lambda session: None, port_allocator=lambda runtime: SimpleNamespace(release_session=lambda: released.__setitem__("value", True)), emit_phase=lambda session, phase, started_at, **extra: events.append((f"phase:{phase}", dict(extra))), - render_final_failure_status=lambda runtime, session, error, *, interactive_tty: rendered_statuses.append(error) - or error, + render_final_failure_status=lambda runtime, session, error, *, interactive_tty: ( + rendered_statuses.append(error) or error + ), ) self.assertEqual(result, 1) @@ -170,6 +581,39 @@ def test_finalize_failed_startup_preserves_existing_startup_failure_prefix_and_s events, ) + def test_finalize_failed_startup_does_not_shadow_existing_state_after_failed_replacement(self) -> None: + session = _session(contexts=[SimpleNamespace(name="feature-a")]) + session.preserve_existing_state_on_failure = True + events: list[tuple[str, dict[str, object]]] = [] + runtime = _runtime_stub( + _emit=lambda event, **payload: events.append((event, payload)), + _write_artifacts=lambda *_args, **_kwargs: self.fail("existing state must remain authoritative"), + _terminate_started_services=lambda _services: None, + ) + + result = finalize_failed_startup( + runtime=runtime, + session=session, + error="existing service did not terminate", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda _session, phase, _started_at, **extra: events.append((f"phase:{phase}", extra)), + render_final_failure_status=lambda _runtime, _session, error, *, interactive_tty: error, + ) + + self.assertEqual(result, 1) + self.assertIn( + ( + "state.failure_write.skipped", + { + "reason": "existing_runtime_state_remains_authoritative", + "run_id": "run-finalization", + }, + ), + events, + ) + self.assertIn(("phase:artifacts_write", {"status": "skipped"}), events) + def test_finalize_successful_startup_writes_artifacts_summary_snapshot_and_returns_zero(self) -> None: context = SimpleNamespace(name="Main") session = _session(contexts=[context]) @@ -216,6 +660,158 @@ def test_finalize_successful_startup_writes_artifacts_summary_snapshot_and_retur self.assertEqual(snapshots[0][0], "before_dashboard_entry") self.assertEqual(snapshots[0][1]["service_count"], 0) + def test_post_commit_finalization_error_marks_authority_committed_before_propagating(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + run_state = RunState(run_id="run-finalization", mode="trees") + writes: list[RunState] = [] + runtime = SimpleNamespace( + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(OSError, "phase sink unavailable"): + finalize_successful_startup( + runtime=runtime, + session=session, + ensure_run_id=lambda _session: None, + validate_plan_agent_handoff=lambda _session, *, phase: None, + build_success_run_state=lambda _runtime, _session: run_state, + emit_preserved_service_merge=lambda _session: None, + emit_phase=lambda *_args, **_kwargs: (_ for _ in ()).throw( + OSError("phase sink unavailable") + ), + requirements_timing_enabled=lambda _route: False, + suppress_timing_output=lambda _route: False, + print_startup_summary=lambda **_kwargs: None, + startup_breakdown_enabled=lambda _route: False, + suppress_progress_output=lambda _route: False, + print_restart_port_rebound_summary=lambda _session: None, + emit_snapshot=lambda *_args, **_kwargs: None, + headless_plan_output_only=lambda _session: False, + print_headless_plan_session_summary=lambda _session: None, + maybe_attach_plan_agent_terminal=lambda _session: None, + finalize_plan_agent_degraded_handoff=lambda _session: 1, + ) + + self.assertEqual(writes, [run_state]) + self.assertTrue(session.authority_committed) + + def test_artifact_writer_post_index_error_marks_authority_before_propagating(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + run_state = RunState(run_id="run-post-index", mode="main") + + def write_artifacts(_state, _contexts, *, errors, on_commit): # noqa: ANN001 + self.assertEqual(errors, []) + on_commit() + raise RuntimeError("alias promotion failed after index commit") + + runtime = SimpleNamespace( + _write_artifacts=write_artifacts, + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(RuntimeError, "alias promotion failed after index commit"): + finalize_successful_startup( + runtime=runtime, + session=session, + ensure_run_id=lambda _session: None, + validate_plan_agent_handoff=lambda _session, *, phase: None, + build_success_run_state=lambda _runtime, _session: run_state, + emit_preserved_service_merge=lambda _session: None, + emit_phase=lambda *_args, **_kwargs: None, + requirements_timing_enabled=lambda _route: False, + suppress_timing_output=lambda _route: False, + print_startup_summary=lambda **_kwargs: None, + startup_breakdown_enabled=lambda _route: False, + suppress_progress_output=lambda _route: False, + print_restart_port_rebound_summary=lambda _session: None, + emit_snapshot=lambda *_args, **_kwargs: None, + headless_plan_output_only=lambda _session: False, + print_headless_plan_session_summary=lambda _session: None, + maybe_attach_plan_agent_terminal=lambda _session: None, + finalize_plan_agent_degraded_handoff=lambda _session: 1, + ) + + self.assertTrue(session.authority_committed) + + def test_generic_kwargs_writer_is_never_probed_or_invoked_twice(self) -> None: + session = _session(contexts=[]) + run_state = RunState(run_id="run-proxy", mode="main") + calls: list[dict[str, object]] = [] + + def proxy_writer(*_args, **kwargs): # noqa: ANN003 + calls.append(dict(kwargs)) + + runtime = SimpleNamespace(_write_artifacts=proxy_writer) + + _write_artifacts_at_commit_boundary(runtime, session, run_state) + + self.assertEqual(calls, [{"errors": []}]) + self.assertTrue(session.authority_committed) + + def test_generic_kwargs_writer_type_error_is_not_retried(self) -> None: + session = _session(contexts=[]) + run_state = RunState(run_id="run-proxy-error", mode="main") + calls: list[dict[str, object]] = [] + + def proxy_writer(*_args, **kwargs): # noqa: ANN003 + calls.append(dict(kwargs)) + raise TypeError("unexpected keyword argument 'on_commit'") + + runtime = SimpleNamespace(_write_artifacts=proxy_writer) + + with self.assertRaisesRegex(TypeError, "unexpected keyword argument"): + _write_artifacts_at_commit_boundary(runtime, session, run_state) + + self.assertEqual(calls, [{"errors": []}]) + self.assertFalse(session.authority_committed) + + def test_committed_authority_exception_never_invokes_failure_finalizer(self) -> None: + session = _session(contexts=[]) + session.authority_committed = True + events: list[tuple[str, dict[str, object]]] = [] + runtime = SimpleNamespace(_emit=lambda event, **payload: events.append((event, payload))) + + result = _finalize_startup_exception( + runtime=runtime, + session=session, + error=OSError("terminal attach failed"), + finalize_failure=lambda **_kwargs: self.fail("committed authority must not be rewritten"), + ) + + self.assertEqual(result, 1) + self.assertEqual(events[-1][0], "startup.post_commit_error") + + def test_execute_startup_lifecycle_finalizes_then_reraises_keyboard_interrupt(self) -> None: + session = _session(contexts=[]) + runtime = SimpleNamespace() + orchestrator = SimpleNamespace(runtime=runtime) + cancellation = KeyboardInterrupt("cancel startup") + + def interrupt_phase(_session: StartupSession) -> None: + raise cancellation + + with ( + patch( + "envctl_engine.startup.lifecycle.create_startup_session", + return_value=session, + ), + patch( + "envctl_engine.startup.lifecycle._pre_start_phases", + return_value=(interrupt_phase,), + ), + patch( + "envctl_engine.startup.lifecycle.finalize_failed_startup", + return_value=1, + ) as finalize_failure, + self.assertRaises(KeyboardInterrupt) as raised, + ): + execute_startup_lifecycle(orchestrator, session.requested_route) + + self.assertIs(raised.exception, cancellation) + finalize_failure.assert_called_once() + self.assertEqual(finalize_failure.call_args.kwargs["error"], str(cancellation)) + def test_finalize_successful_startup_delegates_degraded_plan_agent_handoff(self) -> None: session = _session(contexts=[]) session.plan_agent_handoff_degraded = True @@ -275,7 +871,10 @@ def test_runtime_bound_success_finalization_writes_artifacts_and_snapshot(self) self.assertEqual(writes[0][1], [context]) self.assertEqual(summaries[0][1], [context]) self.assertIn(("validate", {"phase": "success_finalization"}), events) - self.assertIn("before_dashboard_entry", [payload.get("checkpoint") for event, payload in events if event == "ui.plan_handoff.snapshot"]) + self.assertIn( + "before_dashboard_entry", + [payload.get("checkpoint") for event, payload in events if event == "ui.plan_handoff.snapshot"], + ) def test_finalize_plan_agent_degraded_handoff_writes_artifacts_and_attach_result(self) -> None: session = _session(contexts=[]) @@ -319,9 +918,7 @@ def test_finalize_plan_agent_degraded_handoff_artifacts_returns_run_state(self) result = finalize_plan_agent_degraded_handoff_artifacts( runtime=SimpleNamespace( - _write_artifacts=lambda state, contexts, *, errors: writes.append( - (state, list(contexts), list(errors)) - ) + _write_artifacts=lambda state, contexts, *, errors: writes.append((state, list(contexts), list(errors))) ), session=session, ensure_run_id=lambda session: events.append(("ensure", {})), @@ -583,7 +1180,9 @@ def test_print_headless_plan_session_summary_uses_attach_target_override_without print_headless_plan_session_summary( session, attach_target=attach_target, - validate_plan_agent_handoff=lambda session, *, phase: self.fail("override attach target should skip validation"), + validate_plan_agent_handoff=lambda session, *, phase: self.fail( + "override attach target should skip validation" + ), print_fn=lines.append, ) @@ -771,7 +1370,13 @@ def test_restart_port_rebound_summary_lines_deduplicate_port_changes(self) -> No startup_event_index=1, ) events = [ - {"event": "port.rebound", "project": "ignored", "service": "backend", "restart_preferred_port": 1, "port": 2}, + { + "event": "port.rebound", + "project": "ignored", + "service": "backend", + "restart_preferred_port": 1, + "port": 2, + }, { "event": "port.rebound", "project": "feature-a-1", diff --git a/tests/python/startup/test_startup_fingerprint_replacement.py b/tests/python/startup/test_startup_fingerprint_replacement.py index d5d9f6df9..ce8967986 100644 --- a/tests/python/startup/test_startup_fingerprint_replacement.py +++ b/tests/python/startup/test_startup_fingerprint_replacement.py @@ -73,13 +73,16 @@ def test_start_fingerprint_mismatch_replaces_existing_project_services_before_ne terminate_calls: list[tuple[set[str] | None, bool]] = [] call_order: list[str] = [] - engine._try_load_existing_state = lambda mode=None, strict_mode_match=False: previous_state # type: ignore[method-assign] + engine._try_load_existing_state = ( # type: ignore[method-assign] + lambda mode=None, strict_mode_match=False, project_names=None: previous_state + ) engine._discover_projects = lambda mode: [context] # type: ignore[method-assign] engine._terminate_services_from_state = ( # type: ignore[method-assign] lambda state, selected_services, aggressive, verify_ownership: ( call_order.append("terminate"), terminate_calls.append((selected_services, verify_ownership)), - ) + set(), + )[-1] ) engine._start_project_context = ( # type: ignore[method-assign] lambda context, mode, route, run_id: ( @@ -109,7 +112,7 @@ def test_start_fingerprint_mismatch_replaces_existing_project_services_before_ne warnings=[], ), )[1] - ) + ) code = engine.dispatch(parse_route(["--main", "--managed-deps", "--batch"], env={})) diff --git a/tests/python/startup/test_startup_orchestrator_flow_reuse.py b/tests/python/startup/test_startup_orchestrator_flow_reuse.py index 86c8e7ce6..5d3d8f8f4 100644 --- a/tests/python/startup/test_startup_orchestrator_flow_reuse.py +++ b/tests/python/startup/test_startup_orchestrator_flow_reuse.py @@ -1,10 +1,116 @@ # ruff: noqa: F403,F405 from __future__ import annotations +import json +import threading + +from envctl_engine.runtime.lifecycle_operation_lease import ( + release_lifecycle_operation, + run_exclusive_lifecycle_operation, +) +from envctl_engine.state.run_index import StateSelector + from tests.python.startup.startup_orchestrator_flow_test_support import * class StartupOrchestratorFlowReuseTests(StartupOrchestratorFlowTestCase): + def test_start_heals_multiple_historical_active_owners_for_main(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repo(root) + runtime = root / "runtime" + historical_candidates = [] + + for _index in range(2): + engine = self._engine(repo, runtime) + with ( + patch.object(engine, "_should_enter_post_start_interactive", return_value=False), + redirect_stdout(StringIO()), + ): + code = engine.dispatch( + parse_route(["start", "--batch", "--no-infra"], env={"ENVCTL_DEFAULT_MODE": "main"}) + ) + self.assertEqual(code, 0) + historical_candidates.append( + engine.state_repository.run_index.candidates(StateSelector("main", ("main",)))[0] + ) + + engine.state_repository.run_index.replace_all(historical_candidates) + preserved_stale_locks: dict[str, str] = {} + for offset, candidate in enumerate(historical_candidates): + stale_lock = engine.port_planner.lock_dir / f"{44000 + offset}.lock" + stale_lock.write_text( + json.dumps( + { + "owner": "Main:db", + "session": f"historical-{candidate.run_id}", + "pid": 99999999, + "created_at": 0, + } + ), + encoding="utf-8", + ) + preserved_stale_locks[stale_lock.name] = stale_lock.read_text(encoding="utf-8") + third_engine = self._engine(repo, runtime) + with ( + patch.object(third_engine, "_should_enter_post_start_interactive", return_value=False), + redirect_stdout(StringIO()), + ): + third_code = third_engine.dispatch( + parse_route(["start", "--batch", "--no-infra"], env={"ENVCTL_DEFAULT_MODE": "main"}) + ) + + self.assertEqual(third_code, 0) + index_payload = json.loads((third_engine.runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(len(index_payload["entries"]), 1) + self.assertEqual( + set(index_payload["retired_run_ids"]), + {candidate.run_id for candidate in historical_candidates}, + ) + self.assertEqual( + { + path.name: path.read_text(encoding="utf-8") + for path in third_engine.port_planner.lock_dir.glob("*.lock") + }, + preserved_stale_locks, + ) + + def test_repeated_no_infra_start_supersedes_old_authority_without_port_locks(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repo(root) + runtime = root / "runtime" + first_engine = self._engine(repo, runtime) + + with ( + patch.object(first_engine, "_should_enter_post_start_interactive", return_value=False), + redirect_stdout(StringIO()), + ): + first_code = first_engine.dispatch( + parse_route(["start", "--batch", "--no-infra"], env={"ENVCTL_DEFAULT_MODE": "main"}) + ) + + self.assertEqual(first_code, 0) + self.assertEqual(list(first_engine.port_planner.lock_dir.glob("*.lock")), []) + first_index = json.loads((first_engine.runtime_root / "run_index.json").read_text(encoding="utf-8")) + first_run_id = first_index["entries"][0]["run_id"] + + second_engine = self._engine(repo, runtime) + with ( + patch.object(second_engine, "_should_enter_post_start_interactive", return_value=False), + redirect_stdout(StringIO()), + ): + second_code = second_engine.dispatch( + parse_route(["start", "--batch", "--no-infra"], env={"ENVCTL_DEFAULT_MODE": "main"}) + ) + + self.assertEqual(second_code, 0) + self.assertEqual(list(second_engine.port_planner.lock_dir.glob("*.lock")), []) + second_index = json.loads((second_engine.runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(len(second_index["entries"]), 1) + self.assertNotEqual(second_index["entries"][0]["run_id"], first_run_id) + self.assertIn(first_run_id, second_index["retired_run_ids"]) + def test_disabled_startup_reopens_existing_dashboard_run_when_identity_matches(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -33,7 +139,11 @@ def test_disabled_startup_reopens_existing_dashboard_run_when_identity_matches(s with ( patch.object(engine, "_discover_projects", return_value=[context]), patch.object(engine, "_try_load_existing_state", return_value=existing_state), - patch.object(engine, "_write_artifacts", side_effect=AssertionError("fresh dashboard state should not be written")), + patch.object( + engine, + "_write_artifacts", + side_effect=AssertionError("fresh dashboard state should not be written"), + ), patch.object(engine, "_should_enter_post_start_interactive", return_value=False), patch.object( engine.state_repository, @@ -146,17 +256,27 @@ def _record_launch(_runtime: object, *, route: object, created_worktrees: tuple[ patch.object( engine.planning_worktree_orchestrator, "last_plan_selection_result", - return_value=PlanSelectionResult(raw_projects=[], selected_contexts=[context], created_worktrees=()), + return_value=PlanSelectionResult( + raw_projects=[], selected_contexts=[context], created_worktrees=() + ), ), patch("envctl_engine.startup.lifecycle.launch_plan_agent_terminals", side_effect=_record_launch), patch("envctl_engine.planning.plan_agent.omx_transport._tmux_session_exists", return_value=True), - patch("envctl_engine.planning.plan_agent.omx_transport._tmux_display_message_succeeds", return_value=(True, "%42")), + patch( + "envctl_engine.planning.plan_agent.omx_transport._tmux_display_message_succeeds", + return_value=(True, "%42"), + ), patch.object(engine, "_write_artifacts"), patch.object(engine, "_should_enter_post_start_interactive", return_value=False), ): out = StringIO() with redirect_stdout(out): - code = engine.dispatch(parse_route(["--plan", "feature-a", "--omx", "--codex", "--headless"], env={"ENVCTL_DEFAULT_MODE": "trees"})) + code = engine.dispatch( + parse_route( + ["--plan", "feature-a", "--omx", "--codex", "--headless"], + env={"ENVCTL_DEFAULT_MODE": "trees"}, + ) + ) self.assertEqual(code, 0) self.assertEqual(captured_created_worktrees, [["feature-a-1"]]) @@ -166,7 +286,10 @@ def _record_launch(_runtime: object, *, route: object, created_worktrees: tuple[ rendered, ) self.assertIn("attach: tmux attach -t omx-feature-session", rendered) - self.assertIn("new session: ENVCTL_USE_REPO_WRAPPER=1 /tmp/repo/bin/envctl --plan feature-a --omx --codex --new-session --headless", rendered) + self.assertIn( + "new session: ENVCTL_USE_REPO_WRAPPER=1 /tmp/repo/bin/envctl --plan feature-a --omx --codex --new-session --headless", + rendered, + ) self.assertIn("kill: tmux kill-session -t omx-feature-session", rendered) def test_explicit_cmux_plan_launch_reuses_selected_existing_worktree(self) -> None: @@ -207,13 +330,17 @@ def _record_launch(_runtime: object, *, route: object, created_worktrees: tuple[ patch.object( engine.planning_worktree_orchestrator, "last_plan_selection_result", - return_value=PlanSelectionResult(raw_projects=[], selected_contexts=[context], created_worktrees=()), + return_value=PlanSelectionResult( + raw_projects=[], selected_contexts=[context], created_worktrees=() + ), ), patch("envctl_engine.startup.lifecycle.launch_plan_agent_terminals", side_effect=_record_launch), patch.object(engine, "_write_artifacts"), patch.object(engine, "_should_enter_post_start_interactive", return_value=False), ): - code = engine.dispatch(parse_route(["--plan", "feature-a", "--cmux", "--headless"], env={"ENVCTL_DEFAULT_MODE": "trees"})) + code = engine.dispatch( + parse_route(["--plan", "feature-a", "--cmux", "--headless"], env={"ENVCTL_DEFAULT_MODE": "trees"}) + ) self.assertEqual(code, 0) self.assertEqual(captured_created_worktrees, [["feature-a-1"]]) @@ -252,7 +379,9 @@ def test_resume_dashboard_exact_headless_plan_prints_attach_command(self) -> Non patch.object( engine.planning_worktree_orchestrator, "last_plan_selection_result", - return_value=PlanSelectionResult(raw_projects=[], selected_contexts=[context], created_worktrees=()), + return_value=PlanSelectionResult( + raw_projects=[], selected_contexts=[context], created_worktrees=() + ), ), patch( "envctl_engine.startup.lifecycle.launch_plan_agent_terminals", @@ -273,7 +402,10 @@ def test_resume_dashboard_exact_headless_plan_prints_attach_command(self) -> Non out = StringIO() with redirect_stdout(out): code = engine.dispatch( - parse_route(["--plan", "feature-a", "--tmux", "--opencode", "--headless"], env={"ENVCTL_DEFAULT_MODE": "trees"}) + parse_route( + ["--plan", "feature-a", "--tmux", "--opencode", "--headless"], + env={"ENVCTL_DEFAULT_MODE": "trees"}, + ) ) self.assertEqual(code, 0) @@ -317,6 +449,7 @@ def test_resume_reuse_failure_falls_back_to_fresh_run_id_before_failure_write(se patch.object(engine, "_discover_projects", return_value=[context]), patch.object(engine, "_try_load_existing_state", return_value=existing_state), patch.object(engine, "_resume", return_value=1), + patch.object(engine, "_terminate_services_from_state", return_value=set()) as terminate_mock, patch.object(engine, "_new_run_id", return_value="run-fresh-after-resume-failure"), patch.object(engine, "_start_project_context", side_effect=RuntimeError("fresh startup failed")), patch.object( @@ -331,10 +464,78 @@ def test_resume_reuse_failure_falls_back_to_fresh_run_id_before_failure_write(se code = engine.dispatch(parse_route(["start", "--batch"], env={"ENVCTL_DEFAULT_MODE": "main"})) self.assertEqual(code, 1) + self.assertEqual(terminate_mock.call_count, 1) written_state = cast(RunState, captured["state"]) self.assertEqual(written_state.run_id, "run-fresh-after-resume-failure") self.assertNotEqual(written_state.run_id, existing_state.run_id) + def test_resume_failure_after_interactive_release_is_terminal_instead_of_racing_fresh_start(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repo(root) + runtime = root / "runtime" + engine = self._engine(repo, runtime) + context = self._main_context(repo) + metadata = startup_support.build_startup_identity_metadata( + engine, + runtime_mode="main", + project_contexts=[context], + ) + existing_state = RunState( + run_id="run-existing", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=str(repo / "backend"), + pid=1111, + requested_port=8000, + actual_port=8000, + status="running", + ) + }, + metadata=metadata, + ) + competing_results: list[int] = [] + + def released_resume_failure(_route: object) -> int: + self.assertTrue(release_lifecycle_operation(engine)) + competitor = threading.Thread( + target=lambda: competing_results.append( + run_exclusive_lifecycle_operation( + engine, + command="stop", + operation=lambda: 0, + ) + ) + ) + competitor.start() + competitor.join(timeout=2) + self.assertFalse(competitor.is_alive()) + return 1 + + with ( + patch.object(engine, "_discover_projects", return_value=[context]), + patch.object(engine, "_try_load_existing_state", return_value=existing_state), + patch.object(engine, "_resume", side_effect=released_resume_failure), + patch.object( + engine, + "_start_project_context", + side_effect=AssertionError("fresh fallback must not mutate after releasing its lease"), + ) as start_project, + patch.object( + engine, + "_write_artifacts", + side_effect=AssertionError("terminal resume failure must not write fresh state"), + ), + ): + code = engine.dispatch(parse_route(["start", "--batch"], env={"ENVCTL_DEFAULT_MODE": "main"})) + + self.assertEqual(code, 1) + self.assertEqual(competing_results, [0]) + start_project.assert_not_called() + def test_reuse_expand_failure_writes_failed_state_to_fresh_run_id(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -481,4 +682,5 @@ def _start_context(context: object, **_kwargs: object) -> ProjectStartupResult: orphan_mock.assert_not_called() written_state = cast(RunState, captured["state"]) self.assertEqual(set(written_state.services), {"feature-a-1 Backend", "feature-b-1 Backend"}) + self.assertEqual(written_state.metadata["state_source_run_ids"], ["run-existing-expand"]) self.assertEqual(captured["errors"], []) diff --git a/tests/python/startup/test_startup_orchestrator_profiles.py b/tests/python/startup/test_startup_orchestrator_profiles.py index a69b2e0e0..e3f3ff394 100644 --- a/tests/python/startup/test_startup_orchestrator_profiles.py +++ b/tests/python/startup/test_startup_orchestrator_profiles.py @@ -10,6 +10,7 @@ from envctl_engine.state.models import RunState, ServiceRecord from envctl_engine.runtime.engine_runtime_env import requirement_enabled_for_mode from envctl_engine.startup.finalization import build_success_run_state +from envctl_engine.startup.finalization_run_state import _build_run_state from envctl_engine.startup.session import StartupSession from envctl_engine.state.models import RequirementsResult from envctl_engine.startup.startup_selection_support import ( @@ -103,6 +104,84 @@ def test_build_run_state_writes_project_scoped_configured_services_for_multiple_ {"Alpha": ["frontend"], "Zeta": ["frontend"]}, ) + def test_build_run_state_includes_enabled_additional_services_per_project(self) -> None: + additional_services = ( + SimpleNamespace( + name="voice-runtime", + enabled_for_project_root=lambda _mode, _root: True, + ), + SimpleNamespace( + name="worker", + enabled_for_project_root=lambda _mode, root: Path(root).name == "Alpha", + ), + ) + runtime = SimpleNamespace( + config=SimpleNamespace(runtime_scope_id="scope-1", additional_services=additional_services), + _run_dir_path=lambda run_id: Path("/tmp/runtime") / run_id, + _service_enabled_for_mode=lambda _mode, service_name: service_name == "backend", + ) + contexts = [ + SimpleNamespace(name=name, root=Path(f"/tmp/repo/{name}"), ports={}) + for name in ("Alpha", "Beta") + ] + route = Route(command="up", mode="trees", flags={}) + session = StartupSession( + requested_route=route, + effective_route=route, + requested_command="up", + runtime_mode="trees", + run_id="run-1", + selected_contexts=contexts, + ) + + state = build_success_run_state(runtime, session) # type: ignore[arg-type] + + self.assertEqual( + state.metadata["dashboard_project_configured_services"], + { + "Alpha": ["backend", "voice-runtime", "worker"], + "Beta": ["backend", "voice-runtime"], + }, + ) + + def test_stopped_service_metadata_is_removed_only_after_project_success(self) -> None: + entries = [ + {"project": project, "type": "voice-runtime", "name": f"{project} Agent"} + for project in ("Main", "Other") + ] + runtime = SimpleNamespace( + config=SimpleNamespace(runtime_scope_id="scope-1", additional_services=()), + _run_dir_path=lambda run_id: Path("/tmp/runtime") / run_id, + _service_enabled_for_mode=lambda _mode, _service_name: True, + ) + + def session() -> StartupSession: + route = Route(command="start", mode="main", flags={"_restart_request": True}) + value = StartupSession( + requested_route=route, + effective_route=route, + requested_command="restart", + runtime_mode="main", + run_id="run-1", + selected_contexts=[SimpleNamespace(name="Main", root=Path("/repo"), ports={})], + base_metadata={"dashboard_stopped_services": entries}, + ) + replacement = ServiceRecord( + name="opaque-main-agent", + type="voice-runtime", + service_slug="voice-runtime", + project="Main", + cwd="/repo", + ) + value.services_by_project["Main"] = {replacement.name: replacement} + return value + + successful = build_success_run_state(runtime, session()) # type: ignore[arg-type] + failed = _build_run_state(runtime, session(), failed=True) # type: ignore[arg-type] + + self.assertEqual(successful.metadata["dashboard_stopped_services"], [entries[1]]) + self.assertEqual(failed.metadata["dashboard_stopped_services"], entries) + def test_build_run_state_marks_tree_shared_dependency_dashboard_scope(self) -> None: runtime = SimpleNamespace( config=SimpleNamespace(runtime_scope_id="scope-1"), @@ -318,6 +397,34 @@ def test_restart_service_types_accept_additional_service_slugs(self) -> None: self.assertEqual(selected, {"voice-runtime"}) + def test_restart_start_route_honors_normalized_prestop_selection(self) -> None: + route = Route( + command="start", + mode="main", + flags={ + "_restart_request": True, + "_restart_selected_services": ["Main Backend", "Main Frontend"], + }, + ) + + selected = _restart_service_types_for_project( + route=route, + project_name="Main", + default_service_types={"backend", "frontend", "optional-bad"}, + ) + + self.assertEqual(selected, {"backend", "frontend"}) + + route.flags["_restart_selected_services"] = [] + self.assertEqual( + _restart_service_types_for_project( + route=route, + project_name="Main", + default_service_types={"backend", "frontend", "optional-bad"}, + ), + set(), + ) + def test_runtime_scope_flags_select_startup_service_types(self) -> None: backend_route = parse_route(["--backend"], env={}) self.assertEqual( @@ -365,7 +472,7 @@ def test_startup_only_flags_select_individual_worktree_launch_parts(self) -> Non _restart_service_types_for_project( route=route, project_name="feature-a-1", - default_service_types={"backend", "frontend"}, + default_service_types={"backend", "frontend", "voice-runtime"}, ), {"frontend"}, ) @@ -376,7 +483,7 @@ def test_startup_only_flags_select_individual_worktree_launch_parts(self) -> Non _restart_service_types_for_project( route=route, project_name="feature-a-1", - default_service_types={"backend", "frontend"}, + default_service_types={"backend", "frontend", "voice-runtime"}, ), {"backend"}, ) @@ -387,7 +494,7 @@ def test_startup_only_flags_select_individual_worktree_launch_parts(self) -> Non _restart_service_types_for_project( route=route, project_name="feature-a-1", - default_service_types={"backend", "frontend"}, + default_service_types={"backend", "frontend", "voice-runtime"}, ), set(), ) diff --git a/tests/python/startup/test_startup_restart_port_preservation.py b/tests/python/startup/test_startup_restart_port_preservation.py index 889ae3b72..fc4e5d972 100644 --- a/tests/python/startup/test_startup_restart_port_preservation.py +++ b/tests/python/startup/test_startup_restart_port_preservation.py @@ -98,7 +98,9 @@ def is_pid_running(*_args, **_kwargs): # noqa: ANN001 }, ) engine._terminate_services_from_state = ( # type: ignore[method-assign] - lambda state, selected_services, aggressive, verify_ownership: terminated.append(state.run_id) + lambda state, selected_services, aggressive, verify_ownership: ( + terminated.append(state.run_id) or set() + ) ) spinner_calls: list[tuple[str, bool]] = [] @@ -217,7 +219,7 @@ def test_interactive_main_restart_reuses_previous_backend_and_frontend_ports_aft captured_env: dict[str, str] = {} engine._try_load_existing_state = lambda mode=None, strict_mode_match=False: previous_state # type: ignore[method-assign] - engine._terminate_services_from_state = lambda *_args, **_kwargs: None # type: ignore[method-assign] + engine._terminate_services_from_state = lambda *_args, **_kwargs: set() # type: ignore[method-assign] engine._listener_pids_for_port = lambda _port: [] # type: ignore[method-assign] engine._start_project_context = ( # type: ignore[method-assign] lambda context, mode, route, run_id: ( @@ -358,7 +360,7 @@ def test_interactive_main_restart_rebounds_only_when_previous_port_still_unavail captured_env: dict[str, str] = {} engine._try_load_existing_state = lambda mode=None, strict_mode_match=False: previous_state # type: ignore[method-assign] - engine._terminate_services_from_state = lambda *_args, **_kwargs: None # type: ignore[method-assign] + engine._terminate_services_from_state = lambda *_args, **_kwargs: set() # type: ignore[method-assign] engine._listener_pids_for_port = lambda _port: [] # type: ignore[method-assign] engine._start_project_context = ( # type: ignore[method-assign] lambda context, mode, route, run_id: ( @@ -493,7 +495,7 @@ def test_interactive_main_restart_reuses_selected_additional_service_port_after_ captured_state: dict[str, RunState] = {} engine._try_load_existing_state = lambda mode=None, strict_mode_match=False: previous_state # type: ignore[method-assign] - engine._terminate_services_from_state = lambda *_args, **_kwargs: None # type: ignore[method-assign] + engine._terminate_services_from_state = lambda *_args, **_kwargs: set() # type: ignore[method-assign] engine._listener_pids_for_port = lambda _port: [] # type: ignore[method-assign] engine._start_project_context = ( # type: ignore[method-assign] lambda context, mode, route, run_id: ( @@ -592,7 +594,7 @@ def test_interactive_main_restart_rebounds_selected_additional_service_when_prio captured_state: dict[str, RunState] = {} engine._try_load_existing_state = lambda mode=None, strict_mode_match=False: previous_state # type: ignore[method-assign] - engine._terminate_services_from_state = lambda *_args, **_kwargs: None # type: ignore[method-assign] + engine._terminate_services_from_state = lambda *_args, **_kwargs: set() # type: ignore[method-assign] engine._listener_pids_for_port = lambda _port: [] # type: ignore[method-assign] engine._start_project_context = ( # type: ignore[method-assign] lambda context, mode, route, run_id: ( diff --git a/tests/python/startup/test_startup_restart_service_scope.py b/tests/python/startup/test_startup_restart_service_scope.py index c221556bf..c30e65b88 100644 --- a/tests/python/startup/test_startup_restart_service_scope.py +++ b/tests/python/startup/test_startup_restart_service_scope.py @@ -87,7 +87,9 @@ def test_restart_service_only_preserves_other_services_and_requirements(self) -> engine._discover_projects = lambda mode: [context] # type: ignore[method-assign] engine._reserve_project_ports = lambda _context, route=None: None # type: ignore[method-assign] engine._terminate_services_from_state = ( # type: ignore[method-assign] - lambda state, selected_services, aggressive, verify_ownership: terminate_calls.append(selected_services) + lambda state, selected_services, aggressive, verify_ownership: ( + terminate_calls.append(selected_services) or set() + ) ) engine._release_requirement_ports = ( # type: ignore[method-assign] lambda _requirements: release_calls.append("released") @@ -217,7 +219,9 @@ def test_restart_stopped_service_starts_it_without_terminating_running_sibling(s engine._discover_projects = lambda mode: [context] # type: ignore[method-assign] engine._reserve_project_ports = lambda _context, route=None: None # type: ignore[method-assign] engine._terminate_services_from_state = ( # type: ignore[method-assign] - lambda state, selected_services, aggressive, verify_ownership: terminate_calls.append(selected_services) + lambda state, selected_services, aggressive, verify_ownership: ( + terminate_calls.append(selected_services) or set() + ) ) engine._release_requirement_ports = ( # type: ignore[method-assign] lambda _requirements: self.fail("requirements should be preserved for stopped service restart") @@ -354,7 +358,9 @@ def test_restart_service_only_preserves_shared_tree_dependency_state(self) -> No engine._discover_projects = lambda mode: [context] # type: ignore[method-assign] engine._reserve_project_ports = lambda _context, route=None: None # type: ignore[method-assign] engine._terminate_services_from_state = ( # type: ignore[method-assign] - lambda state, selected_services, aggressive, verify_ownership: terminate_calls.append(selected_services) + lambda state, selected_services, aggressive, verify_ownership: ( + terminate_calls.append(selected_services) or set() + ) ) engine._release_requirement_ports = ( # type: ignore[method-assign] lambda _requirements: self.fail("shared requirements should be preserved for app-only restart") diff --git a/tests/python/startup/test_startup_run_reuse_resolution.py b/tests/python/startup/test_startup_run_reuse_resolution.py index 8bdfee133..68d2e8e76 100644 --- a/tests/python/startup/test_startup_run_reuse_resolution.py +++ b/tests/python/startup/test_startup_run_reuse_resolution.py @@ -12,6 +12,7 @@ ) from envctl_engine.startup.run_reuse_support import RunReuseDecision from envctl_engine.startup.session import StartupSession +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord def _session(route: Route) -> StartupSession: @@ -58,6 +59,67 @@ def test_run_reuse_resume_uses_named_resume_handler_owner(self) -> None: self.assertTrue(callable(StartupRunReuseResumeHandler.resume_matching_run)) self.assertTrue(callable(StartupRunReuseResumeHandler.resume_dashboard_run)) + def test_auto_resume_exception_preserves_existing_run_authority(self) -> None: + route = Route(command="start", mode="main") + session = _session(route) + candidate = RunState( + run_id="old-run", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main", + project="Main", + pid=101, + ) + }, + requirements={"Main": RequirementsResult(project="Main")}, + metadata={"authority": "old"}, + ) + runtime = _RuntimeStub() + + def fail_resume(_route: Route) -> int: + raise OSError("resume lock unlink failed") + + runtime._resume = fail_resume # type: ignore[attr-defined] + handler = StartupRunReuseResumeHandler( + runtime=runtime, + session=session, + route=route, + runtime_mode="main", + decision=RunReuseDecision( + candidate_state=candidate, + decision_kind="resume_exact", + reason="exact_match", + selected_projects=[{"name": "Main", "root": None}], + state_projects=[{"name": "Main", "root": None}], + ), + candidate_state=candidate, + reuse_started=0.0, + announce_session_identifiers=lambda current: setattr(current, "identifiers_announced", True), + emit_phase=lambda *_args, **_kwargs: None, + headless_plan_output_only=lambda _session: False, + maybe_attach_plan_agent_terminal=lambda _session: None, + print_headless_plan_session_summary=lambda _session: None, + print_plan_dry_run_preview=lambda _session: None, + configured_service_types_for_mode=lambda _mode: [], + replace_existing_project_services_for_fresh_start=lambda *_args, **_kwargs: self.fail( + "exception must not enter fresh fallback" + ), + ) + + with self.assertRaisesRegex(OSError, "resume lock unlink failed"): + handler.resume_matching_run() + + self.assertIsNone(session.run_id) + self.assertFalse(session.identifiers_announced) + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, candidate.services) + self.assertEqual(session.preserved_requirements, candidate.requirements) + self.assertEqual(session.base_metadata["authority"], "old") + self.assertEqual(session.base_metadata["state_source_run_ids"], ["old-run"]) + def test_planning_prs_branch_runs_pr_action_after_reuse_evaluation_and_skips_startup(self) -> None: route = Route(command="plan", mode="trees", raw_args=[], passthrough_args=[], projects=[], flags={"planning_prs": True}) runtime = _RuntimeStub() @@ -142,7 +204,7 @@ def test_runtime_bound_run_reuse_helper_handles_planning_prs(self) -> None: result = resolve_startup_run_reuse_with_runtime( runtime, session, - terminate_restart_orphan_listeners=lambda **kwargs: None, + terminate_restart_orphan_listeners=lambda **kwargs: set(), validate_attach_target_fn=lambda *args, **kwargs: None, attach_plan_agent_terminal=lambda *args, **kwargs: None, progress_lock=None, diff --git a/tests/python/startup/test_startup_selection_support.py b/tests/python/startup/test_startup_selection_support.py index 2c2a40cce..4b675c867 100644 --- a/tests/python/startup/test_startup_selection_support.py +++ b/tests/python/startup/test_startup_selection_support.py @@ -7,8 +7,12 @@ from envctl_engine.runtime.command_router import Route, parse_route from envctl_engine.startup.startup_selection_support import ( + _restart_service_types_for_project, restart_include_requirements, + restart_target_projects, + restart_target_projects_for_selected_services, select_start_tree_projects, + state_project_names, trees_start_selection_required, ) from envctl_engine.state.models import RunState, ServiceRecord @@ -68,7 +72,9 @@ def test_trees_start_selection_required_for_default_or_explicit_trees_start(self self.assertTrue(trees_start_selection_required(route=self._trees_route(raw_args=[]), runtime_mode="trees")) self.assertFalse( trees_start_selection_required( - route=Route(command="plan", mode="trees", raw_args=["--plan"], passthrough_args=[], projects=[], flags={}), + route=Route( + command="plan", mode="trees", raw_args=["--plan"], passthrough_args=[], projects=[], flags={} + ), runtime_mode="trees", ) ) @@ -86,6 +92,103 @@ def test_trees_start_selection_required_for_default_or_explicit_trees_start(self ) ) + def test_state_project_discovery_prefers_explicit_multiword_service_metadata(self) -> None: + runtime = _RuntimeStub() + service_name = "My Project Optional Bad" + state = RunState( + run_id="run-1", + mode="main", + services={ + service_name: ServiceRecord( + name=service_name, + type="optional-bad", + cwd="/repo/optional-bad", + project="My Project", + service_slug="optional-bad", + ) + }, + ) + + self.assertEqual(state_project_names(runtime=runtime, state=state), {"My Project"}) + self.assertEqual( + restart_target_projects( + state=state, + route=Route(command="restart", mode="main", flags={}), + runtime=runtime, + ), + {"My Project"}, + ) + self.assertEqual( + restart_target_projects_for_selected_services( + selected_services={service_name}, + state=state, + runtime=runtime, + ), + {"My Project"}, + ) + + def test_state_project_discovery_uses_requirement_project_not_collision_storage_key(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-1", + mode="trees", + requirements={ + "Main Restart Collision": SimpleNamespace(project="Main") # type: ignore[dict-item] + }, + ) + + self.assertEqual(state_project_names(runtime=runtime, state=state), {"Main"}) + self.assertEqual( + restart_target_projects( + state=state, + route=Route(command="restart", mode="trees", flags={"runtime_scope": "dependencies"}), + runtime=runtime, + ), + {"Main"}, + ) + def test_restart_target_projects_resolves_stopped_additional_service_metadata(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-1", + mode="trees", + services={ + "Other Backend": ServiceRecord( + name="Other Backend", + type="backend", + cwd="/repo/other/backend", + project="Other", + ) + }, + metadata={ + "dashboard_stopped_services": [ + { + "name": "Customer Platform Voice Runtime", + "project": "Customer Platform", + "type": "voice-runtime", + } + ] + }, + ) + + for selector in ( + "voice-runtime", + "Voice Runtime", + "Customer Platform Voice Runtime", + "service:voice-runtime", + ): + with self.subTest(selector=selector): + targets = restart_target_projects( + state=state, + route=Route( + command="restart", + mode="trees", + flags={"services": [selector]}, + ), + runtime=runtime, + ) + + self.assertEqual(targets, {"Customer Platform"}) + def test_select_start_tree_projects_preselects_from_previous_trees_state(self) -> None: runtime = _RuntimeStub(can_tty=True) runtime.state = RunState( @@ -152,6 +255,22 @@ def test_runtime_scope_flags_control_restart_requirements(self) -> None: self.assertTrue(restart_include_requirements(parse_route(["restart"], env={}))) self.assertFalse(restart_include_requirements(parse_route(["restart", "--service", "Main Frontend"], env={}))) + def test_no_infra_selects_no_additional_application_services(self) -> None: + route = Route( + command="start", + mode="main", + flags={"launch_backend": False, "launch_frontend": False, "launch_dependencies": False}, + ) + + self.assertEqual( + _restart_service_types_for_project( + route=route, + project_name="Main", + default_service_types={"backend", "frontend", "worker"}, + ), + set(), + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/startup/test_support_module_decoupling.py b/tests/python/startup/test_support_module_decoupling.py index f58d1e238..989ff2f30 100644 --- a/tests/python/startup/test_support_module_decoupling.py +++ b/tests/python/startup/test_support_module_decoupling.py @@ -14,9 +14,11 @@ REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" from envctl_engine.startup.resume_restore_support import restore_missing # noqa: E402 +from envctl_engine.startup.resume_restore_project import ResumeProjectRestoreRunner # noqa: E402 import envctl_engine.startup.requirements_project_startup as requirements_project_startup_module # noqa: E402 from envctl_engine.startup.protocols import StartupOrchestratorLike # noqa: E402 from envctl_engine.startup.requirements_execution import ( # noqa: E402 + requirements_for_restart_context, requirements_parallel_enabled, start_requirements_for_project as requirements_start_impl, ) @@ -26,6 +28,7 @@ _maybe_prewarm_docker, _requirements_failure_message, _shared_main_requirements, + ProjectStartupFailure, start_requirements_for_project, start_project_services, start_project_context, @@ -33,7 +36,8 @@ from envctl_engine.startup.startup_orchestrator import StartupOrchestrator # noqa: E402 from envctl_engine.startup.startup_selection_support import _tree_preselected_projects_from_state # noqa: E402 from envctl_engine.config import AppServiceConfig # noqa: E402 -from envctl_engine.runtime.command_router import parse_route # noqa: E402 +from envctl_engine.runtime.command_router import Route, parse_route # noqa: E402 +from envctl_engine.runtime.service_manager import ServiceCleanupError # noqa: E402 from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord # noqa: E402 @@ -67,6 +71,183 @@ def release(self, port: int) -> None: class StartupSupportModuleDecouplingTests(unittest.TestCase): + def test_resume_stale_service_releases_recorded_foreign_session_lock(self) -> None: + released: list[tuple[int, str, str]] = [] + + class RecordingPortAllocator: + session_id = "current-session" + + def release_owned(self, port: int, owner: str, *, expected_session: str) -> bool: + released.append((port, owner, expected_session)) + return True + + def release(self, _port: int, *, owner: str | None = None) -> None: + raise AssertionError(f"current-session release must not be used: {owner}") + + allocator = RecordingPortAllocator() + service = ServiceRecord( + name="Main Frontend", + type="frontend", + cwd="/tmp/repo/frontend", + actual_port=9000, + port_lock_session="prior-session", + project="Main", + ) + runtime = SimpleNamespace( + port_planner=allocator, + env={}, + config=SimpleNamespace(raw={}), + _terminate_service_record=lambda *_args, **_kwargs: True, + _project_name_from_service=lambda _name: "Main", + ) + runner = ResumeProjectRestoreRunner( + orchestrator=SimpleNamespace(), + rt=runtime, + state=RunState(run_id="run-1", mode="main"), + missing_services=[], + route_for_startup=Route(command="resume", mode="main"), + total_projects=1, + use_project_spinner_group=False, + project_spinner_group=SimpleNamespace(), + use_single_spinner=False, + active_spinner=SimpleNamespace(), + port_allocator=allocator, + ) + + terminated_count = runner._stop_stale_services( + service_names=[service.name], + original_services={service.name: service}, + ) + + self.assertEqual(terminated_count, 1) + self.assertEqual(released, [(9000, "Main:frontend", "prior-session")]) + + def test_post_start_truth_failure_propagates_unterminated_service_records(self) -> None: + requirements = RequirementsResult(project="Main", health="healthy") + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=62001, + actual_port=8000, + ) + runtime = SimpleNamespace( + config=SimpleNamespace( + supabase_auth_users=(), + supabase_auth_user_errors=(), + supabase_auth_users_strict=True, + ), + runtime_root=Path("/tmp/runtime"), + _reserve_project_ports=lambda _context, route=None: None, + _requirements_ready=lambda _value: True, + _consume_project_startup_warnings=lambda _project: [], + _emit=lambda *_args, **_kwargs: None, + _assert_project_services_post_start_truth=lambda **_kwargs: (_ for _ in ()).throw( + RuntimeError("backend truth failed") + ), + _terminate_started_services=lambda _services: None, + ) + orchestrator = SimpleNamespace( + runtime=runtime, + _report_progress=lambda *_args, **_kwargs: None, + start_requirements_for_project=lambda *_args, **_kwargs: requirements, + start_project_services=lambda *_args, **_kwargs: {"Main Backend": service}, + ) + + with self.assertRaises(ProjectStartupFailure) as raised: + start_project_context( + orchestrator, + context=SimpleNamespace(name="Main", root=Path("/repo"), ports={}), + mode="main", + route=parse_route(["start"], env={}), + run_id="run-1", + ) + + self.assertIs(raised.exception.unterminated_services["Main Backend"], service) + self.assertIs(raised.exception.requirements, requirements) + + def test_service_attach_failure_preserves_already_started_requirements(self) -> None: + requirements = RequirementsResult( + project="Main", + health="healthy", + redis={"enabled": True, "success": True, "final": 6379}, + ) + runtime = SimpleNamespace( + config=SimpleNamespace( + supabase_auth_users=(), + supabase_auth_user_errors=(), + supabase_auth_users_strict=True, + ), + runtime_root=Path("/tmp/runtime"), + _reserve_project_ports=lambda _context, route=None: None, + _requirements_ready=lambda _value: True, + _emit=lambda *_args, **_kwargs: None, + ) + orchestrator = SimpleNamespace( + runtime=runtime, + _report_progress=lambda *_args, **_kwargs: None, + start_requirements_for_project=lambda *_args, **_kwargs: requirements, + start_project_services=lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("backend launch failed") + ), + ) + + with self.assertRaises(ProjectStartupFailure) as raised: + start_project_context( + orchestrator, + context=SimpleNamespace(name="Main", root=Path("/repo"), ports={}), + mode="main", + route=parse_route(["start"], env={}), + run_id="run-1", + ) + + self.assertEqual(raised.exception.project, "Main") + self.assertIs(raised.exception.requirements, requirements) + + def test_cleanup_exception_still_transports_all_started_resources(self) -> None: + requirements = RequirementsResult(project="Main", health="healthy") + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=62002, + actual_port=8000, + ) + runtime = SimpleNamespace( + config=SimpleNamespace( + supabase_auth_users=(), + supabase_auth_user_errors=(), + supabase_auth_users_strict=True, + ), + runtime_root=Path("/tmp/runtime"), + _reserve_project_ports=lambda _context, route=None: None, + _requirements_ready=lambda _value: True, + _consume_project_startup_warnings=lambda _project: [], + _emit=lambda *_args, **_kwargs: None, + _assert_project_services_post_start_truth=lambda **_kwargs: (_ for _ in ()).throw( + RuntimeError("truth failed") + ), + _terminate_started_services=lambda _services: (_ for _ in ()).throw(OSError("terminator crashed")), + ) + orchestrator = SimpleNamespace( + runtime=runtime, + _report_progress=lambda *_args, **_kwargs: None, + start_requirements_for_project=lambda *_args, **_kwargs: requirements, + start_project_services=lambda *_args, **_kwargs: {service.name: service}, + ) + + with self.assertRaisesRegex(ProjectStartupFailure, "cleanup error: terminator crashed") as raised: + start_project_context( + orchestrator, + context=SimpleNamespace(name="Main", root=Path("/repo"), ports={}), + mode="main", + route=parse_route(["start"], env={}), + run_id="run-1", + ) + + self.assertIs(raised.exception.requirements, requirements) + self.assertIs(raised.exception.unterminated_services[service.name], service) + def test_startup_execution_support_reexports_new_owner_modules(self) -> None: self.assertIs(start_requirements_for_project, requirements_start_impl) self.assertIs(start_project_services, service_start_impl) @@ -304,6 +485,104 @@ def reconcile(project, req, *, project_root=None): # noqa: ANN001, ANN202 self.assertEqual(reused.component("supabase")["resources"]["db"], 5574) self.assertEqual(reused.component("supabase")["resources"]["api"], 54463) + def test_restart_reuses_authoritative_requirements_under_collision_storage_key(self) -> None: + requirements = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6381, "runtime_status": "healthy"}, + ) + events: list[str] = [] + runtime = SimpleNamespace( + _try_load_existing_state=lambda **_kwargs: RunState( + run_id="run-alias", + mode="main", + requirements={"Main Restart Collision": requirements}, + ), + _emit=lambda event, **_payload: events.append(event), + ) + orchestrator = SimpleNamespace( + runtime=runtime, + start_requirements_for_project=lambda *_args, **_kwargs: self.fail( + "authoritative requirements must be reused" + ), + ) + context = SimpleNamespace(name="Main") + route = Route( + command="restart", + mode="main", + projects=["Main"], + flags={"_restart_request": True, "restart_include_requirements": False}, + ) + + reused = requirements_for_restart_context( + cast(StartupOrchestratorLike, orchestrator), + context=context, + mode="main", + route=route, + ) + + self.assertIs(reused, requirements) + self.assertEqual(events, ["requirements.restart.reuse"]) + + def test_shared_main_reuses_authoritative_requirements_under_collision_storage_key(self) -> None: + requirements = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6381, "runtime_status": "healthy"}, + ) + events: list[str] = [] + runtime = SimpleNamespace( + config=SimpleNamespace(execution_root=Path("/tmp/repo")), + _try_load_existing_state=lambda **_kwargs: RunState( + run_id="run-alias", + mode="main", + requirements={"Main Restart Collision": requirements}, + ), + _requirements_ready=lambda candidate: candidate is requirements, + _emit=lambda event, **_payload: events.append(event), + ) + orchestrator = cast(StartupOrchestratorLike, SimpleNamespace(runtime=runtime)) + + with mock.patch( + "envctl_engine.startup.startup_execution_support.requirements_for_restart_context", + side_effect=AssertionError("authoritative shared requirements must be reused"), + ): + reused = _shared_main_requirements(orchestrator, route=parse_route(["start"], env={})) + + self.assertEqual(reused.component("redis")["final"], 6381) + self.assertIn("requirements.shared.reuse", events) + + def test_ambiguous_requirement_aliases_fail_closed_instead_of_starting_duplicate_stack(self) -> None: + state = RunState( + run_id="run-ambiguous", + mode="main", + requirements={ + "Main Restart Collision": RequirementsResult(project="Main"), + "Main Restart Collision 2": RequirementsResult(project="Main"), + }, + ) + runtime = SimpleNamespace( + _try_load_existing_state=lambda **_kwargs: state, + _emit=lambda *_args, **_kwargs: None, + ) + orchestrator = SimpleNamespace( + runtime=runtime, + start_requirements_for_project=lambda *_args, **_kwargs: self.fail( + "ambiguous requirements must not start a duplicate stack" + ), + ) + + with self.assertRaisesRegex(RuntimeError, "Ambiguous requirements authority for Main"): + requirements_for_restart_context( + cast(StartupOrchestratorLike, orchestrator), + context=SimpleNamespace(name="Main"), + mode="main", + route=Route( + command="restart", + mode="main", + projects=["Main"], + flags={"_restart_request": True, "restart_include_requirements": False}, + ), + ) + def test_shared_main_requirements_starts_fresh_when_existing_state_reconciles_unreachable(self) -> None: stale = RequirementsResult( project="Main", @@ -325,7 +604,9 @@ def reconcile(project, req, *, project_root=None): # noqa: ANN001, ANN202 self.assertEqual(project_root, Path("/tmp/repo")) for dependency_id in ("redis", "supabase", "n8n"): component = req.component(dependency_id) - component["runtime_status"] = "healthy" if component.get("final") in {6557, 5610, 5856} else "unreachable" + component["runtime_status"] = ( + "healthy" if component.get("final") in {6557, 5610, 5856} else "unreachable" + ) return [ {"project": project, "component": dependency_id, "status": "unreachable"} for dependency_id in ("redis", "supabase", "n8n") @@ -352,7 +633,7 @@ def reconcile(project, req, *, project_root=None): # noqa: ANN001, ANN202 _reserve_project_ports=lambda context: reserved.append(context.name), _emit=lambda event, **payload: events.append((event, payload)), ) - ) + ), ) with mock.patch( @@ -366,7 +647,9 @@ def reconcile(project, req, *, project_root=None): # noqa: ANN001, ANN202 self.assertEqual(reused.component("n8n")["final"], 5856) self.assertEqual(reserved, ["Main"]) self.assertEqual(start_requirements.call_count, 1) - self.assertIn(("requirements.shared.reuse_stale", {"project": "Main", "source_run_id": "run-stale-main"}), events) + self.assertIn( + ("requirements.shared.reuse_stale", {"project": "Main", "source_run_id": "run-stale-main"}), events + ) def test_start_project_context_rejects_invalid_supabase_auth_user_config_before_services(self) -> None: order: list[str] = [] @@ -403,7 +686,7 @@ def test_start_project_context_rejects_invalid_supabase_auth_user_config_before_ start_requirements_for_project=lambda context, mode, route: requirements, ) - with self.assertRaisesRegex(RuntimeError, "Invalid Supabase Auth user config"): + with self.assertRaisesRegex(ProjectStartupFailure, "Invalid Supabase Auth user config") as raised: start_project_context( orchestrator, context=SimpleNamespace(name="Main", root=Path("/tmp/repo"), ports={}), @@ -413,6 +696,7 @@ def test_start_project_context_rejects_invalid_supabase_auth_user_config_before_ ) self.assertNotIn("services", order) + self.assertIs(raised.exception.requirements, requirements) def test_start_project_context_syncs_supabase_auth_users_before_services(self) -> None: order: list[str] = [] @@ -444,8 +728,10 @@ def project_env(**kwargs): # noqa: ANN003, ANN201 _record_project_startup_warning=lambda project, warning: None, _consume_project_startup_warnings=lambda project: [], _emit=lambda event, **payload: ( - order.append(event), events.append((event, payload)) - ) if event.startswith("supabase.auth_users") else None, + (order.append(event), events.append((event, payload))) + if event.startswith("supabase.auth_users") + else None + ), _assert_project_services_post_start_truth=lambda **kwargs: None, _terminate_started_services=lambda services: None, ) @@ -612,6 +898,50 @@ def test_restore_missing_uses_runtime_port_allocator_without_resume_wrappers(sel self.assertEqual(events[0][0], "resume.restore.execution") self.assertEqual(events[-1][0], "resume.restore.timing") + def test_resume_project_restore_converts_requirement_release_oserror_to_retryable_result(self) -> None: + requirements = RequirementsResult(project="Main", health="degraded") + state = RunState( + run_id="run-release-error", + mode="main", + requirements={"Main Restart Collision": requirements}, + ) + context = SimpleNamespace(name="Main", root=Path("/tmp/repo"), ports={}) + events: list[str] = [] + + def fail_release(_requirements): # noqa: ANN001, ANN202 + raise OSError("requirement lock unlink failed") + + runtime = SimpleNamespace( + env={}, + config=SimpleNamespace(raw={}, additional_services=()), + _emit=lambda event, **_payload: events.append(event), + _resume_context_for_project=lambda _state, _project: context, + _project_name_from_service=lambda _name: "Main", + _requirements_ready=lambda _requirements: False, + _release_requirement_ports=fail_release, + ) + runner = ResumeProjectRestoreRunner( + orchestrator=SimpleNamespace(runtime=runtime), + rt=runtime, + state=state, + missing_services=[], + route_for_startup=Route(command="resume", mode="main"), + total_projects=1, + use_project_spinner_group=False, + project_spinner_group=SimpleNamespace(), + use_single_spinner=False, + active_spinner=SimpleNamespace(), + port_allocator=_PortAllocatorStub(), + ) + + result = runner.restore_project(1, "Main") + + self.assertEqual(result["error"], "requirement lock unlink failed") + self.assertEqual(result["remove_service_names"], []) + self.assertIsNone(result["requirements"]) + self.assertIn("resume.restore.project_timing", events) + self.assertIs(state.requirements["Main Restart Collision"], requirements) + def test_restore_missing_restarts_only_stale_services_when_requirements_are_healthy(self) -> None: released_ports: list[int] = [] terminated_services: list[str] = [] @@ -642,6 +972,7 @@ def release(self, port: int) -> None: name="Main Backend", type="backend", cwd="/tmp/repo/backend", + project="Main", pid=1111, requested_port=8000, actual_port=8000, @@ -651,6 +982,7 @@ def release(self, port: int) -> None: name="Main Frontend", type="frontend", cwd="/tmp/repo/frontend", + project="Main", pid=2222, requested_port=9000, actual_port=9000, @@ -665,6 +997,7 @@ def start_project_services(_context, *, requirements, run_id, route): # noqa: A name="Main Frontend", type="frontend", cwd="/tmp/repo/frontend", + project="Main", pid=3333, requested_port=9000, actual_port=9000, @@ -676,7 +1009,7 @@ def start_project_services(_context, *, requirements, run_id, route): # noqa: A port_planner=RecordingPortAllocator(), env={}, config=SimpleNamespace(raw={}, base_dir=Path("/tmp/repo")), - _project_name_from_service=lambda name: "Main", + _project_name_from_service=lambda _name: "Incorrect Legacy Parse", _requirements_ready=lambda value: value is requirements and value.health == "healthy", _reconcile_project_requirement_truth=lambda project, req, project_root=None: [], _emit=lambda event, **payload: None, @@ -685,7 +1018,9 @@ def start_project_services(_context, *, requirements, run_id, route): # noqa: A _terminate_service_record=lambda service, **kwargs: terminated_services.append(service.name) or True, _service_port=lambda service: service.actual_port, _release_requirement_ports=lambda req: self.fail("healthy reused requirements should not be released"), - _reserve_project_ports=lambda ctx: self.fail("partial service restore should reserve only selected services"), + _reserve_project_ports=lambda ctx: self.fail( + "partial service restore should reserve only selected services" + ), _start_requirements_for_project=lambda *args, **kwargs: self.fail("healthy requirements should be reused"), _start_project_services=start_project_services, ) @@ -693,7 +1028,7 @@ def start_project_services(_context, *, requirements, run_id, route): # noqa: A run_id="run-1", mode="main", services={"Main Backend": backend, "Main Frontend": frontend}, - requirements={"Main": requirements}, + requirements={"main": requirements}, ) errors = restore_missing( @@ -713,10 +1048,178 @@ def start_project_services(_context, *, requirements, run_id, route): # noqa: A self.assertEqual(released_ports, [9000]) self.assertIs(state.services["Main Backend"], backend) self.assertEqual(state.services["Main Frontend"].pid, 3333) + self.assertEqual(state.requirements, {"main": requirements}) self.assertEqual(len(startup_routes), 1) self.assertTrue(startup_routes[0].flags.get("_restart_request")) self.assertEqual(startup_routes[0].flags.get("restart_service_types"), ["frontend"]) + def test_restore_missing_fails_closed_when_stale_service_exit_is_unconfirmed(self) -> None: + release_calls: list[str] = [] + requirements_start_calls: list[str] = [] + service_start_calls: list[str] = [] + context = SimpleNamespace( + name="Main", + root=Path("/tmp/repo"), + ports={ + "backend": SimpleNamespace(assigned=8000, final=8000), + "frontend": SimpleNamespace(assigned=9000, final=9000), + }, + ) + requirements = RequirementsResult( + project="main", + health="degraded", + failures=["redis unavailable"], + ) + stale = ServiceRecord( + name="Main Frontend", + type="frontend", + cwd="/tmp/repo/frontend", + project="Main", + pid=2222, + requested_port=9000, + actual_port=9000, + status="stale", + ) + runtime = SimpleNamespace( + port_planner=_PortAllocatorStub(), + env={}, + config=SimpleNamespace(raw={}, base_dir=Path("/tmp/repo")), + _project_name_from_service=lambda _name: "Main", + _requirements_ready=lambda _value: False, + _emit=lambda _event, **_payload: None, + _tree_parallel_startup_config=lambda **_kwargs: (False, 1), + _resume_context_for_project=lambda _state, _project: context, + _terminate_service_record=lambda _service, **_kwargs: False, + _release_requirement_ports=lambda _requirements: release_calls.append("released"), + _reserve_project_ports=lambda _context: self.fail("ports must not be reserved after failed cleanup"), + _start_requirements_for_project=lambda *_args, **_kwargs: ( + requirements_start_calls.append("started") or RequirementsResult(project="Main") + ), + _start_project_services=lambda *_args, **_kwargs: service_start_calls.append("started") or {}, + ) + state = RunState( + run_id="run-unconfirmed", + mode="main", + services={stale.name: stale}, + requirements={"main": requirements}, + ) + + errors = restore_missing( + SimpleNamespace(runtime=runtime), + state, + [stale.name], + route=parse_route(["resume", "--main"], env={}), + spinner_factory=lambda *args, **kwargs: _SpinnerStub(), + spinner_enabled_fn=lambda env: False, + use_spinner_policy_fn=lambda policy: nullcontext(), + emit_spinner_policy_fn=lambda emit, policy, context: None, + resolve_spinner_policy_fn=lambda env: SimpleNamespace(enabled=False, backend="rich", style="dots"), + ) + + self.assertEqual( + errors, + ["Main: stale service termination could not be confirmed: Main Frontend"], + ) + self.assertEqual(release_calls, []) + self.assertEqual(requirements_start_calls, []) + self.assertEqual(service_start_calls, []) + self.assertIs(state.services[stale.name], stale) + self.assertEqual(state.requirements, {"main": requirements}) + self.assertEqual(requirements.health, "degraded") + + def test_restore_missing_retains_unterminated_replacement_after_confirmed_stale_stop(self) -> None: + context = SimpleNamespace( + name="Main", + root=Path("/tmp/repo"), + ports={ + "backend": SimpleNamespace(assigned=8000, final=8000), + "frontend": SimpleNamespace(assigned=9000, final=9000), + }, + ) + original_requirements = RequirementsResult( + project="main", + health="degraded", + failures=["redis stale"], + ) + replacement_requirements = RequirementsResult(project="Main", health="healthy") + released_requirements: list[RequirementsResult] = [] + stale = ServiceRecord( + name="Main Frontend", + type="frontend", + cwd="/tmp/repo/frontend", + project="Main", + pid=2222, + requested_port=9000, + actual_port=9000, + status="stale", + ) + replacement = ServiceRecord( + name="Main Frontend", + type="frontend", + cwd="/tmp/repo/frontend", + project="Main", + pid=3333, + requested_port=9000, + actual_port=9000, + status="starting", + ) + + def fail_service_start(*_args, **_kwargs): # noqa: ANN002, ANN003 + raise ServiceCleanupError( + "frontend listener startup failed; cleanup could not confirm exit", + {replacement.name: replacement}, + ) + + runtime = SimpleNamespace( + port_planner=_PortAllocatorStub(), + env={}, + config=SimpleNamespace(raw={}, base_dir=Path("/tmp/repo")), + _project_name_from_service=lambda _name: "Main", + _requirements_ready=lambda value: value is replacement_requirements, + _emit=lambda _event, **_payload: None, + _tree_parallel_startup_config=lambda **_kwargs: (False, 1), + _resume_context_for_project=lambda _state, _project: context, + _terminate_service_record=lambda _service, **_kwargs: True, + _release_requirement_ports=lambda value: released_requirements.append(value), + _reserve_project_ports=lambda _context: self.fail( + "partial restore should reserve only selected service ports" + ), + _start_requirements_for_project=lambda *_args, **_kwargs: replacement_requirements, + _start_project_services=fail_service_start, + ) + state = RunState( + run_id="run-service-cleanup", + mode="main", + services={stale.name: stale}, + requirements={"main": original_requirements}, + ) + + errors = restore_missing( + SimpleNamespace(runtime=runtime), + state, + [stale.name], + route=parse_route(["resume", "--main"], env={}), + spinner_factory=lambda *args, **kwargs: _SpinnerStub(), + spinner_enabled_fn=lambda env: False, + use_spinner_policy_fn=lambda policy: nullcontext(), + emit_spinner_policy_fn=lambda emit, policy, context: None, + resolve_spinner_policy_fn=lambda env: SimpleNamespace(enabled=False, backend="rich", style="dots"), + ) + + self.assertEqual( + errors, + ["Main: frontend listener startup failed; cleanup could not confirm exit"], + ) + self.assertEqual(set(state.services), {"Main Frontend"}) + retained = state.services["Main Frontend"] + self.assertEqual(retained.pid, 3333) + self.assertEqual(retained.status, "termination_failed") + self.assertTrue(retained.degraded) + self.assertEqual(state.metadata["termination_failed_services"], ["Main Frontend"]) + self.assertEqual(released_requirements, [original_requirements]) + self.assertEqual(state.requirements, {"main": replacement_requirements}) + self.assertEqual(replacement_requirements.health, "degraded") + def test_requirements_failure_message_summarizes_docker_daemon_outage(self) -> None: requirements = RequirementsResult( project="Main", @@ -884,7 +1387,9 @@ def start_project_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: _project_service_env=lambda context, requirements, route=None: {}, _resolve_backend_env_file=lambda context, backend_cwd: (None, False), _resolve_frontend_env_file=lambda context, frontend_cwd: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service: True, process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: None), _prepare_backend_runtime=prepare_backend_runtime, @@ -963,7 +1468,9 @@ def test_start_project_services_skips_default_backend_frontend_when_no_local_sys _project_service_env=lambda context, requirements, route=None, service_name=None: {}, _resolve_backend_env_file=lambda context, backend_cwd: (None, False), _resolve_frontend_env_file=lambda context, frontend_cwd: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service: True, process_runner=SimpleNamespace( start_background=lambda *args, **kwargs: background_calls.append((args, kwargs)) @@ -1013,6 +1520,95 @@ def test_start_project_services_skips_default_backend_frontend_when_no_local_sys self.assertEqual(len(warnings), 1) self.assertIn("No local app system is configured", str(warnings[0])) + def test_start_project_services_does_not_skip_docker_only_plan_app_system(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + project_root = root / "repo" + project_root.mkdir(parents=True, exist_ok=True) + run_dir = root / "runtime" / "run-1" + events: list[tuple[str, dict[str, object]]] = [] + prep_calls: list[str] = [] + + runtime = SimpleNamespace( + port_planner=_PortAllocatorStub(), + env={}, + config=SimpleNamespace( + raw={"ENVCTL_BACKEND_DOCKER_IMAGE": "example/backend:dev"}, + explicit_keys=("ENVCTL_BACKEND_DOCKER_IMAGE",), + backend_dir_name="backend", + frontend_dir_name="frontend", + additional_services=(), + backend_dependency_env_section_present=False, + frontend_dependency_env_section_present=False, + main_backend_dependency_env_section_present=False, + main_frontend_dependency_env_section_present=False, + trees_backend_dependency_env_section_present=False, + trees_frontend_dependency_env_section_present=False, + service_dependency_env_section_present={}, + mode_service_dependency_env_section_present={}, + all_app_service_names_for_mode=lambda mode, project_root=None: ("backend", "frontend"), + ), + services=SimpleNamespace(), + _invoke_envctl_hook=lambda **kwargs: SimpleNamespace(found=False, success=False, payload=None), + _run_dir_path=lambda run_id: run_dir, + _project_service_env=lambda context, requirements, route=None, service_name=None: {}, + _resolve_backend_env_file=lambda context, backend_cwd: (None, False), + _resolve_frontend_env_file=lambda context, frontend_cwd: None, + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_enabled_for_mode=lambda mode, service: True, + process_runner=SimpleNamespace(), + _prepare_backend_runtime=lambda **kwargs: prep_calls.append("backend"), + _prepare_frontend_runtime=lambda **kwargs: prep_calls.append("frontend"), + _service_command_source=lambda **kwargs: self.fail( + f"host command source must be skipped: {kwargs}" + ), + _command_exists=lambda command: False, + _emit=lambda event, **payload: events.append((event, payload)), + ) + orchestrator = SimpleNamespace( + runtime=runtime, + _restart_service_types_for_project=lambda **kwargs: {"backend", "frontend"}, + _suppress_timing_output=lambda route: True, + ) + context = SimpleNamespace( + name="Main", + root=project_root, + ports={ + "backend": SimpleNamespace(final=8000), + "frontend": SimpleNamespace(final=3000), + }, + ) + route = parse_route( + ["--plan", "feature-a", "--docker", "--entire-system", "--batch"], + env={}, + ) + + with mock.patch( + "envctl_engine.startup.service_project_execution.ServiceAttachRunner" + ) as runner_cls: + runner_cls.return_value.start.return_value = {} + records = start_project_services( + orchestrator, + context, + requirements=RequirementsResult(project="Main", components={}, health="healthy", failures=[]), + run_id="run-1", + route=route, + ) + + self.assertEqual(records, {}) + self.assertEqual(prep_calls, []) + self.assertTrue(runner_cls.call_args.kwargs["docker_mode"]) + self.assertEqual( + runner_cls.call_args.kwargs["prepared_launches"]["backend"].command_source, + "docker_image", + ) + self.assertFalse( + any( + event == "service.attach.skipped" and payload.get("reason") == "no_system_configured" + for event, payload in events + ) + ) + def test_start_project_services_keeps_missing_command_failure_for_explicit_enablement(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -1039,7 +1635,9 @@ def test_start_project_services_keeps_missing_command_failure_for_explicit_enabl _project_service_env=lambda context, requirements, route=None, service_name=None: {}, _resolve_backend_env_file=lambda context, backend_cwd: (None, False), _resolve_frontend_env_file=lambda context, frontend_cwd: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service: True, process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: None), _prepare_backend_runtime=lambda **kwargs: None, @@ -1101,7 +1699,9 @@ def test_start_project_services_keeps_missing_command_failure_for_explicit_addit _project_service_env=lambda context, requirements, route=None, service_name=None: {}, _resolve_backend_env_file=lambda context, backend_cwd: (None, False), _resolve_frontend_env_file=lambda context, frontend_cwd: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service: True, process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: None), _prepare_backend_runtime=lambda **kwargs: None, @@ -1182,12 +1782,16 @@ def test_start_project_services_keeps_autodetectable_default_services_startable( _project_service_env=lambda context, requirements, route=None, service_name=None: {}, _resolve_backend_env_file=lambda context, backend_cwd: (None, False), _resolve_frontend_env_file=lambda context, frontend_cwd: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service: True, process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: None), _prepare_backend_runtime=lambda **kwargs: None, _prepare_frontend_runtime=lambda **kwargs: None, - _service_command_source=lambda service_name, **kwargs: source_calls.append(service_name) or "autodetected", + _service_command_source=lambda service_name, **kwargs: ( + source_calls.append(service_name) or "autodetected" + ), _service_start_command_resolved=lambda service_name, **kwargs: (["npm", "run", "dev"], "autodetected"), _command_exists=lambda command: command in {"npm", "npx"}, _detect_service_actual_port=lambda pid, requested_port, service_name, log_path: requested_port, @@ -1332,15 +1936,17 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: _resolve_frontend_env_file=lambda context, frontend_cwd: None, _prepare_backend_runtime=lambda **kwargs: None, _prepare_frontend_runtime=lambda **kwargs: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=service_enabled_for_mode, _service_command_source=lambda **kwargs: "configured", _service_start_command_resolved=lambda service_name, **kwargs: ("python -c pass", "configured"), _split_command=lambda command, **kwargs: ["python", "-c", "pass"], process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: SimpleNamespace(pid=1234)), - _detect_service_actual_port=lambda service_name, pid, requested_port, **kwargs: 8022 - if service_name == "voice-runtime" - else requested_port, + _detect_service_actual_port=lambda service_name, pid, requested_port, **kwargs: ( + 8022 if service_name == "voice-runtime" else requested_port + ), _listener_truth_enforced=lambda: True, _service_listener_failure_detail=lambda **kwargs: "", _conflict_remaining={}, @@ -1348,7 +1954,9 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: ) orchestrator = SimpleNamespace( runtime=runtime, - _restart_service_types_for_project=lambda route, project_name, default_service_types: set(default_service_types), + _restart_service_types_for_project=lambda route, project_name, default_service_types: set( + default_service_types + ), _suppress_timing_output=lambda route: True, ) context = SimpleNamespace( @@ -1444,13 +2052,18 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: _resolve_frontend_env_file=lambda context, frontend_cwd: None, _prepare_backend_runtime=lambda **kwargs: None, _prepare_frontend_runtime=lambda **kwargs: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service_name: service_name == "voice-runtime", _service_command_source=lambda **kwargs: "configured", - _split_command=lambda command, **kwargs: split_calls.append({"command": command, **kwargs}) or [ - "scripts/envctl/start-voice-runtime.sh", - str(kwargs["port"]), - ], + _split_command=lambda command, **kwargs: ( + split_calls.append({"command": command, **kwargs}) + or [ + "scripts/envctl/start-voice-runtime.sh", + str(kwargs["port"]), + ] + ), process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: SimpleNamespace(pid=1234)), _detect_service_actual_port=lambda **kwargs: 8010, _listener_truth_enforced=lambda: True, @@ -1556,7 +2169,9 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: _resolve_frontend_env_file=lambda context, frontend_cwd: None, _prepare_backend_runtime=lambda **kwargs: None, _prepare_frontend_runtime=lambda **kwargs: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service_name: service_name == "voice-runtime", _service_command_source=lambda **kwargs: "configured", _split_command=lambda command, **kwargs: ["python", "-c", "pass"], diff --git a/tests/python/state/test_fingerprints.py b/tests/python/state/test_fingerprints.py new file mode 100644 index 000000000..e210b7afd --- /dev/null +++ b/tests/python/state/test_fingerprints.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from pathlib import Path + +from envctl_engine.runtime.state_fingerprint_support import ( + state_fingerprint as runtime_state_fingerprint, +) +from envctl_engine.state.fingerprints import file_fingerprint, state_fingerprint, text_fingerprint +from envctl_engine.state.models import RunState + + +def test_shared_fingerprint_helpers_agree_for_text_file_and_runtime_facade(tmp_path: Path) -> None: + payload = "deterministic payload\n" + path = tmp_path / "payload.txt" + path.write_text(payload, encoding="utf-8") + + assert file_fingerprint(path) == text_fingerprint(payload) + + state = RunState(run_id="run-1", mode="trees", metadata={"project_names": ["alpha"]}) + assert runtime_state_fingerprint(state) == state_fingerprint(state) + + +def test_state_fingerprint_changes_with_serialized_state() -> None: + first = RunState(run_id="run-1", mode="main") + second = RunState(run_id="run-1", mode="main", metadata={"changed": True}) + + assert state_fingerprint(first) != state_fingerprint(second) diff --git a/tests/python/state/test_lookup.py b/tests/python/state/test_lookup.py new file mode 100644 index 000000000..5e1fc5be9 --- /dev/null +++ b/tests/python/state/test_lookup.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import unittest + +from envctl_engine.state.lookup import call_state_loader + + +class StateLookupTests(unittest.TestCase): + def test_passes_all_supported_selection_arguments(self) -> None: + captured: dict[str, object] = {} + + def loader( + *, + mode: str | None, + strict_mode_match: bool, + project_names: tuple[str, ...] | None, + ) -> str: + captured.update( + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ) + return "selected" + + result = call_state_loader( + loader, + mode="trees", + strict_mode_match=True, + project_names=("feature-a",), + ) + + self.assertEqual(result, "selected") + self.assertEqual( + captured, + { + "mode": "trees", + "strict_mode_match": True, + "project_names": ("feature-a",), + }, + ) + + def test_omits_unsupported_compatibility_keywords_for_untargeted_lookup(self) -> None: + calls = 0 + + def legacy_loader() -> str: + nonlocal calls + calls += 1 + return "legacy" + + self.assertEqual(call_state_loader(legacy_loader, mode="main"), "legacy") + self.assertEqual(calls, 1) + + def test_targeted_lookup_fails_closed_when_loader_cannot_select_projects(self) -> None: + called = False + + def legacy_loader(*, mode: str | None = None) -> str: + nonlocal called + called = True + return str(mode) + + with self.assertRaisesRegex(TypeError, "accepts project_names"): + call_state_loader(legacy_loader, mode="trees", project_names=("feature-a",)) + self.assertFalse(called) + + def test_var_keyword_loader_receives_selection_arguments(self) -> None: + captured: dict[str, object] = {} + + def flexible_loader(**kwargs: object) -> dict[str, object]: + captured.update(kwargs) + return captured + + self.assertIs( + call_state_loader( + flexible_loader, + mode="trees", + strict_mode_match=True, + project_names=("feature-a",), + ), + captured, + ) + self.assertEqual(captured["project_names"], ("feature-a",)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/state/test_persistence.py b/tests/python/state/test_persistence.py new file mode 100644 index 000000000..2844c6bc4 --- /dev/null +++ b/tests/python/state/test_persistence.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import errno +import os +import tempfile +import threading +import unittest +from pathlib import Path +from unittest.mock import patch + +from envctl_engine.state.persistence import ( + advisory_file_lock, + atomic_write_text, + durable_mkdir, + require_path_component, + scavenge_atomic_write_temps, +) + + +class StatePersistenceTests(unittest.TestCase): + def test_failed_atomic_replaces_preserve_old_value_and_leave_no_temp_files_or_descriptors(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + target = root / "state.json" + target.write_text("old", encoding="utf-8") + descriptor_root = Path("/dev/fd") + descriptors_before = len(list(descriptor_root.iterdir())) if descriptor_root.is_dir() else None + + with patch("envctl_engine.state.persistence.os.replace", side_effect=OSError("forced failure")): + for _ in range(25): + with self.assertRaisesRegex(OSError, "forced failure"): + atomic_write_text(target, "new") + + descriptors_after = len(list(descriptor_root.iterdir())) if descriptor_root.is_dir() else None + self.assertEqual(target.read_text(encoding="utf-8"), "old") + self.assertEqual(list(root.glob(".*.tmp")), []) + if descriptors_before is not None: + self.assertEqual(descriptors_after, descriptors_before) + + def test_target_local_scavenge_removes_only_exact_regular_atomic_temps(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + target = root / "state.json" + stale = root / ".state.json.abcdefgh.tmp" + unrelated_target = root / ".other.json.abcdefgh.tmp" + malformed_token = root / ".state.json.not-atomic.tmp" + generic_tmp = root / "notes.tmp" + external = root / "external" + symlink_temp = root / ".state.json.ijklmnop.tmp" + for path in (stale, unrelated_target, malformed_token, generic_tmp): + path.write_text(path.name, encoding="utf-8") + external.write_text("sentinel", encoding="utf-8") + symlink_temp.symlink_to(external) + + removed = scavenge_atomic_write_temps(target) + + self.assertEqual(removed, 1) + self.assertFalse(stale.exists()) + self.assertTrue(unrelated_target.is_file()) + self.assertTrue(malformed_token.is_file()) + self.assertTrue(generic_tmp.is_file()) + self.assertTrue(symlink_temp.is_symlink()) + self.assertEqual(external.read_text(encoding="utf-8"), "sentinel") + + def test_target_local_scavenge_rejects_symlink_parent_without_touching_target(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + external = root / "external" + external.mkdir() + stale = external / ".state.json.abcdefgh.tmp" + stale.write_text("sentinel", encoding="utf-8") + linked_parent = root / "linked-parent" + linked_parent.symlink_to(external, target_is_directory=True) + + with self.assertRaises(OSError): + scavenge_atomic_write_temps(linked_parent / "state.json") + + self.assertEqual(stale.read_text(encoding="utf-8"), "sentinel") + + def test_path_component_validation_accepts_safe_names_and_rejects_traversal(self) -> None: + self.assertEqual(require_path_component("run-2026_07.11", label="run_id"), "run-2026_07.11") + for value in ("", " ", ".", "..", "../run", "nested/run", "nested\\run", "/tmp/run", "bad\0id"): + with self.subTest(value=value): + with self.assertRaisesRegex(ValueError, "path component"): + require_path_component(value, label="run_id") + + def test_lock_descriptor_is_closed_if_stream_construction_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "state.lock" + opened_descriptors: list[int] = [] + closed_descriptors: list[int] = [] + real_open = os.open + real_close = os.close + + def tracked_open(*args: object, **kwargs: object) -> int: + descriptor = real_open(*args, **kwargs) # type: ignore[arg-type] + opened_descriptors.append(descriptor) + return descriptor + + def tracked_close(descriptor: int) -> None: + closed_descriptors.append(descriptor) + real_close(descriptor) + + with ( + patch("envctl_engine.state.persistence.os.open", side_effect=tracked_open), + patch("envctl_engine.state.persistence.os.fdopen", side_effect=OSError("fdopen failed")), + patch("envctl_engine.state.persistence.os.close", side_effect=tracked_close), + ): + with self.assertRaisesRegex(OSError, "fdopen failed"): + with advisory_file_lock(lock_path, exclusive=True): + self.fail("lock acquisition unexpectedly succeeded") + + self.assertEqual(len(opened_descriptors), 1) + self.assertEqual(closed_descriptors, opened_descriptors) + + def test_lock_rejects_symlink_without_touching_its_target(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + target = root / "external-target" + target.write_text("sentinel", encoding="utf-8") + lock_path = root / "state.lock" + lock_path.symlink_to(target) + + with self.assertRaises(OSError): + with advisory_file_lock(lock_path, exclusive=True): + self.fail("symlink lock unexpectedly succeeded") + + self.assertEqual(target.read_text(encoding="utf-8"), "sentinel") + + def test_lock_timeout_does_not_leak_or_enter_critical_section(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "state.lock" + holder_entered = threading.Event() + release_holder = threading.Event() + + def hold_lock() -> None: + with advisory_file_lock(lock_path, exclusive=True): + holder_entered.set() + release_holder.wait(timeout=2) + + holder = threading.Thread(target=hold_lock) + holder.start() + self.assertTrue(holder_entered.wait(timeout=1)) + try: + with self.assertRaisesRegex(TimeoutError, "Timed out acquiring lock"): + with advisory_file_lock(lock_path, exclusive=True, timeout_seconds=0.02): + self.fail("contended lock unexpectedly succeeded") + finally: + release_holder.set() + holder.join(timeout=1) + self.assertFalse(holder.is_alive()) + + def test_unsupported_directory_fsync_does_not_undo_committed_replace(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + target = Path(tmpdir) / "state.json" + with patch( + "envctl_engine.state.persistence.os.fsync", + side_effect=[None, OSError(errno.EINVAL, "directory fsync unsupported")], + ): + atomic_write_text(target, "committed") + + self.assertEqual(target.read_text(encoding="utf-8"), "committed") + + def test_durable_mkdir_fsyncs_each_new_directory_and_its_parent(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + target = root / "runs" / "run-a" / "revisions" + synced: list[Path] = [] + + with patch( + "envctl_engine.state.persistence.fsync_directory", + side_effect=lambda path: synced.append(path), + ): + durable_mkdir(target) + + self.assertTrue(target.is_dir()) + for directory in (root / "runs", root / "runs" / "run-a", target): + self.assertIn(directory, synced) + self.assertIn(directory.parent, synced) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/state/test_project_runtime_resolution.py b/tests/python/state/test_project_runtime_resolution.py index 0c6cc5435..7dc7298c5 100644 --- a/tests/python/state/test_project_runtime_resolution.py +++ b/tests/python/state/test_project_runtime_resolution.py @@ -8,6 +8,7 @@ active_project_names, cwd_runtime_warnings, dependency_mode_summary, + filter_project_scoped_metadata, infer_cwd_project, requested_project_not_running_payload, resolve_requested_project_state, @@ -29,6 +30,21 @@ def test_active_project_names_include_requirements_services_and_metadata_roots(s self.assertEqual(active_project_names(state), ["alpha", "beta", "delta", "gamma"]) + def test_requirement_collision_storage_key_is_not_exposed_as_an_active_project(self) -> None: + state = RunState( + run_id="run-collision", + mode="main", + requirements={ + "Main": RequirementsResult(project="Main"), + "Main Restart Collision": RequirementsResult(project="Main"), + }, + ) + + self.assertEqual(active_project_names(state), ["Main"]) + resolution = resolve_requested_project_state(state, [], command="endpoints") + self.assertTrue(resolution.ok) + self.assertEqual(resolution.selected_projects, ["Main"]) + def test_exact_requested_project_match_filters_state_to_canonical_project(self) -> None: state = RunState( run_id="run-active", @@ -55,6 +71,80 @@ def test_exact_requested_project_match_filters_state_to_canonical_project(self) self.assertEqual(set(resolution.state.requirements), {"Alpha"}) self.assertEqual(resolution.state.metadata["project_roots"], {"Alpha": "/tmp/a"}) + def test_project_selection_filters_every_project_scoped_metadata_surface(self) -> None: + state = RunState( + run_id="run-metadata", + mode="trees", + services={ + "Alpha Backend": ServiceRecord(name="Alpha Backend", type="backend", cwd="/tmp/a", project="Alpha"), + "Beta Voice": ServiceRecord(name="Beta Voice", type="voice", cwd="/tmp/b", project="Beta"), + }, + metadata={ + "project_roots": {"Alpha": "/tmp/a", "Beta": "/tmp/b"}, + "project_pr_links": {"Alpha": "pr-a", "Beta": "pr-b"}, + "project_test_summaries": {"Alpha": {"failed": 0}, "Beta": {"failed": 1}}, + "project_test_results_root": "/tmp/results", + "project_test_results_updated_at": "now", + "project_action_reports": {"Alpha": {"test": {}}, "Beta": {"test": {}}}, + "dashboard_project_configured_services": { + "Alpha": ["backend"], + "Beta": ["voice"], + }, + "dashboard_configured_service_types": ["backend", "voice"], + "dashboard_stopped_services": [ + {"name": "Alpha Backend", "project": "Alpha", "type": "backend"}, + {"name": "Beta Voice", "project": "Beta", "type": "voice"}, + {"project": "Alpha", "type": "frontend"}, + ], + "startup_identity": { + "mode": "trees", + "projects": [ + {"name": "Alpha", "root": "/tmp/a"}, + {"name": "Beta", "root": "/tmp/b"}, + ], + "fingerprint": "stale", + }, + "unrelated": "preserved", + }, + ) + + resolution = resolve_requested_project_state(state, ["Alpha"], command="health") + + self.assertTrue(resolution.ok) + assert resolution.state is not None + metadata = resolution.state.metadata + for key in ( + "project_roots", + "project_pr_links", + "project_test_summaries", + "project_action_reports", + "dashboard_project_configured_services", + ): + self.assertEqual(set(metadata[key]), {"Alpha"}) + self.assertEqual( + metadata["dashboard_stopped_services"], + [{"name": "Alpha Backend", "project": "Alpha", "type": "backend"}], + ) + self.assertEqual(metadata["dashboard_configured_service_types"], ["backend"]) + self.assertEqual(metadata["startup_identity"]["projects"], [{"name": "Alpha", "root": "/tmp/a"}]) + self.assertNotEqual(metadata["startup_identity"]["fingerprint"], "stale") + self.assertNotIn("project_test_results_root", metadata) + self.assertNotIn("project_test_results_updated_at", metadata) + self.assertEqual(metadata["unrelated"], "preserved") + + def test_filter_removes_test_companion_fields_when_no_selected_summary_remains(self) -> None: + metadata = filter_project_scoped_metadata( + { + "project_test_summaries": {"Beta": {"failed": 1}}, + "project_test_results_root": "/tmp/results", + "project_test_results_updated_at": "now", + }, + {"Alpha"}, + ) + + self.assertNotIn("project_test_summaries", metadata) + self.assertNotIn("project_test_results_root", metadata) + self.assertNotIn("project_test_results_updated_at", metadata) def test_case_mismatched_project_selector_fails_closed(self) -> None: state = RunState( @@ -148,14 +238,18 @@ def test_dependency_mode_summary_prefers_explicit_metadata_and_derives_legacy_sh mode="trees", metadata={"dependency_mode": "isolated", "shared_dependencies": False}, ) - self.assertEqual(dependency_mode_summary(explicit), {"dependency_mode": "isolated", "shared_dependencies": False}) + self.assertEqual( + dependency_mode_summary(explicit), {"dependency_mode": "isolated", "shared_dependencies": False} + ) legacy_shared = RunState( run_id="run-legacy", mode="trees", metadata={"dashboard_dependency_scope": "shared"}, ) - self.assertEqual(dependency_mode_summary(legacy_shared), {"dependency_mode": "shared", "shared_dependencies": True}) + self.assertEqual( + dependency_mode_summary(legacy_shared), {"dependency_mode": "shared", "shared_dependencies": True} + ) unknown = RunState(run_id="run-unknown", mode="trees") self.assertEqual(dependency_mode_summary(unknown), {"dependency_mode": "unknown", "shared_dependencies": None}) diff --git a/tests/python/state/test_run_index.py b/tests/python/state/test_run_index.py new file mode 100644 index 000000000..db8a61471 --- /dev/null +++ b/tests/python/state/test_run_index.py @@ -0,0 +1,437 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import patch + +from envctl_engine.state.persistence import atomic_write_text +from envctl_engine.state.run_index import RunIndex, RunIndexCandidate, StateSelector + + +class RunIndexTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.runtime_dir = Path(self.temporary_directory.name) / "runtime" + self.runtime_root = self.runtime_dir / "repo-scope" + self.index = RunIndex( + runtime_root=self.runtime_root, + runtime_dir=self.runtime_dir, + runtime_scope_id="repo-scope", + ) + self.revision_number = 0 + + def _state_path(self, run_id: str) -> Path: + self.revision_number += 1 + path = self.runtime_root / "runs" / run_id / "revisions" / f"revision-{self.revision_number}" / "run_state.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{}\n", encoding="utf-8") + return path + + def _record(self, run_id: str, *, mode: str = "trees", projects: list[str]) -> Path: + path = self._state_path(run_id) + self.index.record( + state_path=path, + run_id=run_id, + mode=mode, + project_names=projects, + ) + return path.resolve() + + def test_new_records_replace_overlapping_project_owners_without_crossing_modes(self) -> None: + _ = self._record("exact-old", projects=["Feature-A"]) + containing = self._record("containing", projects=["feature-a", "feature-b"]) + partial = self._record("partial-overlap", projects=["feature-c", "feature-d"]) + exact_new = self._record("exact-new", projects=["FEATURE-A"]) + _ = self._record("disjoint", projects=["feature-z"]) + wrong_mode = self._record("wrong-mode", mode="main", projects=["feature-a"]) + + self.assertEqual( + self.index.candidate_paths(StateSelector(mode="TREES", project_names=("feature-a",))), + [exact_new], + ) + self.assertEqual( + self.index.candidate_paths(StateSelector(mode="trees", project_names=("feature-b",))), + [containing], + ) + self.assertEqual( + self.index.candidate_paths(StateSelector(mode="main", project_names=("feature-a",))), + [wrong_mode], + ) + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-a", "feature-c"))), + [exact_new, partial], + ) + + def test_record_heals_multiple_historical_owners_for_the_same_project(self) -> None: + run_a = self._state_path("run-a") + run_b = self._state_path("run-b") + self.index.replace_all( + [ + RunIndexCandidate(run_a, "run-a", "main", ("main",), 1), + RunIndexCandidate(run_b, "run-b", "main", ("main",), 2), + ] + ) + + run_c = self._record("run-c", mode="main", projects=["Main"]) + + self.assertEqual( + self.index.candidate_paths(StateSelector("main", ("main",))), + [run_c], + ) + payload = json.loads(self.index.index_path.read_text(encoding="utf-8")) + self.assertEqual(payload["retired_run_ids"], ["run-a", "run-b"]) + + def test_record_trims_mixed_historical_owner_without_losing_unrelated_project(self) -> None: + run_a = self._state_path("run-a") + run_b = self._state_path("run-b") + self.index.replace_all( + [ + RunIndexCandidate(run_a, "run-a", "trees", ("feature-x", "feature-y"), 1), + RunIndexCandidate(run_b, "run-b", "trees", ("feature-x",), 2), + ] + ) + + run_c = self._record("run-c", projects=["feature-x"]) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-x",))), + [run_c], + ) + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-y",))), + [run_a.resolve()], + ) + payload = json.loads(self.index.index_path.read_text(encoding="utf-8")) + entries = {entry["run_id"]: entry["project_names"] for entry in payload["entries"]} + self.assertEqual(entries, {"run-a": ["feature-y"], "run-c": ["feature-x"]}) + self.assertEqual(payload["retired_run_ids"], ["run-b"]) + + def test_preserves_independent_entries_during_concurrent_records(self) -> None: + run_ids = [f"run-{index:02d}" for index in range(32)] + state_paths = {run_id: self._state_path(run_id).resolve() for run_id in run_ids} + + def record(run_id: str) -> None: + self.index.record( + state_path=state_paths[run_id], + run_id=run_id, + mode="trees", + project_names=[run_id], + ) + + with ThreadPoolExecutor(max_workers=12) as executor: + list(executor.map(record, run_ids)) + + candidates = self.index.candidate_paths(StateSelector(mode="trees", project_names=())) + self.assertEqual(len(candidates), len(run_ids)) + self.assertEqual(set(candidates), set(state_paths.values())) + payload = json.loads((self.runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(len(payload["entries"]), len(run_ids)) + + def test_corrupt_index_fails_closed_and_later_record_replaces_it(self) -> None: + self.runtime_root.mkdir(parents=True, exist_ok=True) + index_path = self.runtime_root / "run_index.json" + index_path.write_text('{"entries":', encoding="utf-8") + + self.assertEqual(self.index.candidate_paths(StateSelector(mode=None, project_names=())), []) + + recovered = self._record("recovered", projects=["feature-a"]) + self.assertEqual( + self.index.candidate_paths(StateSelector(mode="trees", project_names=("feature-a",))), + [recovered], + ) + payload = json.loads(index_path.read_text(encoding="utf-8")) + self.assertEqual(payload["version"], 4) + self.assertGreaterEqual(payload["generation"], 1) + + def test_clean_registry_write_scavenges_only_its_crash_temp_targets(self) -> None: + self.runtime_root.mkdir(parents=True, exist_ok=True) + stale_primary = self.runtime_root / ".run_index.json.abcdefgh.tmp" + stale_backup = self.runtime_root / ".run_index.backup.json.ijklmnop.tmp" + unrelated = self.runtime_root / ".run_state.json.abcdefgh.tmp" + for path in (stale_primary, stale_backup, unrelated): + path.write_text("crash debris", encoding="utf-8") + + self.index.initialize_empty() + + self.assertFalse(stale_primary.exists()) + self.assertFalse(stale_backup.exists()) + self.assertTrue(unrelated.is_file()) + + def test_rejects_record_paths_outside_runtime_dir(self) -> None: + outside = Path(self.temporary_directory.name) / "runtime-sibling" / "run_state.json" + outside.parent.mkdir(parents=True, exist_ok=True) + outside.write_text("{}\n", encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "escapes runtime_root"): + self.index.record( + state_path=outside, + run_id="outside", + mode="trees", + project_names=["feature-a"], + ) + + def test_rejects_state_path_from_another_runtime_scope(self) -> None: + foreign = self.runtime_dir / "another-scope" / "runs" / "foreign" / "run_state.json" + foreign.parent.mkdir(parents=True, exist_ok=True) + foreign.write_text("{}\n", encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "escapes runtime_root"): + self.index.record( + state_path=foreign, + run_id="foreign", + mode="trees", + project_names=["feature-a"], + ) + + def test_rejects_symlinked_or_noncanonical_revision_paths(self) -> None: + real_path = self._state_path("safe") + alias_path = real_path.parent / "alias.json" + alias_path.symlink_to(real_path) + + for path in (alias_path, real_path.parent / ".." / real_path.parent.name / real_path.name): + with self.subTest(path=path): + with self.assertRaisesRegex(ValueError, "canonical"): + self.index.record( + state_path=path, + run_id="safe", + mode="trees", + project_names=["feature-a"], + ) + + def test_runtime_root_retarget_after_construction_is_rejected_without_external_writes(self) -> None: + external = Path(self.temporary_directory.name) / "external" + external.mkdir() + sentinel = external / "sentinel.txt" + sentinel.write_text("untouched", encoding="utf-8") + self.runtime_root.parent.mkdir(parents=True, exist_ok=True) + self.runtime_root.symlink_to(external, target_is_directory=True) + + with self.assertRaisesRegex(RuntimeError, "symlink"): + self.index.candidates(StateSelector(mode=None, project_names=())) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "untouched") + self.assertEqual(list(external.iterdir()), [sentinel]) + + def test_traversal_entry_fails_closed_and_wrong_scope_is_not_loaded(self) -> None: + safe = self._record("safe", projects=["feature-a"]) + index_path = self.runtime_root / "run_index.json" + payload = json.loads(index_path.read_text(encoding="utf-8")) + payload["entries"].append( + { + "state_path": str(self.runtime_root / ".." / ".." / "escaped.json"), + "run_id": "escaped", + "mode": "trees", + "project_names": ["feature-a"], + "sequence": 2, + "runtime_scope_id": "repo-scope", + } + ) + index_path.write_text(json.dumps(payload), encoding="utf-8") + self.assertEqual(self.index.candidate_paths(StateSelector("trees", ("feature-a",))), [safe]) + + payload["entries"] = [entry for entry in payload["entries"] if entry["run_id"] == "safe"] + payload["runtime_scope_id"] = "another-scope" + index_path.write_text(json.dumps(payload), encoding="utf-8") + (self.runtime_root / "run_index.backup.json").write_text(json.dumps(payload), encoding="utf-8") + self.assertEqual(self.index.candidate_paths(StateSelector("trees", ("feature-a",))), []) + self.assertTrue(safe.is_file()) + + def test_purge_removes_index_without_preventing_future_records(self) -> None: + before = self._record("before-purge", projects=["feature-a"]) + self.index.purge() + payload = json.loads((self.runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(payload["entries"], []) + self.assertEqual(payload["retired_run_ids"], ["before-purge"]) + self.assertEqual(self.index.candidate_paths(StateSelector(None, ())), []) + + with self.assertRaisesRegex(RuntimeError, "retired"): + self.index.record( + state_path=before, + run_id="before-purge", + mode="trees", + project_names=["feature-a"], + ) + + after = self._record("after-purge", projects=["feature-b"]) + self.assertEqual(self.index.candidate_paths(StateSelector("trees", ("feature-b",))), [after]) + + def test_empty_update_preserves_existing_project_ownership(self) -> None: + state_path = self._record("run-a", projects=["feature-a"]) + + self.index.record( + state_path=state_path, + run_id="run-a", + mode="trees", + project_names=[], + ) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-a",))), + [state_path], + ) + + def test_rejects_invalid_run_id_components_before_indexing(self) -> None: + for run_id in (".", "..", "../other", "nested/run", "nested\\run", "/absolute"): + with self.subTest(run_id=run_id): + with self.assertRaisesRegex(ValueError, "path component"): + self.index.record( + state_path=self.runtime_root / "runs" / "safe" / "run_state.json", + run_id=run_id, + mode="trees", + project_names=["feature-a"], + ) + + def test_remove_deactivates_only_requested_run(self) -> None: + run_a = self._record("run-a", projects=["feature-a"]) + run_b = self._record("run-b", projects=["feature-b"]) + + self.assertTrue(self.index.remove("run-a")) + self.assertFalse(self.index.remove("run-a")) + self.assertEqual(self.index.candidate_paths(StateSelector("trees", ("feature-a",))), []) + self.assertEqual(self.index.candidate_paths(StateSelector("trees", ("feature-b",))), [run_b]) + self.assertTrue(run_a.is_file()) + + def test_superseded_run_is_durably_fenced_from_reclaiming_ownership(self) -> None: + run_a = self._record("run-a", projects=["feature-a"]) + run_b = self._state_path("run-b") + self.index.record( + state_path=run_b, + run_id="run-b", + mode="trees", + project_names=["feature-a"], + supersede_run_ids=["run-a"], + ) + + with self.assertRaisesRegex(RuntimeError, "retired"): + self.index.record( + state_path=run_a, + run_id="run-a", + mode="trees", + project_names=["feature-a"], + ) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-a",))), + [run_b.resolve()], + ) + + def test_removing_untracked_source_id_still_fences_late_writer(self) -> None: + self.index.initialize_empty() + self.assertFalse(self.index.remove("late-source")) + + with self.assertRaisesRegex(RuntimeError, "retired"): + self.index.record( + state_path=self._state_path("late-source"), + run_id="late-source", + mode="trees", + project_names=["feature-a"], + ) + + def test_single_copy_failure_keeps_newest_generation_readable_and_repairable(self) -> None: + run_a = self._record("run-a", projects=["feature-a"]) + run_b = self._state_path("run-b") + + def write_backup_only(path: Path, text: str) -> None: + if path == self.index.index_path: + raise OSError("primary unavailable") + atomic_write_text(path, text) + + with patch("envctl_engine.state.run_index.atomic_write_text", side_effect=write_backup_only): + self.index.record( + state_path=run_b, + run_id="run-b", + mode="trees", + project_names=["feature-b"], + ) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ())), + [run_b.resolve(), run_a], + ) + primary_before = json.loads(self.index.index_path.read_text(encoding="utf-8")) + backup_before = json.loads(self.index.backup_path.read_text(encoding="utf-8")) + self.assertGreater(backup_before["generation"], primary_before["generation"]) + + self.index.repair_copies() + + primary_after = json.loads(self.index.index_path.read_text(encoding="utf-8")) + backup_after = json.loads(self.index.backup_path.read_text(encoding="utf-8")) + self.assertEqual(primary_after, backup_after) + self.assertEqual( + {entry["run_id"] for entry in primary_after["entries"]}, + {"run-a", "run-b"}, + ) + + def test_equal_generation_divergence_fails_closed_without_overwriting_either_copy(self) -> None: + self._record("run-a", projects=["feature-a"]) + indexed_entry = self.index.candidates(StateSelector("trees", ()))[0] + primary_before = self.index.index_path.read_text(encoding="utf-8") + backup_payload = json.loads(self.index.backup_path.read_text(encoding="utf-8")) + backup_payload["entries"][0]["project_names"] = ["feature-b"] + backup_before = json.dumps(backup_payload, indent=2, sort_keys=True) + "\n" + self.index.backup_path.write_text(backup_before, encoding="utf-8") + + with self.assertRaisesRegex(RuntimeError, "diverged at generation"): + self.index.candidate_paths(StateSelector("trees", ())) + with self.assertRaisesRegex(RuntimeError, "diverged at generation"): + self.index.repair_copies() + with self.assertRaisesRegex(RuntimeError, "diverged at generation"): + self.index.initialize_empty() + with self.assertRaisesRegex(RuntimeError, "diverged at generation"): + self.index.replace_all([indexed_entry]) + + self.assertEqual(self.index.index_path.read_text(encoding="utf-8"), primary_before) + self.assertEqual(self.index.backup_path.read_text(encoding="utf-8"), backup_before) + + def test_total_registry_write_failure_preserves_last_committed_generation(self) -> None: + run_a = self._record("run-a", projects=["feature-a"]) + run_b = self._state_path("run-b") + + with patch( + "envctl_engine.state.run_index.atomic_write_text", + side_effect=OSError("storage unavailable"), + ): + with self.assertRaisesRegex(OSError, "storage unavailable"): + self.index.record( + state_path=run_b, + run_id="run-b", + mode="trees", + project_names=["feature-b"], + ) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ())), + [run_a], + ) + + def test_visible_registry_replace_is_commit_even_if_post_replace_fsync_reports_error(self) -> None: + state_path = self._state_path("run-visible") + + def replace_then_report_error(path: Path, text: str) -> None: + atomic_write_text(path, text) + raise OSError("directory fsync result unknown") + + with patch( + "envctl_engine.state.run_index.atomic_write_text", + side_effect=replace_then_report_error, + ): + self.index.record( + state_path=state_path, + run_id="run-visible", + mode="trees", + project_names=["feature-a"], + ) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-a",))), + [state_path.resolve()], + ) + self.assertTrue(state_path.is_file()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/state/test_state_action_log_support.py b/tests/python/state/test_state_action_log_support.py index ee9500ab0..c58fef5d5 100644 --- a/tests/python/state/test_state_action_log_support.py +++ b/tests/python/state/test_state_action_log_support.py @@ -103,6 +103,30 @@ def test_clear_service_logs_truncates_available_files_and_reports_missing_paths( self.assertIn("Main Frontend: log missing", rendered) self.assertIn("Main Worker: log=n/a", rendered) + def test_clear_service_logs_advances_docker_cursor_without_snapshot_file(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + log_path = Path(tmp) / "docker-backend.log" + marker = Path(f"{log_path}.docker-since") + state = RunState( + run_id="run-docker", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=tmp, + log_path=str(log_path), + runtime_kind="docker", + ) + }, + ) + + summary = StateActionLogSupport.clear_service_logs(state, quiet=True) + + self.assertEqual(summary, (1, 0, 0, 0)) + self.assertFalse(log_path.exists()) + self.assertTrue(marker.is_file()) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/state/test_state_action_orchestrator_logs.py b/tests/python/state/test_state_action_orchestrator_logs.py index fb6056b21..b773917a5 100644 --- a/tests/python/state/test_state_action_orchestrator_logs.py +++ b/tests/python/state/test_state_action_orchestrator_logs.py @@ -3,6 +3,7 @@ from contextlib import redirect_stdout from io import StringIO import json +import signal import tempfile import unittest from types import SimpleNamespace @@ -15,6 +16,7 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_runtime_support import normalize_log_line from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord +from envctl_engine.state.action_command_support import _interrupt_log_follow_on_termination from envctl_engine.state.action_orchestrator import StateActionOrchestrator from envctl_engine.test_output.parser_base import strip_ansi from envctl_engine.ui.target_selector import TargetSelection @@ -75,6 +77,17 @@ def _emit(self, event: str, **payload: object) -> None: class StateActionOrchestratorLogsTests(unittest.TestCase): + def test_log_follow_termination_signal_interrupts_for_finally_cleanup(self) -> None: + previous = signal.getsignal(signal.SIGTERM) + + with self.assertRaises(KeyboardInterrupt): + with _interrupt_log_follow_on_termination(True): + handler = signal.getsignal(signal.SIGTERM) + self.assertTrue(callable(handler)) + handler(signal.SIGTERM, None) # type: ignore[operator] + + self.assertIs(signal.getsignal(signal.SIGTERM), previous) + def test_runtime_facade_routes_state_dependencies(self) -> None: state = RunState(run_id="run-0", mode="main") runtime = _RuntimeStub(state) @@ -146,6 +159,68 @@ def test_logs_service_filter_accepts_additional_service_slug_and_display_forms(s assert runtime.seen_logs_state is not None self.assertEqual(set(runtime.seen_logs_state.services), {"Main Voice Runtime"}) + def test_logs_combined_project_and_service_filters_intersect(self) -> None: + state = RunState( + run_id="run-combined-filter", + mode="trees", + services={ + "Alpha Backend": ServiceRecord( + name="Alpha Backend", type="backend", cwd="/tmp/alpha", project="Alpha" + ), + "Alpha Frontend": ServiceRecord( + name="Alpha Frontend", type="frontend", cwd="/tmp/alpha", project="Alpha" + ), + "Beta Backend": ServiceRecord( + name="Beta Backend", type="backend", cwd="/tmp/beta", project="Beta" + ), + }, + ) + runtime = _RuntimeStub(state) + orchestrator = StateActionOrchestrator(runtime) + route = Route( + command="logs", + mode="trees", + projects=["Alpha"], + flags={"services": ["backend"], "json": True}, + ) + + output = StringIO() + with redirect_stdout(output): + code = orchestrator.execute(route) + + payload = json.loads(output.getvalue()) + self.assertEqual(code, 0) + self.assertEqual([service["name"] for service in payload["services"]], ["Alpha Backend"]) + + def test_logs_project_filter_uses_additional_service_project_metadata(self) -> None: + service = ServiceRecord( + name="Opaque Runtime Process", + type="voice-runtime", + cwd="/tmp/customer-platform/voice", + project="Customer Platform", + service_slug="voice-runtime", + ) + requirements = RequirementsResult(project="Customer Platform") + state = RunState( + run_id="run-metadata-project", + mode="trees", + services={service.name: service}, + requirements={"Customer Platform": requirements}, + ) + runtime = _RuntimeStub(state) + runtime._project_name_from_service = lambda _name: "Opaque Runtime" # type: ignore[method-assign] + orchestrator = StateActionOrchestrator(runtime) + + code = orchestrator.execute( + Route(command="logs", mode="trees", projects=["Customer Platform"]) + ) + + self.assertEqual(code, 0) + self.assertIsNotNone(runtime.seen_logs_state) + assert runtime.seen_logs_state is not None + self.assertEqual(set(runtime.seen_logs_state.services), {service.name}) + self.assertEqual(runtime.seen_logs_state.requirements, {"Customer Platform": requirements}) + def test_health_prints_enabled_dependency_statuses(self) -> None: state = RunState( run_id="run-1", diff --git a/tests/python/state/test_state_aggregation.py b/tests/python/state/test_state_aggregation.py new file mode 100644 index 000000000..d4d5993e2 --- /dev/null +++ b/tests/python/state/test_state_aggregation.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord +from envctl_engine.state.run_index import RunIndexCandidate +from envctl_engine.state.state_aggregation import ( + IndexedState, + filter_state_to_owned_projects, + select_indexed_owners, + state_from_indexed_owners, +) + + +def _candidate( + run_id: str, + *projects: str, + sequence: int, + activation_sequence: int, +) -> RunIndexCandidate: + return RunIndexCandidate( + state_path=Path("/runtime") / run_id / "run_state.json", + run_id=run_id, + mode="trees", + project_names=tuple(project.casefold() for project in projects), + sequence=sequence, + activation_sequence=activation_sequence, + ) + + +def _service_project_name(_name: str, service: ServiceRecord) -> str: + return str(service.project or "") + + +def _project_names(state: RunState) -> list[str]: + raw = state.metadata.get("project_names") + return [str(name) for name in raw] if isinstance(raw, list) else [] + + +def _source_run_ids(state: RunState) -> list[str]: + raw = state.metadata.get("state_source_run_ids") + return [str(run_id) for run_id in raw] if isinstance(raw, list) else [] + + +def test_owner_selection_is_total_for_an_empty_inventory() -> None: + assert ( + select_indexed_owners( + [], + selected_projects=(), + service_project_name=_service_project_name, + ) + == [] + ) + + +def test_owner_selection_filters_shadowed_run_to_projects_it_still_owns() -> None: + newest = IndexedState( + candidate=_candidate("new", "alpha", sequence=2, activation_sequence=3), + state=RunState( + run_id="new", + mode="trees", + services={ + "alpha backend": ServiceRecord( + name="alpha backend", + type="backend", + cwd="/alpha", + project="alpha", + ) + }, + metadata={"project_names": ["alpha"]}, + ), + ) + previous = IndexedState( + candidate=_candidate("old", "alpha", "beta", sequence=1, activation_sequence=1), + state=RunState( + run_id="old", + mode="trees", + services={ + project: ServiceRecord( + name=project, + type="backend", + cwd=f"/{project}", + project=project, + ) + for project in ("alpha", "beta") + }, + metadata={"project_names": ["alpha", "beta"]}, + ), + ) + + selected = select_indexed_owners( + [newest, previous], + selected_projects=(), + service_project_name=_service_project_name, + ) + + assert [indexed.candidate.run_id for indexed in selected] == ["new", "old"] + assert set(selected[0].state.services) == {"alpha backend"} + assert set(selected[1].state.services) == {"beta"} + assert selected[1].state.metadata["project_names"] == ["beta"] + + +def test_shadowed_multi_project_owner_does_not_relabel_opaque_services_as_remaining_project() -> None: + newest = IndexedState( + candidate=_candidate("new", "alpha", sequence=2, activation_sequence=3), + state=RunState( + run_id="new", + mode="trees", + services={ + "New Alpha": ServiceRecord( + name="New Alpha", + type="worker", + cwd="/new-alpha", + pid=3, + project="Alpha", + ) + }, + metadata={"project_names": ["Alpha"]}, + ), + ) + previous = IndexedState( + candidate=_candidate("old", "alpha", "beta", sequence=1, activation_sequence=1), + state=RunState( + run_id="old", + mode="trees", + services={ + "Opaque Old Alpha": ServiceRecord( + name="Opaque Old Alpha", + type="worker", + cwd="/old-alpha", + pid=1, + ), + "Opaque Beta": ServiceRecord( + name="Opaque Beta", + type="worker", + cwd="/beta", + pid=2, + ), + }, + metadata={ + "project_names": ["Alpha", "Beta"], + "project_roots": {"Alpha": "/old-alpha", "Beta": "/beta"}, + }, + ), + ) + + selected = select_indexed_owners( + [newest, previous], + selected_projects=(), + service_project_name=_service_project_name, + ) + + old = next(item.state for item in selected if item.candidate.run_id == "old") + assert {service.pid for service in old.services.values()} == {2} + assert {service.project for service in old.services.values()} == {"Beta"} + + +def test_multi_project_opaque_service_without_root_evidence_fails_closed() -> None: + ambiguous = IndexedState( + candidate=_candidate("ambiguous", "alpha", "beta", sequence=1, activation_sequence=1), + state=RunState( + run_id="ambiguous", + mode="trees", + services={ + "Opaque Shared Runtime": ServiceRecord( + name="Opaque Shared Runtime", + type="worker", + cwd="/unknown", + pid=101, + ) + }, + metadata={"project_names": ["Alpha", "Beta"]}, + ), + ) + + with pytest.raises(RuntimeError, match="Cannot safely determine project ownership"): + select_indexed_owners( + [ambiguous], + selected_projects=(), + service_project_name=_service_project_name, + ) + + +def test_merged_state_uses_activation_owner_for_global_identity() -> None: + activated = IndexedState( + candidate=_candidate("activated", "alpha", sequence=1, activation_sequence=5), + state=RunState( + run_id="activated", + mode="trees", + metadata={ + "project_names": ["alpha"], + "project_roots": {"alpha": "/alpha"}, + "global_marker": "activated", + "state_source_run_ids": ["ancestor"], + }, + ), + ) + later_sequence = IndexedState( + candidate=_candidate("stable", "beta", sequence=2, activation_sequence=2), + state=RunState( + run_id="stable", + mode="trees", + metadata={ + "project_names": ["beta"], + "project_roots": {"beta": "/beta"}, + "global_marker": "stable", + }, + ), + ) + + merged = state_from_indexed_owners( + [activated, later_sequence], + project_names_from_state=_project_names, + source_run_ids=_source_run_ids, + ) + + assert merged is not None + assert merged.run_id == "activated" + assert merged.metadata["global_marker"] == "activated" + assert merged.metadata["project_names"] == ["alpha", "beta"] + assert merged.metadata["state_source_run_ids"] == ["activated", "ancestor", "stable"] + + +def test_owned_project_filter_keeps_every_authoritative_requirement_storage_row() -> None: + state = RunState( + run_id="collision", + mode="trees", + requirements={ + "Main": RequirementsResult(project="Main", redis={"enabled": True, "final": 6380}), + "Main Restart Collision": RequirementsResult( + project="Main", + redis={"enabled": True, "final": 6381}, + ), + "Other": RequirementsResult(project="Other", redis={"enabled": True, "final": 6390}), + }, + ) + + filtered = filter_state_to_owned_projects( + state, + frozenset({"main"}), + service_project_name=_service_project_name, + ) + + assert set(filtered.requirements) == {"Main", "Main Restart Collision"} + + +def test_indexed_merge_disambiguates_opaque_legacy_names_using_single_owner_context() -> None: + alpha = IndexedState( + candidate=_candidate("alpha-run", "Alpha", sequence=1, activation_sequence=1), + state=RunState( + run_id="alpha-run", + mode="trees", + services={ + "Opaque Shared Runtime": ServiceRecord( + name="Opaque Shared Runtime", + type="worker", + cwd="/alpha", + pid=101, + ) + }, + metadata={"project_names": ["Alpha"]}, + ), + ) + beta = IndexedState( + candidate=_candidate("beta-run", "Beta", sequence=2, activation_sequence=2), + state=RunState( + run_id="beta-run", + mode="trees", + services={ + "Opaque Shared Runtime": ServiceRecord( + name="Opaque Shared Runtime", + type="worker", + cwd="/beta", + pid=202, + ) + }, + metadata={"project_names": ["Beta"]}, + ), + ) + + merged = state_from_indexed_owners( + [alpha, beta], + project_names_from_state=_project_names, + source_run_ids=_source_run_ids, + ) + + assert merged is not None + assert {service.pid for service in merged.services.values()} == {101, 202} + assert {service.project for service in merged.services.values()} == {"Alpha", "Beta"} + assert len(merged.services) == 2 + + selected = select_indexed_owners( + [alpha, beta], + selected_projects=(), + service_project_name=_service_project_name, + ) + operational = state_from_indexed_owners( + selected, + project_names_from_state=_project_names, + source_run_ids=_source_run_ids, + ) + + assert operational is not None + assert {service.pid for service in operational.services.values()} == {101, 202} + assert {service.project for service in operational.services.values()} == {"Alpha", "Beta"} diff --git a/tests/python/state/test_state_repository_contract.py b/tests/python/state/test_state_repository_contract.py index 1fc8b92d4..f23b70a09 100644 --- a/tests/python/state/test_state_repository_contract.py +++ b/tests/python/state/test_state_repository_contract.py @@ -4,20 +4,25 @@ import json import tempfile import unittest +from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from threading import Event from types import MappingProxyType from types import SimpleNamespace +from unittest.mock import patch REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" _models = importlib.import_module("envctl_engine.state.models") _state = importlib.import_module("envctl_engine.state") _state_repository = importlib.import_module("envctl_engine.state.repository") +_run_index = importlib.import_module("envctl_engine.state.run_index") RunState = _models.RunState ServiceRecord = _models.ServiceRecord dump_state = _state.dump_state load_state = _state.load_state RuntimeStateRepository = _state_repository.RuntimeStateRepository +StateSelector = _run_index.StateSelector class StateRepositoryContractTests(unittest.TestCase): @@ -212,10 +217,10 @@ def test_save_resume_state_updates_scoped_and_legacy_runtime_files(self) -> None self.assertTrue((legacy_root / "runtime_map.json").is_file()) main_pointer = runtime_root / ".last_state.main" self.assertTrue(main_pointer.is_file()) - self.assertEqual( - main_pointer.read_text(encoding="utf-8").strip(), - str(runtime_root / "runs" / "run-1" / "run_state.json"), - ) + pointer_target = Path(main_pointer.read_text(encoding="utf-8").strip()) + self.assertEqual(pointer_target.name, "run_state.json") + self.assertEqual(pointer_target.parent.parent.name, "revisions") + self.assertEqual(pointer_target.parent.parent.parent.name, "run-1") def test_save_selected_stop_state_updates_scoped_and_legacy_runtime_files(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -230,7 +235,19 @@ def test_save_selected_stop_state_updates_scoped_and_legacy_runtime_files(self) runtime_scope_id="repo-123", compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, ) - state = RunState(run_id="run-2", mode="trees", metadata={"repo_scope_id": "repo-123"}) + state = RunState( + run_id="run-2", + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/tmp/feature-a/backend", + project="feature-a", + ) + }, + metadata={"repo_scope_id": "repo-123"}, + ) repo.save_selected_stop_state( state=state, @@ -290,10 +307,10 @@ def test_save_resume_main_pointer_survives_following_trees_runtime_update(self) ) main_pointer = runtime_root / ".last_state.main" main_pointer_target = main_pointer.read_text(encoding="utf-8").strip() - self.assertEqual( - main_pointer_target, - str(runtime_root / "runs" / "run-main-resume" / "run_state.json"), - ) + main_target = Path(main_pointer_target) + self.assertEqual(main_target.name, "run_state.json") + self.assertEqual(main_target.parent.parent.name, "revisions") + self.assertEqual(main_target.parent.parent.parent.name, "run-main-resume") tree_state = RunState( run_id="run-tree-update", @@ -368,7 +385,9 @@ def test_save_run_main_preserves_existing_tree_pointers_for_mode_switch_resume(s ) self.assertTrue(tree_pointer.is_file()) - self.assertEqual(tree_pointer.read_text(encoding="utf-8").strip(), str(tree_state_path)) + migrated_target = Path(tree_pointer.read_text(encoding="utf-8").strip()) + self.assertEqual(load_state(str(migrated_target)).run_id, "run-tree") + self.assertIn("revisions", migrated_target.parts) def test_load_latest_ignores_invalid_scoped_pointer_and_uses_next_valid_candidate(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -472,7 +491,9 @@ def test_save_run_trees_preserves_existing_main_pointer_for_mode_switch_resume(s ) self.assertTrue(main_pointer.is_file()) - self.assertEqual(main_pointer.read_text(encoding="utf-8").strip(), str(main_state_path)) + migrated_target = Path(main_pointer.read_text(encoding="utf-8").strip()) + self.assertEqual(load_state(str(migrated_target)).run_id, "run-main") + self.assertIn("revisions", migrated_target.parts) loaded_main = repo.load_latest(mode="main", strict_mode_match=True) self.assertIsNotNone(loaded_main) @@ -495,7 +516,9 @@ def test_save_run_trees_uses_safe_pointer_filename_for_project_name(self) -> Non compat_mode=RuntimeStateRepository.SCOPED_ONLY, ) - tree_context = SimpleNamespace(name="Feature/API v2", root=Path(tmpdir) / "trees" / "Feature_API_v2", ports={}) + tree_context = SimpleNamespace( + name="Feature/API v2", root=Path(tmpdir) / "trees" / "Feature_API_v2", ports={} + ) state = RunState(run_id="run-tree-safe-name", mode="trees", metadata={"repo_scope_id": "repo-123"}) repo.save_run( state=state, @@ -506,87 +529,2002 @@ def test_save_run_trees_uses_safe_pointer_filename_for_project_name(self) -> Non runtime_map_builder=lambda _state: {"projection": {}}, ) - safe_pointer = runtime_root / ".last_state.trees.Feature_API_v2" + pointer_name = repo._tree_pointer_name("Feature/API v2") + self.assertIsNotNone(pointer_name) + safe_pointer = runtime_root / str(pointer_name) unsafe_pointer_parent = runtime_root / ".last_state.trees.Feature" self.assertTrue(safe_pointer.is_file()) + self.assertIn("Feature_API_v2-", safe_pointer.name) self.assertFalse(unsafe_pointer_parent.exists()) loaded = repo.load_latest(mode="trees", strict_mode_match=True) self.assertIsNotNone(loaded) assert loaded is not None self.assertEqual(loaded.run_id, "run-tree-safe-name") - def test_resume_orchestrator_does_not_write_legacy_state_files_directly(self) -> None: - resume_path = REPO_ROOT / "python/envctl_engine/startup/resume_orchestrator.py" - raw = resume_path.read_text(encoding="utf-8") - self.assertNotIn('runtime_legacy_root / "run_state.json"', raw) - self.assertNotIn('runtime_legacy_root / "runtime_map.json"', raw) + def test_independent_tree_runs_remain_selection_addressable(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) - def test_lifecycle_cleanup_orchestrator_does_not_write_legacy_state_files_directly(self) -> None: - cleanup_path = REPO_ROOT / "python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py" - raw = cleanup_path.read_text(encoding="utf-8") - self.assertNotIn('runtime_legacy_root / "run_state.json"', raw) - self.assertNotIn('runtime_legacy_root / "runtime_map.json"', raw) + for project_name in ("feature-zeta", "feature-alpha"): + state = RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=f"/tmp/{project_name}", + pid=123, + requested_port=8000, + actual_port=8000, + status="running", + ) + }, + metadata={"repo_scope_id": "repo-123"}, + ) + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + self.assertTrue((runtime_root / ".last_state.trees.feature-zeta").is_file()) + self.assertTrue((runtime_root / ".last_state.trees.feature-alpha").is_file()) + zeta = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-zeta"], + ) + alpha = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-alpha"], + ) - def test_engine_runtime_legacy_compat_helper_is_removed(self) -> None: - runtime_path = REPO_ROOT / "python/envctl_engine/runtime/engine_runtime.py" - raw = runtime_path.read_text(encoding="utf-8") - self.assertNotIn("def _write_legacy_runtime_compat_files(", raw) + self.assertIsNotNone(zeta) + self.assertIsNotNone(alpha) + assert zeta is not None and alpha is not None + self.assertEqual(zeta.run_id, "run-feature-zeta") + self.assertEqual(alpha.run_id, "run-feature-alpha") - def test_aggressive_purge_removes_runtime_scoped_test_results_with_run(self) -> None: + def test_selection_aware_lookup_uses_new_owner_when_superset_supersedes_exact_run(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: runtime_dir = Path(tmpdir) / "runtime" runtime_root = runtime_dir / "scope" - legacy_root = runtime_dir / "python-engine" - runtime_root.mkdir(parents=True, exist_ok=True) - legacy_root.mkdir(parents=True, exist_ok=True) repo = RuntimeStateRepository( runtime_root=runtime_root, - runtime_legacy_root=legacy_root, + runtime_legacy_root=runtime_dir / "python-engine", runtime_dir=runtime_dir, runtime_scope_id="repo-123", compat_mode=RuntimeStateRepository.SCOPED_ONLY, ) - run_state = RunState(run_id="run-1", mode="main", metadata={"repo_scope_id": "repo-123"}) + exact_context = SimpleNamespace(name="feature-a", root=Path(tmpdir) / "feature-a", ports={}) + repo.save_run( + state=RunState(run_id="run-exact", mode="trees", metadata={"repo_scope_id": "repo-123"}), + contexts=[exact_context], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + superset_contexts = [ + exact_context, + SimpleNamespace(name="feature-b", root=Path(tmpdir) / "feature-b", ports={}), + ] + repo.save_run( + state=RunState(run_id="run-superset", mode="trees", metadata={"repo_scope_id": "repo-123"}), + contexts=superset_contexts, + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + loaded = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual(loaded.run_id, "run-superset") + + def test_save_run_atomically_supersedes_every_source_run(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + contexts = [ + SimpleNamespace(name="feature-a", root=Path(tmpdir) / "feature-a", ports={}), + SimpleNamespace(name="feature-b", root=Path(tmpdir) / "feature-b", ports={}), + ] + old_state = RunState( + run_id="run-old", + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/tmp/feature-a", + project="feature-a", + ), + "feature-b Backend": ServiceRecord( + name="feature-b Backend", + type="backend", + cwd="/tmp/feature-b", + project="feature-b", + ), + }, + metadata={"repo_scope_id": "repo-123"}, + ) + repo.save_run( + state=old_state, + contexts=contexts, + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + replacement = RunState( + run_id="run-new", + mode="trees", + services=dict(old_state.services), + metadata={ + "repo_scope_id": "repo-123", + "state_source_run_ids": ["run-old"], + }, + ) + + repo.save_run( + state=replacement, + contexts=[contexts[0]], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + self.assertEqual([state.run_id for state in repo.load_all(mode="trees")], ["run-new"]) + loaded_b = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-b"], + ) + self.assertIsNotNone(loaded_b) + assert loaded_b is not None + self.assertEqual(loaded_b.run_id, "run-new") + self.assertEqual(set(loaded_b.services), {"feature-a Backend", "feature-b Backend"}) + with self.assertRaisesRegex(RuntimeError, "retired"): + repo.save_run( + state=old_state, + contexts=contexts, + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + def test_merged_state_uses_most_recently_activated_run_for_global_metadata(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + def save(project: str, run_id: str, marker: str) -> None: + repo.save_resume_state( + state=RunState( + run_id=run_id, + mode="trees", + services={ + f"{project} Backend": ServiceRecord( + name=f"{project} Backend", + type="backend", + cwd=f"/tmp/{project}", + project=project, + ) + }, + metadata={"repo_scope_id": "repo-123", "activation_marker": marker}, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + save("feature-a", "run-a", "a-original") + save("feature-b", "run-b", "b-newer-sequence") + save("feature-a", "run-a", "a-reactivated") + loaded = repo.load_latest(mode="trees", strict_mode_match=True) + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual(loaded.run_id, "run-a") + self.assertEqual(loaded.metadata["activation_marker"], "a-reactivated") + self.assertEqual(set(loaded.services), {"feature-a Backend", "feature-b Backend"}) + + def test_selection_aware_lookup_does_not_return_disjoint_latest_state(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + context = SimpleNamespace(name="feature-a", root=Path(tmpdir) / "feature-a", ports={}) + repo.save_run( + state=RunState(run_id="run-a", mode="trees", metadata={"repo_scope_id": "repo-123"}), + contexts=[context], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + loaded = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-b"], + ) + + self.assertIsNone(loaded) + + def test_multi_project_lookup_consolidates_disjoint_active_runs_without_losing_history(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + for project_name in ("feature-a", "feature-b"): + repo.save_resume_state( + state=RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=f"/tmp/{project_name}/backend", + project=project_name, + pid=100 if project_name == "feature-a" else 200, + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + merged = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a", "feature-b"], + ) + self.assertIsNotNone(merged) + assert merged is not None + self.assertEqual(set(merged.services), {"feature-a Backend", "feature-b Backend"}) + self.assertEqual(set(merged.metadata["project_names"]), {"feature-a", "feature-b"}) + repo.save_resume_state( - state=run_state, + state=merged, emit=lambda *_args, **_kwargs: None, runtime_map_builder=lambda _state: {"projection": {}}, ) - test_summary = ( - repo.test_results_dir_path("run-1", "run_20260309_100000") / "Main" / "failed_tests_summary.txt" + active = repo.load_all(mode="trees") + self.assertEqual([state.run_id for state in active], ["run-feature-b"]) + self.assertTrue((runtime_root / "runs" / "run-feature-a" / "run_state.json").is_file()) + + def test_load_all_filters_projects_subtracted_from_a_mixed_historical_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + mixed_services = { + f"{project} Backend": ServiceRecord( + name=f"{project} Backend", + type="backend", + cwd=f"/tmp/{project}/backend", + project=project, + ) + for project in ("feature-a", "feature-b") + } + repo.save_resume_state( + state=RunState(run_id="run-mixed", mode="trees", services=mixed_services), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, + ) + repo.save_resume_state( + state=RunState( + run_id="run-new-a", + mode="trees", + services={"feature-a Backend": mixed_services["feature-a Backend"]}, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, ) - test_summary.parent.mkdir(parents=True, exist_ok=True) - test_summary.write_text("No failed tests.\n", encoding="utf-8") - repo.purge(aggressive=False) - self.assertTrue(test_summary.is_file()) + active = {state.run_id: set(state.services) for state in repo.load_all(mode="trees")} - repo.purge(aggressive=True) - self.assertFalse(repo.run_dir_path("run-1").exists()) - self.assertFalse(test_summary.exists()) + self.assertEqual( + active, + { + "run-mixed": {"feature-b Backend"}, + "run-new-a": {"feature-a Backend"}, + }, + ) - def test_tree_diffs_dir_path_scopes_to_run_and_runtime_root(self) -> None: + def test_load_all_uses_sole_index_owner_for_opaque_legacy_service(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + state = RunState( + run_id="run-alpha", + mode="trees", + services={ + "Opaque Runtime": ServiceRecord( + name="Opaque Runtime", + type="worker", + cwd="/unmapped/path", + ) + }, + ) + context = SimpleNamespace(name="Alpha", root=Path(tmpdir) / "alpha", ports={}) + repo.save_run( + state=state, + contexts=[context], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + latest = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["Alpha"], + ) + active = repo.load_all(mode="trees") + + self.assertIsNotNone(latest) + assert latest is not None + self.assertEqual(latest.services["Opaque Runtime"].project, "Alpha") + self.assertEqual(len(active), 1) + self.assertEqual(active[0].services["Opaque Runtime"].project, "Alpha") + + def test_aggregate_and_subset_saves_keep_project_metadata_owned_by_live_projects(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: runtime_dir = Path(tmpdir) / "runtime" runtime_root = runtime_dir / "scope" - legacy_root = runtime_dir / "python-engine" - runtime_root.mkdir(parents=True, exist_ok=True) - legacy_root.mkdir(parents=True, exist_ok=True) repo = RuntimeStateRepository( runtime_root=runtime_root, - runtime_legacy_root=legacy_root, + runtime_legacy_root=runtime_dir / "python-engine", runtime_dir=runtime_dir, runtime_scope_id="repo-123", compat_mode=RuntimeStateRepository.SCOPED_ONLY, ) - self.assertEqual(repo.tree_diffs_dir_path(), runtime_root / "tree-diffs") + for project_name, service_type in (("feature-a", "backend"), ("feature-b", "frontend")): + repo.save_resume_state( + state=RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} {service_type}": ServiceRecord( + name=f"{project_name} {service_type}", + type=service_type, + cwd=f"/tmp/{project_name}/{service_type}", + project=project_name, + ) + }, + metadata={ + "project_roots": { + project_name: f"/tmp/{project_name}", + "stale-project": "/tmp/stale", + }, + "project_pr_links": { + project_name: f"pr-{project_name}", + "stale-project": "pr-stale", + }, + "project_test_summaries": { + project_name: {"failed": 0}, + "stale-project": {"failed": 99}, + }, + "project_test_results_root": f"/tmp/results/{project_name}", + "project_test_results_updated_at": project_name, + "project_action_reports": { + project_name: {"test": {"status": "success"}}, + "stale-project": {"test": {"status": "failed"}}, + }, + "dashboard_project_configured_services": { + project_name: [service_type], + "stale-project": ["worker"], + }, + "dashboard_configured_service_types": [service_type, "worker"], + "dashboard_stopped_services": [ + { + "name": f"{project_name} stopped", + "project": project_name, + "type": service_type, + }, + { + "name": "stale stopped", + "project": "stale-project", + "type": "worker", + }, + ], + "startup_identity": { + "mode": "trees", + "projects": [ + {"name": project_name, "root": f"/tmp/{project_name}"}, + {"name": "stale-project", "root": "/tmp/stale"}, + ], + "fingerprint": project_name, + }, + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + merged = repo.load_latest(mode="trees", strict_mode_match=True) + self.assertIsNotNone(merged) + assert merged is not None + for key in ( + "project_roots", + "project_pr_links", + "project_test_summaries", + "project_action_reports", + "dashboard_project_configured_services", + ): + self.assertEqual(set(merged.metadata[key]), {"feature-a", "feature-b"}) self.assertEqual( - repo.tree_diffs_dir_path("run-1", "review"), - runtime_root / "runs" / "run-1" / "tree-diffs" / "review", + {item["project"] for item in merged.metadata["dashboard_stopped_services"]}, + {"feature-a", "feature-b"}, + ) + self.assertEqual( + merged.metadata["dashboard_configured_service_types"], + ["backend", "frontend"], + ) + self.assertEqual( + {project["name"] for project in merged.metadata["startup_identity"]["projects"]}, + {"feature-a", "feature-b"}, + ) + self.assertNotIn("project_test_results_root", merged.metadata) + self.assertNotIn("project_test_results_updated_at", merged.metadata) + + selected = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + self.assertIsNotNone(selected) + assert selected is not None + for key in ( + "project_roots", + "project_pr_links", + "project_test_summaries", + "project_action_reports", + "dashboard_project_configured_services", + ): + self.assertEqual(set(selected.metadata[key]), {"feature-a"}) + self.assertEqual( + {item["project"] for item in selected.metadata["dashboard_stopped_services"]}, + {"feature-a"}, + ) + self.assertEqual(selected.metadata["dashboard_configured_service_types"], ["backend"]) + + merged.services = { + name: service for name, service in merged.services.items() if service.project == "feature-a" + } + repo.save_resume_state( + state=merged, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + saved_subset = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + self.assertIsNotNone(saved_subset) + assert saved_subset is not None + for key in ( + "project_roots", + "project_pr_links", + "project_test_summaries", + "project_action_reports", + "dashboard_project_configured_services", + ): + self.assertEqual(set(saved_subset.metadata[key]), {"feature-a"}) + self.assertEqual( + {item["project"] for item in saved_subset.metadata["dashboard_stopped_services"]}, + {"feature-a"}, + ) + self.assertIsNone( + repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-b"], + ) + ) + + def test_deactivate_run_preserves_other_active_run_and_historical_artifacts(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + for project_name in ("feature-a", "feature-b"): + repo.save_resume_state( + state=RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=f"/tmp/{project_name}/backend", + project=project_name, + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, + ) + + self.assertTrue(repo.deactivate_run("run-feature-b")) + self.assertFalse(repo.deactivate_run("run-feature-b")) + self.assertIsNone(repo.load_latest(mode="trees", strict_mode_match=True, project_names=["feature-b"])) + remaining = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + self.assertIsNotNone(remaining) + assert remaining is not None + self.assertEqual(remaining.run_id, "run-feature-a") + self.assertEqual(load_state(str(runtime_root / "run_state.json")).run_id, "run-feature-a") + self.assertTrue((runtime_root / "runs" / "run-feature-b" / "run_state.json").is_file()) + + self.assertTrue(repo.deactivate_run("run-feature-a")) + self.assertEqual(repo.load_all(mode="trees"), []) + self.assertFalse((runtime_root / "run_state.json").exists()) + self.assertTrue((runtime_root / "runs" / "run-feature-a" / "run_state.json").is_file()) + + def test_concurrent_saves_publish_coherent_current_snapshot_and_keep_both_runs(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + first_builder_entered = Event() + release_first_builder = Event() + second_builder_entered = Event() + + def save(project_name: str) -> None: + state = RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=f"/tmp/{project_name}/backend", + project=project_name, + ) + }, + ) + + def runtime_map_builder(_state: RunState) -> dict[str, object]: + if project_name == "feature-a": + first_builder_entered.set() + self.assertTrue(release_first_builder.wait(timeout=2)) + else: + second_builder_entered.set() + return {"run_id": state.run_id} + + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=runtime_map_builder, + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + first = executor.submit(save, "feature-a") + self.assertTrue(first_builder_entered.wait(timeout=2)) + second = executor.submit(save, "feature-b") + self.assertTrue(second_builder_entered.wait(timeout=0.1)) + release_first_builder.set() + first.result(timeout=2) + second.result(timeout=2) + + current_state = load_state(str(runtime_root / "run_state.json")) + current_map = json.loads((runtime_root / "runtime_map.json").read_text(encoding="utf-8")) + self.assertEqual(current_state.run_id, current_map["run_id"]) + self.assertEqual( + {state.run_id for state in repo.load_all(mode="trees")}, {"run-feature-a", "run-feature-b"} + ) + self.assertEqual(list(runtime_root.rglob("*.tmp")), []) + + def test_purge_deactivates_index_without_deleting_history_unless_aggressive(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_resume_state( + state=RunState(run_id="run-a", mode="main"), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + historical_state = runtime_root / "runs" / "run-a" / "run_state.json" + + repo.purge(aggressive=False) + + self.assertIsNone(repo.load_latest()) + index_payload = json.loads((runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(index_payload["entries"], []) + self.assertTrue(historical_state.is_file()) + + repo.purge(aggressive=True) + self.assertFalse(historical_state.exists()) + + def test_invalid_run_ids_and_artifact_components_cannot_escape_runtime_root(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + for run_id in (".", "..", "../escaped", "nested/run", "nested\\run", "/absolute"): + with self.subTest(run_id=run_id): + runtime_root = runtime_dir / f"scope-{abs(hash(run_id))}" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id=f"scope-{abs(hash(run_id))}", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + with self.assertRaisesRegex(ValueError, "path component"): + repo.save_resume_state( + state=RunState(run_id=run_id, mode="trees"), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + self.assertFalse(runtime_root.exists()) + + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "safe-scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="safe-scope", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + for invalid_component in ("..", "../escaped", "/absolute", "nested/name"): + with self.subTest(component=invalid_component): + with self.assertRaisesRegex(ValueError, "path component"): + repo.test_results_dir_path("run-a", invalid_component) + with self.assertRaisesRegex(ValueError, "path component"): + repo.tree_diffs_dir_path("run-a", invalid_component) + + def test_missing_or_corrupt_index_is_rebuilt_from_scoped_run_history(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + for project_name in ("feature-a", "feature-b"): + state_path = runtime_root / "runs" / f"run-{project_name}" / "run_state.json" + state_path.parent.mkdir(parents=True, exist_ok=True) + dump_state( + RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=f"/tmp/{project_name}/backend", + project=project_name, + ) + }, + ), + str(state_path), + ) + + recovered_a = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + self.assertIsNotNone(recovered_a) + assert recovered_a is not None + self.assertEqual(recovered_a.run_id, "run-feature-a") + self.assertEqual({state.run_id for state in repo.load_all()}, {"run-feature-a", "run-feature-b"}) + + (runtime_root / "run_index.json").write_text('{"entries":', encoding="utf-8") + recovered_b = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-b"], + ) + self.assertIsNotNone(recovered_b) + assert recovered_b is not None + self.assertEqual(recovered_b.run_id, "run-feature-b") + rebuilt_payload = json.loads((runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(len(rebuilt_payload["entries"]), 2) + + def test_resume_orchestrator_does_not_write_legacy_state_files_directly(self) -> None: + resume_path = REPO_ROOT / "python/envctl_engine/startup/resume_orchestrator.py" + raw = resume_path.read_text(encoding="utf-8") + self.assertNotIn('runtime_legacy_root / "run_state.json"', raw) + self.assertNotIn('runtime_legacy_root / "runtime_map.json"', raw) + + def test_lifecycle_cleanup_orchestrator_does_not_write_legacy_state_files_directly(self) -> None: + cleanup_paths = ( + REPO_ROOT / "python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py", + REPO_ROOT / "python/envctl_engine/runtime/lifecycle_full_cleanup.py", + ) + for cleanup_path in cleanup_paths: + with self.subTest(path=cleanup_path.name): + raw = cleanup_path.read_text(encoding="utf-8") + self.assertNotIn('runtime_legacy_root / "run_state.json"', raw) + self.assertNotIn('runtime_legacy_root / "runtime_map.json"', raw) + + def test_engine_runtime_legacy_compat_helper_is_removed(self) -> None: + runtime_path = REPO_ROOT / "python/envctl_engine/runtime/engine_runtime.py" + raw = runtime_path.read_text(encoding="utf-8") + self.assertNotIn("def _write_legacy_runtime_compat_files(", raw) + + def test_aggressive_purge_removes_runtime_scoped_test_results_with_run(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + runtime_root.mkdir(parents=True, exist_ok=True) + legacy_root.mkdir(parents=True, exist_ok=True) + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + run_state = RunState(run_id="run-1", mode="main", metadata={"repo_scope_id": "repo-123"}) + repo.save_resume_state( + state=run_state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + test_summary = ( + repo.test_results_dir_path("run-1", "run_20260309_100000") / "Main" / "failed_tests_summary.txt" + ) + test_summary.parent.mkdir(parents=True, exist_ok=True) + test_summary.write_text("No failed tests.\n", encoding="utf-8") + + repo.purge(aggressive=False) + self.assertTrue(test_summary.is_file()) + + repo.purge(aggressive=True) + self.assertFalse(repo.run_dir_path("run-1").exists()) + self.assertFalse(test_summary.exists()) + + def test_tree_diffs_dir_path_scopes_to_run_and_runtime_root(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + runtime_root.mkdir(parents=True, exist_ok=True) + legacy_root.mkdir(parents=True, exist_ok=True) + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + self.assertEqual(repo.tree_diffs_dir_path(), (runtime_root / "tree-diffs").resolve()) + self.assertEqual( + repo.tree_diffs_dir_path("run-1", "review"), + (runtime_root / "runs" / "run-1" / "tree-diffs" / "review").resolve(), + ) + + def test_runtime_artifact_writer_updates_active_revision_and_derived_aliases(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + state = RunState(run_id="run-1", mode="main") + repo.save_run( + state=state, + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + accepted = repo.write_runtime_artifact( + run_id="run-1", + artifact_name="runtime_readiness_report.json", + text='{"passed": true}', + ) + + self.assertTrue(accepted) + candidate = repo.run_index.candidates(_state_repository.StateSelector(mode=None, project_names=()))[0] + artifact_paths = ( + candidate.state_path.parent / "runtime_readiness_report.json", + repo.run_dir_path("run-1") / "runtime_readiness_report.json", + runtime_root / "runtime_readiness_report.json", + legacy_root / "runtime_readiness_report.json", + ) + for path in artifact_paths: + with self.subTest(path=path): + self.assertEqual(path.read_text(encoding="utf-8"), '{"passed": true}') + + def test_retired_runtime_artifact_writer_cannot_recreate_files_after_deactivation(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + repo.save_run( + state=RunState(run_id="run-retired", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + self.assertTrue(repo.deactivate_run("run-retired")) + + accepted = repo.write_runtime_artifact( + run_id="run-retired", + artifact_name="runtime_readiness_report.json", + text='{"stale": true}', + ) + + self.assertFalse(accepted) + self.assertFalse((repo.run_dir_path("run-retired") / "runtime_readiness_report.json").exists()) + self.assertFalse((runtime_root / "runtime_readiness_report.json").exists()) + self.assertFalse((legacy_root / "runtime_readiness_report.json").exists()) + + def test_stale_runtime_artifact_writer_cannot_recreate_files_after_aggressive_purge(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + repo.save_run( + state=RunState(run_id="run-purged", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + repo.purge(aggressive=True) + + readiness_accepted = repo.write_runtime_artifact( + run_id="run-purged", + artifact_name="runtime_readiness_report.json", + text='{"stale": true}', + ) + events_accepted = repo.write_runtime_artifact( + run_id="run-purged", + artifact_name="events.jsonl", + text='{"event": "stale"}\n', + ) + + self.assertFalse(readiness_accepted) + self.assertFalse(events_accepted) + self.assertFalse(repo.run_dir_path("run-purged").exists()) + for root in (runtime_root, legacy_root): + with self.subTest(root=root): + self.assertFalse((root / "runtime_readiness_report.json").exists()) + self.assertFalse((root / "events.jsonl").exists()) + + def test_purge_waits_for_inflight_runtime_artifact_write_and_removes_its_output(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_run( + state=RunState(run_id="run-racing", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + artifact_write_started = Event() + release_artifact_write = Event() + original_write_text = repo._write_text + + def blocking_write_text(path: Path, text: str) -> None: + if ( + path.name == "runtime_readiness_report.json" + and "revisions" in path.parts + and not artifact_write_started.is_set() + ): + artifact_write_started.set() + self.assertTrue(release_artifact_write.wait(timeout=2)) + original_write_text(path, text) + + repo._write_text = blocking_write_text # type: ignore[method-assign] + with ThreadPoolExecutor(max_workers=2) as executor: + writer = executor.submit( + repo.write_runtime_artifact, + run_id="run-racing", + artifact_name="runtime_readiness_report.json", + text='{"passed": true}', + ) + self.assertTrue(artifact_write_started.wait(timeout=2)) + purge = executor.submit(repo.purge, aggressive=True) + release_artifact_write.set() + self.assertTrue(writer.result(timeout=2)) + purge.result(timeout=2) + + self.assertFalse(repo.run_dir_path("run-racing").exists()) + self.assertFalse((runtime_root / "runtime_readiness_report.json").exists()) + + def test_unbound_runtime_artifact_is_scoped_and_artifact_name_is_allowlisted(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + accepted = repo.write_runtime_artifact( + run_id=None, + artifact_name="events.jsonl", + text='{"event": "doctor"}\n', + ) + + self.assertTrue(accepted) + self.assertEqual( + (runtime_root / "events.jsonl").read_text(encoding="utf-8"), + '{"event": "doctor"}\n', + ) + self.assertFalse((legacy_root / "events.jsonl").exists()) + with self.assertRaisesRegex(ValueError, "unsupported mutable runtime artifact"): + repo.write_runtime_artifact( + run_id=None, + artifact_name="run_state.json", + text="{}", + ) + with self.assertRaisesRegex(ValueError, "path component"): + repo.write_runtime_artifact( + run_id=None, + artifact_name="../events.jsonl", + text="", + ) + + def test_unbound_runtime_artifact_never_attaches_to_an_active_run(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_run( + state=RunState(run_id="run-active", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[{"event": "run-bound"}], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + candidate = repo.run_index.candidates(_state_repository.StateSelector(mode=None, project_names=()))[0] + + accepted = repo.write_runtime_artifact( + run_id=None, + artifact_name="events.jsonl", + text='{"event": "scope-diagnostic"}\n', + ) + + self.assertTrue(accepted) + self.assertEqual( + (runtime_root / "events.jsonl").read_text(encoding="utf-8"), + json.dumps({"event": "run-bound"}, sort_keys=True) + "\n", + ) + self.assertEqual( + (runtime_root / "diagnostics" / "events.jsonl").read_text(encoding="utf-8"), + '{"event": "scope-diagnostic"}\n', + ) + expected_run_text = json.dumps({"event": "run-bound"}, sort_keys=True) + "\n" + self.assertEqual( + (repo.run_dir_path("run-active") / "events.jsonl").read_text(encoding="utf-8"), + expected_run_text, + ) + self.assertEqual( + (candidate.state_path.parent / "events.jsonl").read_text(encoding="utf-8"), + expected_run_text, + ) + + def test_retired_overlapping_run_artifact_writer_is_fenced_from_current_alias(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + for run_id in ("run-old", "run-new"): + repo.save_run( + state=RunState(run_id=run_id, mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[{"event": run_id}], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + accepted = repo.write_runtime_artifact( + run_id="run-old", + artifact_name="events.jsonl", + text='{"event": "late-old-writer"}\n', + ) + + self.assertFalse(accepted) + self.assertEqual( + (repo.run_dir_path("run-old") / "events.jsonl").read_text(encoding="utf-8"), + json.dumps({"event": "run-old"}, sort_keys=True) + "\n", + ) + expected_current = json.dumps({"event": "run-new"}, sort_keys=True) + "\n" + self.assertEqual((runtime_root / "events.jsonl").read_text(encoding="utf-8"), expected_current) + self.assertEqual((legacy_root / "events.jsonl").read_text(encoding="utf-8"), expected_current) + + def test_corrupt_authoritative_revision_does_not_reactivate_retired_project_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + for run_id, pid in (("run-old", 101), ("run-new", 202)): + repo.save_resume_state( + state=RunState( + run_id=run_id, + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/tmp/feature-a/backend", + project="feature-a", + pid=pid, + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, + ) + + candidates = repo.run_index.candidates(_state_repository.StateSelector(mode="trees", project_names=())) + new_candidate = next(candidate for candidate in candidates if candidate.run_id == "run-new") + new_candidate.state_path.write_text('{"broken":', encoding="utf-8") + + loaded = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + + self.assertIsNone(loaded) + self.assertFalse((runtime_root / "run_state.json").exists()) + self.assertFalse((runtime_root / ".last_state").exists()) + + def test_generic_lookup_skips_corrupt_newest_mode_and_returns_newest_valid_mode(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_resume_state( + state=RunState( + run_id="run-main", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + repo.save_resume_state( + state=RunState( + run_id="run-trees", + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/tmp/feature-a/backend", + project="feature-a", + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + corrupt_candidate = repo.run_index.candidates( + _state_repository.StateSelector(mode="trees", project_names=()) + )[0] + corrupt_candidate.state_path.write_text('{"broken":', encoding="utf-8") + + loaded = repo.load_latest() + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual((loaded.run_id, loaded.mode), ("run-main", "main")) + self.assertEqual(load_state(str(runtime_root / "run_state.json")).run_id, "run-main") + + def test_non_strict_project_lookup_filters_projects_before_newest_mode(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_resume_state( + state=RunState( + run_id="run-tree", + mode="trees", + services={ + "FeatureA Backend": ServiceRecord( + name="FeatureA Backend", + type="backend", + cwd="/tmp/FeatureA/backend", + project="FeatureA", + ) + }, + metadata={"project_names": ["FeatureA"]}, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + repo.save_resume_state( + state=RunState( + run_id="run-main", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + ) + }, + metadata={"project_names": ["Main"]}, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + loaded = repo.load_latest( + mode="main", + strict_mode_match=False, + project_names=["FeatureA"], + ) + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual((loaded.run_id, loaded.mode), ("run-tree", "trees")) + + def test_resuming_older_mode_reactivates_it_without_changing_project_ownership_precedence(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + main_state = RunState( + run_id="run-main", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + ) + }, + ) + tree_state = RunState( + run_id="run-tree", + mode="trees", + services={ + "FeatureA Backend": ServiceRecord( + name="FeatureA Backend", + type="backend", + cwd="/tmp/FeatureA/backend", + project="FeatureA", + ) + }, + ) + for state in (main_state, tree_state): + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + self.assertEqual(repo.load_latest().run_id, "run-tree") # type: ignore[union-attr] + + repo.save_resume_state( + state=main_state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + current = repo.load_latest() + self.assertIsNotNone(current) + assert current is not None + self.assertEqual((current.run_id, current.mode), ("run-main", "main")) + tree = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["FeatureA"], + ) + self.assertIsNotNone(tree) + assert tree is not None + self.assertEqual(tree.run_id, "run-tree") + + def test_corrupt_primary_and_backup_after_deactivation_fails_closed_without_resurrection(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_resume_state( + state=RunState( + run_id="retired-run", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/backend", + project="Main", + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + self.assertTrue(repo.deactivate_run("retired-run")) + historical_state = repo.run_dir_path("retired-run") / "run_state.json" + self.assertTrue(historical_state.is_file()) + repo.run_index.index_path.write_text('{"broken":', encoding="utf-8") + repo.run_index.backup_path.write_text('{"broken":', encoding="utf-8") + + with self.assertRaisesRegex(RuntimeError, "both the primary and backup"): + repo.load_latest() + + self.assertTrue(historical_state.is_file()) + self.assertFalse((runtime_root / "run_state.json").exists()) + + def test_failed_legacy_registry_publish_cleans_staged_revision_directories(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_state = runtime_root / "runs" / "legacy-run" / "run_state.json" + legacy_state.parent.mkdir(parents=True, exist_ok=True) + dump_state( + RunState( + run_id="legacy-run", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/backend", + project="Main", + ) + }, + ), + str(legacy_state), + ) + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + with patch.object(repo.run_index, "replace_all", side_effect=OSError("registry unavailable")): + with self.assertRaisesRegex(OSError, "registry unavailable"): + repo.load_latest() + + revisions = legacy_state.parent / "revisions" + self.assertEqual(list(revisions.iterdir()) if revisions.is_dir() else [], []) + self.assertTrue(legacy_state.is_file()) + self.assertFalse((runtime_root / "run_registry.json").exists()) + + def test_read_only_and_scoped_purges_never_mutate_legacy_runtime(self) -> None: + for compat_mode in ( + RuntimeStateRepository.COMPAT_READ_ONLY, + RuntimeStateRepository.SCOPED_ONLY, + ): + with self.subTest(compat_mode=compat_mode), tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + legacy_run = legacy_root / "runs" / "legacy-run" + legacy_run.mkdir(parents=True, exist_ok=True) + sentinels = { + legacy_root / "run_state.json": "legacy-state", + legacy_root / "events.jsonl": "legacy-event", + legacy_root / ".last_state": "legacy-pointer", + legacy_run / "run_state.json": "legacy-history", + legacy_root / ".events.jsonl.stale.tmp": "legacy-temp", + } + for path, text in sentinels.items(): + path.write_text(text, encoding="utf-8") + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=compat_mode, + ) + + repo.purge(aggressive=True) + + for path, expected in sentinels.items(): + self.assertEqual(path.read_text(encoding="utf-8"), expected) + + def test_first_operation_read_only_purge_cannot_reimport_legacy_after_index_loss(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + legacy_root.mkdir(parents=True) + legacy_state = legacy_root / "run_state.json" + dump_state(RunState(run_id="legacy-run", mode="main"), str(legacy_state)) + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_ONLY, + ) + + repo.purge(aggressive=False) + self.assertTrue(legacy_state.is_file()) + self.assertTrue((runtime_root / "run_registry.json").is_file()) + repo.run_index.index_path.unlink() + repo.run_index.backup_path.unlink() + + with self.assertRaisesRegex(RuntimeError, "both the primary and backup"): + repo.load_latest() + + self.assertTrue(legacy_state.is_file()) + self.assertFalse((runtime_root / "run_state.json").exists()) + + def test_runtime_artifact_directories_reject_symlink_redirection(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + external = Path(tmpdir) / "external" + external.mkdir() + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + runs_root = repo.run_dir_path(None) + runs_root.mkdir(parents=True) + run_link = runs_root / "run-a" + run_link.symlink_to(external, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "symlink"): + repo.run_dir_path("run-a") + run_link.unlink() + + run_dir = repo.run_dir_path("run-a") + run_dir.mkdir() + for name, resolver in ( + ("test-results", lambda: repo.test_results_dir_path("run-a")), + ("tree-diffs", lambda: repo.tree_diffs_dir_path("run-a")), + ): + with self.subTest(name=name): + link = run_dir / name + link.symlink_to(external, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "symlink"): + resolver() + link.unlink() + + root_tree_diffs = runtime_root / "tree-diffs" + root_tree_diffs.symlink_to(external, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "symlink"): + repo.tree_diffs_dir_path() + self.assertEqual(list(external.iterdir()), []) + + def test_runtime_root_retarget_is_rejected_before_lock_or_purge_writes(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + runtime_root.mkdir(parents=True) + legacy_root.mkdir() + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + external = Path(tmpdir) / "external" + external.mkdir() + sentinel = external / "run_state.json" + sentinel.write_text("outside-sentinel", encoding="utf-8") + runtime_root.rmdir() + runtime_root.symlink_to(external, target_is_directory=True) + + with self.assertRaisesRegex(RuntimeError, "symlink"): + repo.purge(aggressive=True) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "outside-sentinel") + self.assertEqual(list(external.iterdir()), [sentinel]) + + def test_legacy_root_retarget_is_rejected_before_compatibility_writes(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + runtime_root.mkdir(parents=True) + legacy_root.mkdir() + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + external = Path(tmpdir) / "external" + external.mkdir() + sentinel = external / "run_state.json" + sentinel.write_text("outside-sentinel", encoding="utf-8") + legacy_root.rmdir() + legacy_root.symlink_to(external, target_is_directory=True) + + with self.assertRaisesRegex(RuntimeError, "symlink"): + repo.save_resume_state( + state=RunState(run_id="run-a", mode="main"), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "outside-sentinel") + self.assertEqual(list(external.iterdir()), [sentinel]) + + def test_runtime_update_carries_forward_ports_errors_events_and_readiness(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + state = RunState( + run_id="run-a", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/backend", + project="Main", + pid=123, + ) + }, + ) + repo.save_run( + state=state, + contexts=[self._context(Path(tmpdir))], + errors=["diagnostic"], + events=[{"event": "initial"}], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + self.assertTrue( + repo.write_runtime_artifact( + run_id="run-a", + artifact_name="runtime_readiness_report.json", + text='{"passed": true}', + ) + ) + artifact_names = ( + "ports_manifest.json", + "error_report.json", + "events.jsonl", + "runtime_readiness_report.json", + ) + expected = {name: (runtime_root / name).read_text(encoding="utf-8") for name in artifact_names} + + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {"Main": {}}}, + ) + + candidate = repo.run_index.candidates(_state_repository.StateSelector(mode="main", project_names=()))[0] + for artifact_name, expected_text in expected.items(): + for root in (runtime_root, legacy_root, candidate.state_path.parent): + with self.subTest(artifact_name=artifact_name, root=root): + actual_text = (root / artifact_name).read_text(encoding="utf-8") + if artifact_name == "error_report.json": + payload = json.loads(actual_text) + self.assertEqual(payload["errors"], ["diagnostic"]) + self.assertEqual(payload["source_run_ids"], ["run-a"]) + else: + self.assertEqual(actual_text, expected_text) + + def test_partial_tree_stop_rebuilds_pointer_inventory_without_resurrecting_removed_project(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + state = RunState( + run_id="run-ab", + mode="trees", + services={ + project: ServiceRecord( + name=project, + type="backend", + cwd=f"/tmp/{project}/backend", + project=project, + ) + for project in ("FeatureA", "FeatureB") + }, + ) + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + pointer_a_name = repo._tree_pointer_name("FeatureA") + pointer_b_name = repo._tree_pointer_name("FeatureB") + assert pointer_a_name is not None and pointer_b_name is not None + old_target = Path((runtime_root / pointer_a_name).read_text(encoding="utf-8").strip()) + self.assertTrue(old_target.is_file()) + + state.services.pop("FeatureA") + repo.save_selected_stop_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + for root in (runtime_root, legacy_root): + self.assertFalse((root / pointer_a_name).exists()) + self.assertTrue((root / pointer_b_name).is_file()) + self.assertTrue(old_target.is_file()) + self.assertIsNone( + repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["FeatureA"], + ) + ) + remaining = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["FeatureB"], + ) + self.assertIsNotNone(remaining) + assert remaining is not None + self.assertEqual(set(remaining.services), {"FeatureB"}) + ports_payload = json.loads((runtime_root / "ports_manifest.json").read_text(encoding="utf-8")) + self.assertEqual( + [project["project"] for project in ports_payload["projects"]], + ["FeatureB"], + ) + + def test_selected_stop_can_preserve_authoritative_metadata_only_tree_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + state = RunState( + run_id="run-metadata-ab", + mode="trees", + metadata={ + "project_names": ["FeatureA", "FeatureB"], + "project_roots": { + "FeatureA": "/tmp/FeatureA", + "FeatureB": "/tmp/FeatureB", + }, + }, + ) + repo.save_selected_stop_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + authoritative_project_names=["FeatureA", "FeatureB"], + ) + + state.metadata["project_names"] = ["FeatureB"] + state.metadata["project_roots"] = {"FeatureB": "/tmp/FeatureB"} + repo.save_selected_stop_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + authoritative_project_names=["FeatureB"], + ) + + candidates = repo.run_index.candidates(StateSelector(mode="trees", project_names=())) + self.assertEqual( + [(candidate.run_id, candidate.project_names) for candidate in candidates], + [("run-metadata-ab", ("featureb",))], + ) + self.assertIsNone( + repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["FeatureA"], + ) + ) + remaining = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["FeatureB"], + ) + self.assertIsNotNone(remaining) + assert remaining is not None + self.assertEqual(remaining.services, {}) + self.assertEqual(remaining.requirements, {}) + self.assertEqual(remaining.metadata["project_names"], ["FeatureB"]) + self.assertEqual(remaining.metadata["project_roots"], {"FeatureB": "/tmp/FeatureB"}) + current = json.loads(repo.run_state_path().read_text(encoding="utf-8")) + self.assertEqual(current["metadata"]["project_names"], ["FeatureB"]) + pointer_a = repo._tree_pointer_name("FeatureA") + pointer_b = repo._tree_pointer_name("FeatureB") + assert pointer_a is not None and pointer_b is not None + self.assertFalse((runtime_root / pointer_a).exists()) + self.assertTrue((runtime_root / pointer_b).exists()) + + def test_lossy_tree_names_and_mixed_case_get_distinct_stable_pointers(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + projects = ("Feature/API v2", "Feature_API_v2") + for index, project in enumerate(projects, start=1): + repo.save_run( + state=RunState(run_id=f"run-{index}", mode="trees"), + contexts=[SimpleNamespace(name=project, root=Path(tmpdir) / f"tree-{index}", ports={})], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + pointer_names = [repo._tree_pointer_name(project) for project in projects] + self.assertTrue(all(pointer_names)) + self.assertEqual(len(set(pointer_names)), 2) + pointed_runs = { + load_state((runtime_root / str(pointer_name)).read_text(encoding="utf-8").strip()).run_id + for pointer_name in pointer_names + } + self.assertEqual(pointed_runs, {"run-1", "run-2"}) + self.assertEqual( + {pointer.name for pointer in runtime_root.glob(".last_state.trees.*")}, + set(pointer_names), + ) + + def test_post_commit_per_run_alias_failure_is_repaired_on_next_read(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + run_alias = repo.run_dir_path("run-a") / "run_state.json" + original_write_text = repo._write_text + failed = False + + def fail_first_run_alias(path: Path, text: str) -> None: + nonlocal failed + if path == run_alias and not failed: + failed = True + raise OSError("alias unavailable") + original_write_text(path, text) + + repo._write_text = fail_first_run_alias # type: ignore[method-assign] + repo.save_run( + state=RunState(run_id="run-a", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + self.assertTrue(failed) + self.assertFalse(run_alias.exists()) + repo._write_text = original_write_text # type: ignore[method-assign] + + loaded = repo.load_latest() + + self.assertIsNotNone(loaded) + self.assertTrue(run_alias.is_file()) + self.assertEqual(load_state(str(run_alias)).run_id, "run-a") + + def test_non_os_post_commit_failure_still_signals_durable_authority(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + committed: list[str] = [] + original_publish = repo._publish_revision_aliases + repo._publish_revision_aliases = ( # type: ignore[method-assign] + lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("alias logic failed")) + ) + + with self.assertRaisesRegex(RuntimeError, "alias logic failed"): + repo.save_run( + state=RunState(run_id="run-committed", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + on_commit=lambda: committed.append("run-committed"), + ) + + self.assertEqual(committed, ["run-committed"]) + repo._publish_revision_aliases = original_publish # type: ignore[method-assign] + loaded = repo.load_latest(mode="main", strict_mode_match=True) + self.assertIsNotNone(loaded) + self.assertEqual(loaded.run_id, "run-committed") + + def test_registry_publish_then_interrupt_retains_revision_and_signals_commit(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + committed: list[str] = [] + repo.run_index.initialize_empty() + original_atomic_write = _run_index.atomic_write_text + published = False + + def publish_backup_then_interrupt(path: Path, text: str) -> None: + nonlocal published + original_atomic_write(path, text) + if path == repo.run_index.backup_path and not published: + published = True + raise KeyboardInterrupt + + with ( + patch( + "envctl_engine.state.run_index.atomic_write_text", + side_effect=publish_backup_then_interrupt, + ), + self.assertRaises(KeyboardInterrupt), + ): + repo.save_run( + state=RunState(run_id="run-interrupt", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + on_commit=lambda: committed.append("run-interrupt"), + ) + + self.assertTrue(published) + self.assertEqual(committed, ["run-interrupt"]) + candidates = repo.run_index.candidates(StateSelector(mode="main", project_names=())) + self.assertEqual(len(candidates), 1) + self.assertEqual(candidates[0].run_id, "run-interrupt") + self.assertTrue(candidates[0].state_path.is_file()) + loaded = repo.load_latest(mode="main", strict_mode_match=True) + self.assertIsNotNone(loaded) + self.assertEqual(loaded.run_id, "run-interrupt") + + def test_registry_interrupt_before_publish_removes_uncommitted_revision(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + committed: list[str] = [] + repo.run_index.initialize_empty() + + with ( + patch( + "envctl_engine.state.run_index.atomic_write_text", + side_effect=KeyboardInterrupt, + ), + self.assertRaises(KeyboardInterrupt), + ): + repo.save_run( + state=RunState(run_id="run-pre-publish", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + on_commit=lambda: committed.append("run-pre-publish"), + ) + + self.assertEqual(committed, []) + self.assertEqual( + repo.run_index.candidates(StateSelector(mode="main", project_names=())), + [], ) + revisions = repo.run_dir_path("run-pre-publish") / "revisions" + self.assertEqual(list(revisions.glob("*")), []) if __name__ == "__main__": diff --git a/tests/python/state/test_state_repository_hardening.py b/tests/python/state/test_state_repository_hardening.py new file mode 100644 index 000000000..7a84f305a --- /dev/null +++ b/tests/python/state/test_state_repository_hardening.py @@ -0,0 +1,638 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from envctl_engine.state import dump_state, load_state +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord +from envctl_engine.state.persistence import fsync_directory +from envctl_engine.state.repository import RuntimeStateRepository +from envctl_engine.state.run_index import StateSelector + + +class StateRepositoryHardeningTests(unittest.TestCase): + @staticmethod + def _repository( + root: Path, + *, + scope: str = "scope-a", + compat_mode: str = RuntimeStateRepository.SCOPED_ONLY, + ) -> RuntimeStateRepository: + runtime_dir = root / "runtime" + return RuntimeStateRepository( + runtime_root=runtime_dir / scope, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id=scope, + compat_mode=compat_mode, + ) + + @staticmethod + def _state( + run_id: str, + project: str, + *, + port: int = 8000, + metadata: dict[str, object] | None = None, + ) -> RunState: + return RunState( + run_id=run_id, + mode="main" if project.casefold() == "main" else "trees", + services={ + f"{project} Backend": ServiceRecord( + name=f"{project} Backend", + type="backend", + cwd=f"/tmp/{project}/backend", + project=project, + requested_port=port, + actual_port=port, + status="running", + ) + }, + metadata=dict(metadata or {}), + ) + + @staticmethod + def _context(root: Path, project: str, port: int) -> object: + return SimpleNamespace( + name=project, + root=root / project, + ports={ + "backend": SimpleNamespace( + requested=port, + assigned=port, + final=port, + source="requested", + retries=0, + ) + }, + ) + + @staticmethod + def _save_resume(repo: RuntimeStateRepository, state: RunState) -> None: + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda saved: {"run_id": saved.run_id}, + ) + + @staticmethod + def _save_run( + repo: RuntimeStateRepository, + state: RunState, + *, + context: object, + errors: list[str] | None = None, + ) -> None: + repo.save_run( + state=state, + contexts=[context], + errors=list(errors or []), + events=[{"event": state.run_id}], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda saved: {"run_id": saved.run_id}, + ) + + def test_clean_save_scavenges_crash_temps_for_rewritten_alias_targets(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + state = self._state("run-a", "Main") + self._save_resume(repo, state) + run_dir = repo.run_dir_path("run-a") + stale_root_alias = repo.runtime_root / ".run_state.json.abcdefgh.tmp" + stale_run_alias = run_dir / ".run_state.json.ijklmnop.tmp" + stale_index = repo.runtime_root / ".run_index.json.qrstuvwx.tmp" + for path in (stale_root_alias, stale_run_alias, stale_index): + path.write_text("crash debris", encoding="utf-8") + + self._save_resume(repo, state) + + self.assertFalse(stale_root_alias.exists()) + self.assertFalse(stale_run_alias.exists()) + self.assertFalse(stale_index.exists()) + + def test_revision_retention_keeps_indexed_revision_and_one_grace_revision(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + revisions_dir = repo.run_dir_path("run-a") / "revisions" + external = root / "external" + external.mkdir() + + with patch( + "envctl_engine.state.repository.fsync_directory", + wraps=fsync_directory, + ) as synced_directory: + for port in range(8100, 8106): + self._save_resume(repo, self._state("run-a", "feature-a", port=port)) + real_revisions = [ + path for path in revisions_dir.iterdir() if path.is_dir() and not path.is_symlink() + ] + self.assertLessEqual(len(real_revisions), 2) + + candidate = repo.run_index.candidates(StateSelector(mode="trees", project_names=()))[0] + real_revisions = [path for path in revisions_dir.iterdir() if path.is_dir() and not path.is_symlink()] + self.assertEqual(len(real_revisions), 2) + self.assertIn(candidate.state_path.parent, real_revisions) + self.assertTrue(any(call.args == (revisions_dir,) for call in synced_directory.call_args_list)) + + external_sentinel = external / "sentinel" + external_sentinel.write_text("safe", encoding="utf-8") + revision_link = revisions_dir / "foreign-link" + revision_link.symlink_to(external, target_is_directory=True) + self._save_resume(repo, self._state("run-a", "feature-a", port=9000)) + self.assertTrue(revision_link.is_symlink()) + self.assertEqual(external_sentinel.read_text(encoding="utf-8"), "safe") + + def test_revision_pruning_is_deferred_until_failed_promotion_is_repaired(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + for port in (8100, 8101): + self._save_resume(repo, self._state("run-a", "feature-a", port=port)) + revisions_dir = repo.run_dir_path("run-a") / "revisions" + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 2) + + with patch.object( + repo, + "_promote_latest_active_state", + side_effect=OSError("alias promotion unavailable"), + ): + self._save_resume(repo, self._state("run-a", "feature-a", port=8102)) + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 3) + + loaded = repo.load_latest(mode="trees", strict_mode_match=True) + self.assertIsNotNone(loaded) + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 2) + + def test_supersession_prunes_failed_promotion_backlog_for_retired_run(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + for port in (8100, 8101): + self._save_resume(repo, self._state("run-old", "feature-a", port=port)) + revisions_dir = repo.run_dir_path("run-old") / "revisions" + with patch.object( + repo, + "_promote_latest_active_state", + side_effect=OSError("alias promotion unavailable"), + ): + for port in (8102, 8103, 8104): + self._save_resume(repo, self._state("run-old", "feature-a", port=port)) + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 5) + + replacement = self._state( + "run-new", + "feature-a", + port=8200, + metadata={"state_source_run_ids": ["run-old"]}, + ) + self._save_resume(repo, replacement) + + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 2) + payload = json.loads(repo.run_index.index_path.read_text(encoding="utf-8")) + self.assertEqual([entry["run_id"] for entry in payload["entries"]], ["run-new"]) + self.assertIn("run-old", payload["retired_run_ids"]) + + def test_revision_cleanup_io_failures_do_not_lose_state_and_are_retried(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + for port in (8100, 8101): + self._save_resume(repo, self._state("run-old", "feature-a", port=port)) + revisions_dir = repo.run_dir_path("run-old") / "revisions" + with patch.object( + repo, + "_promote_latest_active_state", + side_effect=OSError("alias promotion unavailable"), + ): + for port in (8102, 8103, 8104): + self._save_resume(repo, self._state("run-old", "feature-a", port=port)) + + replacement = self._state( + "run-new", + "feature-a", + port=8200, + metadata={"state_source_run_ids": ["run-old"]}, + ) + with patch( + "envctl_engine.state.repository.shutil.rmtree", + side_effect=OSError("cleanup unavailable"), + ): + self._save_resume(repo, replacement) + + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 5) + cleanup_dir = repo.runtime_root / repo._REVISION_CLEANUP_DIR_NAME + self.assertEqual(len(list(cleanup_dir.glob("*.json"))), 1) + for index in range(100): + clean_history = repo.run_dir_path(f"inactive-{index}") / "revisions" / "only-revision" + clean_history.mkdir(parents=True) + original_prune = repo._prune_run_revisions_unlocked + inspected_run_ids: list[str] = [] + + def recording_prune(candidate: object) -> None: + inspected_run_ids.append(str(getattr(candidate, "run_id", ""))) + original_prune(candidate) # type: ignore[arg-type] + + with patch.object( + repo, + "_prune_run_revisions_unlocked", + side_effect=recording_prune, + ): + loaded = repo.load_latest(mode="trees", strict_mode_match=True) + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual(loaded.run_id, "run-new") + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 2) + self.assertEqual(set(inspected_run_ids), {"run-new", "run-old"}) + self.assertFalse(cleanup_dir.exists()) + + with patch( + "envctl_engine.state.repository.fsync_directory", + side_effect=OSError("directory fsync failed"), + ): + for port in (8201, 8202): + self._save_resume(repo, self._state("run-new", "feature-a", port=port)) + loaded = repo.load_latest(mode="trees", strict_mode_match=True) + self.assertIsNotNone(loaded) + self.assertLessEqual( + len([path for path in (repo.run_dir_path("run-new") / "revisions").iterdir() if path.is_dir()]), + 2, + ) + + def test_superseded_cleanup_is_queued_when_alias_promotion_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + with patch.object( + repo, + "_promote_latest_active_state", + side_effect=OSError("alias promotion unavailable"), + ): + for port in range(8100, 8105): + self._save_resume(repo, self._state("run-old", "feature-a", port=port)) + self._save_resume( + repo, + self._state( + "run-new", + "feature-a", + port=8200, + metadata={"state_source_run_ids": ["run-old"]}, + ), + ) + + old_revisions = repo.run_dir_path("run-old") / "revisions" + cleanup_dir = repo.runtime_root / repo._REVISION_CLEANUP_DIR_NAME + self.assertEqual( + len([path for path in old_revisions.iterdir() if path.is_dir()]), + 5, + ) + self.assertEqual(len(list(cleanup_dir.glob("*.json"))), 1) + + loaded = repo.load_latest(mode="trees", strict_mode_match=True) + + self.assertIsNotNone(loaded) + self.assertLessEqual( + len([path for path in old_revisions.iterdir() if path.is_dir()]), + 2, + ) + self.assertFalse(cleanup_dir.exists()) + + def test_current_ancillary_alias_signatures_detect_tampering_and_source_updates(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository( + root, + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + self._save_run( + repo, + self._state("run-a", "Main"), + context=self._context(root, "Main", 8000), + errors=["expected error"], + ) + self.assertTrue( + repo.write_runtime_artifact( + run_id="run-a", + artifact_name="runtime_readiness_report.json", + text='{"passed": true}', + ) + ) + candidate = repo.run_index.candidates(StateSelector(mode="main", project_names=()))[0] + scoped_ports = repo.ports_manifest_path() + expected_ports = scoped_ports.read_text(encoding="utf-8") + scoped_ports.write_text("x" * len(expected_ports), encoding="utf-8") + repo.load_latest() + self.assertEqual(scoped_ports.read_text(encoding="utf-8"), expected_ports) + + legacy_error = repo.runtime_legacy_root / "error_report.json" + expected_error = legacy_error.read_text(encoding="utf-8") + legacy_error.write_text("x" * len(expected_error), encoding="utf-8") + repo.load_latest() + repaired_error = json.loads(legacy_error.read_text(encoding="utf-8")) + self.assertEqual(repaired_error["errors"], ["expected error"]) + self.assertEqual( + legacy_error.read_text(encoding="utf-8"), + repo.error_report_path().read_text(encoding="utf-8"), + ) + + source_events = candidate.state_path.parent / "events.jsonl" + updated_events = '{"event": "late-source-update"}\n' + source_events.write_text(updated_events, encoding="utf-8") + repo.load_latest() + self.assertEqual( + (repo.runtime_root / "events.jsonl").read_text(encoding="utf-8"), + updated_events, + ) + self.assertEqual( + (repo.runtime_legacy_root / "events.jsonl").read_text(encoding="utf-8"), + updated_events, + ) + + def test_per_run_alias_manifest_uses_signatures_and_repairs_tampering(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + self._save_run( + repo, + self._state("run-a", "Main"), + context=self._context(root, "Main", 8000), + ) + self.assertTrue( + repo.write_runtime_artifact( + run_id="run-a", + artifact_name="runtime_readiness_report.json", + text='{"passed": true}', + ) + ) + repo.load_latest() + candidate = repo.run_index.candidates(StateSelector(mode="main", project_names=()))[0] + + observed_reads: list[Path] = [] + original_read_text = Path.read_text + + def recording_read_text(path: Path, *args: object, **kwargs: object) -> str: + observed_reads.append(path) + return original_read_text(path, *args, **kwargs) + + indexed = repo._load_valid_index_states( + [candidate], + allowed_root=str(repo.runtime_dir), + ) + observed_reads.clear() + with patch.object(Path, "read_text", recording_read_text): + repo._reconcile_run_aliases_unlocked(indexed) + + source_dir = candidate.state_path.parent + run_dir = repo.run_dir_path("run-a") + compared_artifacts = {source_dir / artifact_name for artifact_name in repo._RUN_ALIAS_ARTIFACT_NAMES} | { + run_dir / artifact_name for artifact_name in repo._RUN_ALIAS_ARTIFACT_NAMES + } + self.assertTrue(compared_artifacts.isdisjoint(observed_reads)) + + tampered_alias = run_dir / "events.jsonl" + expected_events = (source_dir / "events.jsonl").read_text(encoding="utf-8") + tampered_alias.write_text("x" * len(expected_events), encoding="utf-8") + repo.load_latest() + self.assertEqual(tampered_alias.read_text(encoding="utf-8"), expected_events) + + def test_deactivate_does_not_retire_unknown_run_ids(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + repo = self._repository(Path(tmpdir)) + self._save_resume(repo, self._state("run-active", "Main")) + + self.assertFalse(repo.deactivate_run("never-active")) + self.assertTrue(repo.deactivate_runs(["run-active", "also-never-active"])) + + payload = json.loads(repo.run_index.index_path.read_text(encoding="utf-8")) + self.assertEqual(payload["retired_run_ids"], ["run-active"]) + + def test_legacy_migration_preserves_failed_health_state_and_all_artifacts(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime_dir = root / "runtime" + legacy_root = runtime_dir / "python-engine" + legacy_root.mkdir(parents=True) + failed_state = RunState( + run_id="legacy-failed", + mode="main", + requirements={ + "Main": RequirementsResult( + project="Main", + health="degraded", + failures=["redis unavailable"], + components={ + "redis": { + "enabled": True, + "success": False, + "runtime_status": "unreachable", + } + }, + ) + }, + metadata={"failed": True}, + ) + dump_state(failed_state, str(legacy_root / "run_state.json")) + artifacts = { + "runtime_map.json": '{"legacy": true}', + "ports_manifest.json": '{"projects": [{"project": "Main", "ports": {}}]}', + "error_report.json": '{"errors": ["redis unavailable"]}', + "events.jsonl": '{"event": "failed"}\n', + "runtime_readiness_report.json": '{"passed": false}', + } + for artifact_name, text in artifacts.items(): + (legacy_root / artifact_name).write_text(text, encoding="utf-8") + + repo = self._repository( + root, + compat_mode=RuntimeStateRepository.COMPAT_READ_ONLY, + ) + loaded = repo.load_latest(mode="main", strict_mode_match=True) + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertTrue(loaded.metadata["failed"]) + self.assertEqual(loaded.requirements["Main"].health, "degraded") + self.assertEqual( + loaded.requirements["Main"].component("redis")["runtime_status"], + "unreachable", + ) + candidate = repo.run_index.candidates(StateSelector(mode="main", project_names=()))[0] + for artifact_name, expected in artifacts.items(): + self.assertEqual( + (candidate.state_path.parent / artifact_name).read_text(encoding="utf-8"), + expected, + ) + + def test_unbound_diagnostics_do_not_replace_active_aliases_and_purge_cleans_them(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository( + root, + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + runtime_root = repo.runtime_root + legacy_root = repo.runtime_legacy_root + no_run_event = '{"event": "no-run"}\n' + self.assertTrue( + repo.write_runtime_artifact( + run_id=None, + artifact_name="events.jsonl", + text=no_run_event, + ) + ) + self.assertEqual((runtime_root / "events.jsonl").read_text(encoding="utf-8"), no_run_event) + self.assertEqual((legacy_root / "events.jsonl").read_text(encoding="utf-8"), no_run_event) + + self._save_run( + repo, + self._state("run-a", "Main"), + context=self._context(root, "Main", 8000), + ) + active_event = (runtime_root / "events.jsonl").read_text(encoding="utf-8") + diagnostic_event = '{"event": "doctor"}\n' + self.assertTrue( + repo.write_runtime_artifact( + run_id=None, + artifact_name="events.jsonl", + text=diagnostic_event, + ) + ) + self.assertEqual((runtime_root / "events.jsonl").read_text(encoding="utf-8"), active_event) + self.assertEqual((legacy_root / "events.jsonl").read_text(encoding="utf-8"), active_event) + self.assertEqual( + (runtime_root / "diagnostics" / "events.jsonl").read_text(encoding="utf-8"), + diagnostic_event, + ) + + repo.purge(aggressive=False) + self.assertFalse((runtime_root / "diagnostics").exists()) + + def test_shared_legacy_pointers_are_only_removed_by_their_own_scope(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo_a = self._repository( + root, + scope="scope-a", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + repo_b = self._repository( + root, + scope="scope-b", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + self._save_run( + repo_a, + self._state("run-a", "feature-a"), + context=self._context(root, "feature-a", 8100), + ) + self._save_run( + repo_b, + self._state("run-b", "feature-b"), + context=self._context(root, "feature-b", 8200), + ) + legacy_root = repo_a.runtime_legacy_root + pointer_b_name = repo_b._tree_pointer_name("feature-b") + assert pointer_b_name is not None + pointer_b = legacy_root / pointer_b_name + generic_pointer = legacy_root / ".last_state" + self.assertIn("scope-b", Path(pointer_b.read_text(encoding="utf-8")).parts) + self.assertIn("scope-b", Path(generic_pointer.read_text(encoding="utf-8")).parts) + + self.assertTrue(repo_a.deactivate_run("run-a")) + + self.assertTrue(pointer_b.is_file()) + self.assertTrue(generic_pointer.is_file()) + self.assertIn("scope-b", Path(pointer_b.read_text(encoding="utf-8")).parts) + self.assertIn("scope-b", Path(generic_pointer.read_text(encoding="utf-8")).parts) + + def test_partial_restart_ports_merge_current_context_with_preserved_project(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + initial = RunState( + run_id="run-ab", + mode="trees", + services={ + **self._state("unused", "feature-a", port=8100).services, + **self._state("unused", "feature-b", port=8200).services, + }, + ) + repo.save_run( + state=initial, + contexts=[ + self._context(root, "feature-a", 8100), + self._context(root, "feature-b", 8200), + ], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda saved: {"run_id": saved.run_id}, + ) + restarted = RunState( + run_id="run-restart-a", + mode="trees", + services={ + **self._state("unused", "feature-a", port=8111).services, + **self._state("unused", "feature-b", port=8200).services, + }, + metadata={"state_source_run_ids": ["run-ab"]}, + ) + self._save_run( + repo, + restarted, + context=self._context(root, "feature-a", 8111), + ) + + payload = json.loads(repo.ports_manifest_path().read_text(encoding="utf-8")) + ports_by_project = { + project["project"]: project["ports"]["backend"]["final"] for project in payload["projects"] + } + self.assertEqual(ports_by_project, {"feature-a": 8111, "feature-b": 8200}) + + def test_aggregate_aliases_merge_independent_owner_ports_and_errors(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + self._save_run( + repo, + self._state("run-a", "feature-a", port=8100), + context=self._context(root, "feature-a", 8100), + errors=["feature-a failed"], + ) + self._save_run( + repo, + self._state("run-b", "feature-b", port=8200), + context=self._context(root, "feature-b", 8200), + errors=["feature-b failed"], + ) + + current = load_state(str(repo.run_state_path())) + self.assertEqual(set(current.services), {"feature-a Backend", "feature-b Backend"}) + ports = json.loads(repo.ports_manifest_path().read_text(encoding="utf-8")) + errors = json.loads(repo.error_report_path().read_text(encoding="utf-8")) + self.assertEqual( + {project["project"] for project in ports["projects"]}, + {"feature-a", "feature-b"}, + ) + self.assertEqual(set(errors["errors"]), {"feature-a failed", "feature-b failed"}) + + self.assertTrue(repo.deactivate_run("run-a")) + remaining = load_state(str(repo.run_state_path())) + ports = json.loads(repo.ports_manifest_path().read_text(encoding="utf-8")) + errors = json.loads(repo.error_report_path().read_text(encoding="utf-8")) + self.assertEqual(set(remaining.services), {"feature-b Backend"}) + self.assertEqual( + {project["project"] for project in ports["projects"]}, + {"feature-b"}, + ) + self.assertEqual(errors["errors"], ["feature-b failed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/state/test_state_roundtrip.py b/tests/python/state/test_state_roundtrip.py index 781aad674..05bb181fb 100644 --- a/tests/python/state/test_state_roundtrip.py +++ b/tests/python/state/test_state_roundtrip.py @@ -6,7 +6,7 @@ REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" -from envctl_engine.state.models import RunState, ServiceRecord +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord from envctl_engine.state import dump_state, load_legacy_shell_state, load_state, merge_states @@ -25,9 +25,26 @@ def test_json_state_roundtrip_preserves_services(self) -> None: pid=12345, requested_port=8000, actual_port=8001, + port_lock_session="planner-session-1", listener_pids=[12345, 12346], status="running", started_at=1234.5, + runtime_kind="docker", + container_id="sha256:abc", + container_name="envctl-app-tree-alpha-backend", + container_image="envctl/example-backend:dev", + container_launch_token="launch-token-123", + container_cleanup_pending_since=1_700_000_000.25, + ) + }, + requirements={ + "Tree Alpha": RequirementsResult( + project="Tree Alpha", + db={ + "enabled": True, + "final": 5432, + "port_lock_session": "planner-session-1", + }, ) }, metadata={"source": "python"}, @@ -38,10 +55,56 @@ def test_json_state_roundtrip_preserves_services(self) -> None: self.assertEqual(loaded.run_id, "run-123") self.assertEqual(loaded.services["Tree Alpha Backend"].actual_port, 8001) + self.assertEqual( + loaded.services["Tree Alpha Backend"].port_lock_session, + "planner-session-1", + ) self.assertEqual(loaded.services["Tree Alpha Backend"].listener_pids, [12345, 12346]) self.assertEqual(loaded.services["Tree Alpha Backend"].started_at, 1234.5) + self.assertEqual( + loaded.requirements["Tree Alpha"].db["port_lock_session"], + "planner-session-1", + ) + self.assertEqual(loaded.services["Tree Alpha Backend"].runtime_kind, "docker") + self.assertEqual(loaded.services["Tree Alpha Backend"].container_id, "sha256:abc") + self.assertEqual( + loaded.services["Tree Alpha Backend"].container_name, + "envctl-app-tree-alpha-backend", + ) + self.assertEqual( + loaded.services["Tree Alpha Backend"].container_image, + "envctl/example-backend:dev", + ) + self.assertEqual( + loaded.services["Tree Alpha Backend"].container_launch_token, + "launch-token-123", + ) + self.assertEqual( + loaded.services["Tree Alpha Backend"].container_cleanup_pending_since, + 1_700_000_000.25, + ) self.assertEqual(loaded.metadata["source"], "python") + def test_requirement_storage_key_roundtrip_preserves_authoritative_owner_project(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) / "run_state.json" + state = RunState( + run_id="run-collision", + mode="main", + requirements={ + "Main Restart Collision": RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6381}, + ) + }, + ) + + dump_state(state, str(path)) + loaded = load_state(str(path), allowed_root=tmpdir) + + self.assertEqual(loaded.requirements["Main Restart Collision"].project, "Main") + self.assertEqual(loaded.requirements["Main Restart Collision"].redis["final"], 6381) + def test_legacy_shell_state_is_loaded_without_source_execution(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: legacy = Path(tmpdir) / "legacy.state" @@ -95,6 +158,32 @@ def test_merge_states_roundtrip_keeps_latest_ports(self) -> None: self.assertEqual(merged.run_id, "run-b") self.assertEqual(merged.services["Tree Alpha Frontend"].actual_port, 9002) + def test_merge_states_preserves_same_storage_name_owned_by_distinct_projects(self) -> None: + alpha = ServiceRecord( + name="Opaque Shared Runtime", + type="worker", + cwd="/alpha", + pid=101, + project="Alpha", + ) + beta = ServiceRecord( + name="Opaque Shared Runtime", + type="worker", + cwd="/beta", + pid=202, + project="Beta", + ) + + merged = merge_states( + [ + RunState(run_id="alpha", mode="trees", services={alpha.name: alpha}), + RunState(run_id="beta", mode="trees", services={beta.name: beta}), + ] + ) + + self.assertEqual({service.pid for service in merged.services.values()}, {101, 202}) + self.assertEqual(len(merged.services), 2) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/state/test_state_shell_compatibility.py b/tests/python/state/test_state_shell_compatibility.py index f293f9a3a..a2ef3bcac 100644 --- a/tests/python/state/test_state_shell_compatibility.py +++ b/tests/python/state/test_state_shell_compatibility.py @@ -134,6 +134,21 @@ def test_engine_runtime_resolves_state_from_pointer_files(self) -> None: assert loaded is not None self.assertEqual(loaded.mode, "main") self.assertIn("Main Backend", loaded.services) + primary_index = runtime.runtime_root / "run_index.json" + backup_index = runtime.runtime_root / "run_index.backup.json" + self.assertEqual( + primary_index.read_text(encoding="utf-8"), + backup_index.read_text(encoding="utf-8"), + ) + migrated_pointer = Path((runtime.runtime_root / ".last_state.main").read_text(encoding="utf-8").strip()) + self.assertTrue(migrated_pointer.is_file()) + self.assertTrue(migrated_pointer.is_relative_to(runtime.runtime_root / "runs")) + self.assertTrue(state_path.is_file()) + + loaded_again = runtime._try_load_existing_state() + self.assertIsNotNone(loaded_again) + assert loaded_again is not None + self.assertEqual(loaded_again.run_id, loaded.run_id) if __name__ == "__main__": diff --git a/tests/python/ui/test_dashboard_pr_link_rendering.py b/tests/python/ui/test_dashboard_pr_link_rendering.py index b52d1b3bf..7dcfde672 100644 --- a/tests/python/ui/test_dashboard_pr_link_rendering.py +++ b/tests/python/ui/test_dashboard_pr_link_rendering.py @@ -59,6 +59,19 @@ def test_project_root_prefers_metadata_then_service_component_parent(self) -> No ) }, ) + opaque_service_state = RunState( + run_id="run-opaque", + mode="main", + services={ + "Opaque API Process": ServiceRecord( + name="Opaque API Process", + type="backend", + cwd=str(backend), + project="Main", + service_slug="backend", + ) + }, + ) self.assertEqual( pr_link_rendering.dashboard_project_root(SimpleNamespace(), state=metadata_state, project="Main"), @@ -68,6 +81,14 @@ def test_project_root_prefers_metadata_then_service_component_parent(self) -> No pr_link_rendering.dashboard_project_root(SimpleNamespace(), state=service_state, project="Main"), repo, ) + self.assertEqual( + pr_link_rendering.dashboard_project_root( + SimpleNamespace(), + state=opaque_service_state, + project="Main", + ), + repo, + ) def test_lookup_pr_caches_missing_and_positive_results(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: diff --git a/tests/python/ui/test_dashboard_restart_selection_support.py b/tests/python/ui/test_dashboard_restart_selection_support.py index 09b157c91..777d0db2b 100644 --- a/tests/python/ui/test_dashboard_restart_selection_support.py +++ b/tests/python/ui/test_dashboard_restart_selection_support.py @@ -36,6 +36,19 @@ def test_dashboard_stopped_services_by_project_with_data(self) -> None: result = dashboard_stopped_services_by_project(state) self.assertEqual(result, {"p1": {"backend": "p1 Backend", "frontend": "p1 Frontend"}}) + def test_dashboard_stopped_services_by_project_includes_additional_services(self) -> None: + state = _make_state(metadata={ + "dashboard_stopped_services": [ + {"project": "p1", "type": "voice-runtime", "name": "opaque-agent"}, + {"project": "p1", "type": "worker", "name": ""}, + ] + }) + + self.assertEqual( + dashboard_stopped_services_by_project(state), + {"p1": {"voice-runtime": "opaque-agent", "worker": "p1 Worker"}}, + ) + def test_dashboard_project_configured_services_empty(self) -> None: state = _make_state() result = dashboard_project_configured_services(state) diff --git a/tests/python/ui/test_selection_support.py b/tests/python/ui/test_selection_support.py index bca70906e..62c026f5f 100644 --- a/tests/python/ui/test_selection_support.py +++ b/tests/python/ui/test_selection_support.py @@ -97,6 +97,30 @@ def test_services_from_selection_handles_all_projects_and_services(self) -> None {"Main Backend", "Main Frontend"}, ) + def test_multiword_additional_service_selection_uses_record_project_metadata(self) -> None: + service = ServiceRecord( + name="Opaque Runtime Process", + type="voice-runtime", + cwd="/tmp/customer-platform/voice", + project="Customer Platform", + service_slug="voice-runtime", + ) + state = RunState(run_id="run-extra", mode="trees", services={service.name: service}) + runtime = SimpleNamespace(_project_name_from_service=lambda _name: "Opaque Runtime") + + self.assertEqual( + [project.name for project in project_names_from_state(runtime, state)], + ["Customer Platform"], + ) + self.assertEqual( + services_from_selection( + runtime, + TargetSelection(project_names=["customer platform"]), + state, + ), + {service.name}, + ) + def test_no_target_selected_message_and_explicit_target_detection_match_route_context(self) -> None: runtime = SimpleNamespace(_selectors_from_passthrough=lambda args: {"svc"} if args else set()) @@ -108,6 +132,14 @@ def test_no_target_selected_message_and_explicit_target_detection_match_route_co "--project ", no_target_selected_message("restart", route=_route(mode="trees"), interactive_allowed=False), ) + self.assertIn( + "--all --yes", + no_target_selected_message("ship", route=_route(mode="trees"), interactive_allowed=False), + ) + self.assertNotIn( + "--all --yes", + no_target_selected_message("test", route=_route(mode="trees"), interactive_allowed=False), + ) self.assertTrue(route_has_explicit_target(_route(flags={"all": True}), runtime)) self.assertTrue(route_has_explicit_target(_route(projects=["Main"]), runtime))