diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 71394a09..394404e0 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -253,11 +253,16 @@ envctl pr-preview-controller --command start --pr-number 287 Behavior: - starts or reconciles a preview when the configured label is added +- treats an explicit `--command start` as a forced reconciliation: an existing + same-head runtime is stopped before the import/start path runs again, so + changed CI environment or secret inputs cannot be hidden by Git identity - redeploys labeled PRs when GitHub sends a new-commit event - loads a tracked `.envctl` from the exact PR head when present; otherwise it falls back to the controller-generated configuration - publishes explicitly listed `ENVCTL_PREVIEW_PUBLIC_SERVICES` on deterministic TLS hosts and projects matching HTTP/WebSocket source URLs into launch env +- rejects a routed service whose JSON health response explicitly reports + `"ready": false`, even when the endpoint itself returns HTTP 2xx - creates `ENVCTL_PREVIEW_GENERATED_SECRETS` ephemerally for each preview start, so shared service credentials do not need to be stored in GitHub - also redeploys during scheduled reconciliation if the saved preview head is stale @@ -404,6 +409,14 @@ envctl kill --backend --headless # alias for stop --backend envctl kill-all --headless # alias for stop-all ``` +Plain dependency cleanup is intentionally storage-preserving: `stop --dependencies` +and plain `stop-all` release saved dependency records and scoped port locks, but +leave envctl-managed Docker dependency stacks and their storage available for a +later start to reuse. Use `stop-all --remove-volumes` to remove supported managed +containers and volumes, or `blast-all` for aggressive ecosystem cleanup. +Explicit `--main` and `--trees` cleanup is mode-scoped: it never stops or retires +the other mode's saved runtime. + For AI plan-agent launches, `--entire-system` starts configured or autodetected local app services. In a repo/worktree with no explicit backend, frontend, or additional-service configuration and no supported autodetectable app layout, envctl reports that no local app system is configured and continues without app services; the AI session still launches. @@ -538,6 +551,11 @@ Commit defaults: - write one complete next commit message in `.envctl-commit-message.md` rather than multiple fragmented summaries only when using the fallback ledger - envctl-local control artifacts (`.envctl*`, `.codegraph/`, `.serena/project.local.yml`, `MAIN_TASK.md`, `OLD_TASK_*.md`, `trees/`) stay local; if a broad `git add .` stages them, `envctl commit` unstages those protected paths before committing normal changes +PR defaults: + +- `envctl pr` and `envctl ship` target `dev` when that remote branch exists, otherwise `main`, otherwise `master`; use `--pr-base ` to override the selection explicitly +- an existing open PR for the current head branch is rediscovered before creation, so rerunning the command reports or updates the same PR instead of opening a duplicate + Optional plan-agent launch config for `--plan` and `--import`: `envctl ship` is the preferred AI handoff command. From inside the current diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 227ffd30..8625ae8a 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -139,6 +139,13 @@ Only simple `${VAR}` placeholders are supported. Shell-style defaults, command s Frontend launch-env sections may use `ENVCTL_SOURCE_SUPABASE_URL` and `ENVCTL_SOURCE_SUPABASE_ANON_KEY`. They cannot reference `ENVCTL_SOURCE_SUPABASE_SERVICE_ROLE_KEY`; envctl rejects that mapping before launching the frontend process. +Every launched service also receives `ENVCTL_PYTHON_EXECUTABLE`, the absolute +interpreter path running envctl. Repository-owned shell launchers can use this +as a bootstrap interpreter on minimal hosts where a `python` command alias is +not installed. Applications should still create and execute their own project +environment rather than importing application dependencies from envctl's +environment. + ## Managed Supabase Ports Managed Supabase exposes two distinct local resources: diff --git a/docs/reference/important-flags.md b/docs/reference/important-flags.md index 4441f34f..b15bdc3e 100644 --- a/docs/reference/important-flags.md +++ b/docs/reference/important-flags.md @@ -41,9 +41,9 @@ pass `--interactive` only when you want prompts. | `--frontend` | Runtime scope: run or stop frontend app services only. | | `--fullstack` | Runtime scope: run or stop backend and frontend app services. | | `--both` | Alias for `--fullstack`. | -| `--dependencies` | Runtime scope: run or stop dependencies only (DB/Redis/etc.), without app services. | +| `--dependencies` | Runtime scope: run dependencies only, or detach their saved records/port locks on stop without removing reusable managed Docker stacks. | | `--deps` | Alias for `--dependencies`. | -| `--entire-system` | Runtime scope: run or stop dependencies plus backend and frontend services. | +| `--entire-system` | Runtime scope: run dependencies plus app services; on stop, terminate app services and detach dependency records/locks while keeping managed Docker stacks reusable. | | `--only-frontend` | Startup/plan modifier: launch only the frontend app service; skip backend, managed dependencies, and dependency prep. | | `--only-backend` | Startup/plan modifier: launch only the backend app service; skip frontend, managed dependencies, and dependency prep. | | `--no-deps` | Startup/plan modifier: skip managed dependencies and plan-agent dependency prep. | diff --git a/docs/user/common-workflows.md b/docs/user/common-workflows.md index f9de3774..a4872cd3 100644 --- a/docs/user/common-workflows.md +++ b/docs/user/common-workflows.md @@ -68,6 +68,12 @@ envctl stop --entire-system --headless envctl kill-all --headless ``` +`stop --dependencies` and plain `stop-all` preserve managed Docker dependency +stacks and storage for reuse while releasing their saved records and port locks. +Use `stop-all --remove-volumes` or `blast-all` when those managed containers must +also be removed. Pass `--main` or `--trees` to clean only that saved runtime mode; +the other mode remains active. + When you invoke a specific action command directly, envctl stays non-interactive by default. For example, `envctl kill-all`, `envctl pr`, `envctl test`, `envctl logs`, and `envctl migrate` behave like their `--headless` forms. Add `--interactive` only when you want envctl to prompt diff --git a/docs/user/getting-started.md b/docs/user/getting-started.md index ce08c790..c96f8c62 100644 --- a/docs/user/getting-started.md +++ b/docs/user/getting-started.md @@ -188,7 +188,7 @@ Why this works well: - `explain-startup` shows what the runtime is about to do - `--resume` is the fastest normal start path after previous runs - `dashboard`, `logs`, and `test` cover the common daily loop -- `stop-all` gives you a clean shutdown +- `stop-all` cleanly shuts down tracked app services and releases dependency records/locks; managed Docker dependency stacks and storage remain reusable (use `stop-all --remove-volumes` or `blast-all` to remove them) If you want a fresh start instead of resume: diff --git a/python/envctl_engine/actions/action_commit_support.py b/python/envctl_engine/actions/action_commit_support.py index 3ccaa4e2..43335d25 100644 --- a/python/envctl_engine/actions/action_commit_support.py +++ b/python/envctl_engine/actions/action_commit_support.py @@ -19,7 +19,7 @@ resolve_commit_message_request, write_commit_message_file as _write_commit_message_file, ) -from envctl_engine.actions.action_protected_artifacts import EnvctlProtectedPathPartition +from envctl_engine.actions.action_protected_artifacts import EnvctlProtectedPathPartition, unstaged_stageable_paths from envctl_engine.ui.path_links import render_paths_in_terminal_text RunGitFn = Callable[[Path, list[str]], subprocess.CompletedProcess[str]] @@ -88,6 +88,7 @@ def _stage_commit_candidates(self, git_root: Path) -> EnvctlProtectedPathPartiti return None partition = self.dependencies.partition_envctl_protected_paths(pre_stage_status.stdout) + paths_to_stage = unstaged_stageable_paths(pre_stage_status.stdout) unstaged_protected_paths = self._unstage_protected_paths(git_root, partition) if unstaged_protected_paths is None: return None @@ -97,7 +98,7 @@ def _stage_commit_candidates(self, git_root: Path) -> EnvctlProtectedPathPartiti return None partition = refreshed_partition - if not self._stage_paths(git_root, partition.stageable_paths): + if not self._stage_paths(git_root, paths_to_stage): return None self._print_skipped_protected_paths(unstaged_protected_paths, partition.protected_skipped_paths) return partition @@ -141,7 +142,7 @@ def _refreshed_partition_after_unstage(self, git_root: Path) -> EnvctlProtectedP def _stage_paths(self, git_root: Path, stageable_paths: list[str]) -> bool: if not stageable_paths: return True - add = self.dependencies.run_git(git_root, ["add", "--", *stageable_paths]) + add = self.dependencies.run_git(git_root, ["add", "--all", "--", *stageable_paths]) if add.returncode != 0: self.dependencies.print_error("git add failed", add) return False diff --git a/python/envctl_engine/actions/action_git_state_support.py b/python/envctl_engine/actions/action_git_state_support.py index d99a73ab..53928f91 100644 --- a/python/envctl_engine/actions/action_git_state_support.py +++ b/python/envctl_engine/actions/action_git_state_support.py @@ -1,10 +1,14 @@ from __future__ import annotations from dataclasses import dataclass +import json +import os from pathlib import Path import subprocess from typing import Callable +from envctl_engine.shared.git_branch_support import detect_preferred_default_branch + @dataclass(frozen=True, slots=True) class DirtyWorktreeReport: @@ -20,6 +24,12 @@ def dirty(self) -> bool: return self.staged or self.unstaged or self.untracked +@dataclass(frozen=True, slots=True) +class ExistingPullRequest: + url: str + base_branch: str + + def resolve_git_root(project_root: Path, repo_root: Path) -> Path: for candidate in (project_root, repo_root): if (candidate / ".git").exists(): @@ -71,53 +81,131 @@ def classify_dirty_porcelain(status_output: str) -> tuple[bool, bool, bool]: def detect_default_branch(git_root: Path, *, git_output: Callable[[Path, list[str]], str]) -> str: - ref = git_output(git_root, ["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]).strip() - if ref.startswith("origin/"): - return ref.split("origin/", 1)[1] - for candidate in ("main", "master"): - if git_output(git_root, ["rev-parse", "--verify", candidate]).strip(): - return candidate - return "main" + return detect_preferred_default_branch( + lambda args: git_output(git_root, list(args)), + probe_remote_heads=True, + ) -def existing_pr_url( +def existing_pull_request( git_root: Path, branch: str, *, gh_path: str | None, run_process: Callable[..., subprocess.CompletedProcess[str]], -) -> str: +) -> ExistingPullRequest | None: branch_name = branch.strip() if not branch_name or branch_name in {"HEAD", "unknown"}: - return "" + return None if gh_path is None: - return "" + return None listed = run_process( - [gh_path, "pr", "list", "--head", branch_name, "--state", "open", "--json", "url", "--jq", ".[0].url"], + [ + gh_path, + "pr", + "list", + "--head", + branch_name, + "--state", + "open", + "--limit", + "1", + "--json", + "url,baseRefName", + ], cwd=str(git_root), text=True, capture_output=True, check=False, ) if listed.returncode != 0: - return "" - return listed.stdout.strip() + return None + try: + payload = json.loads(listed.stdout or "[]") + except (TypeError, json.JSONDecodeError): + return None + if not isinstance(payload, list) or not payload or not isinstance(payload[0], dict): + return None + url = str(payload[0].get("url") or "").strip() + if not url: + return None + return ExistingPullRequest( + url=url, + base_branch=str(payload[0].get("baseRefName") or "").strip(), + ) -def run_git( +def existing_pr_url( git_root: Path, - args: list[str], + branch: str, *, + gh_path: str | None, + run_process: Callable[..., subprocess.CompletedProcess[str]], +) -> str: + existing = existing_pull_request( + git_root, + branch, + gh_path=gh_path, + run_process=run_process, + ) + return existing.url if existing is not None else "" + + +def update_pull_request_base( + git_root: Path, + pull_request: ExistingPullRequest, + base_branch: str, + *, + gh_path: str | None, run_process: Callable[..., subprocess.CompletedProcess[str]], ) -> subprocess.CompletedProcess[str]: + branch = base_branch.strip() + command = [*([gh_path] if gh_path else ["gh"]), "pr", "edit", pull_request.url, "--base", branch] + if gh_path is None: + return subprocess.CompletedProcess( + command, + 127, + stdout="", + stderr=( + "gh is required to update the existing pull request base. " + f"Install or authenticate gh, then run: gh pr edit {pull_request.url} --base {branch}" + ), + ) return run_process( - ["git", "-C", str(git_root), *args], + command, + cwd=str(git_root), text=True, capture_output=True, check=False, ) +def run_git( + git_root: Path, + args: list[str], + *, + run_process: Callable[..., subprocess.CompletedProcess[str]], +) -> subprocess.CompletedProcess[str]: + command = ["git", "-C", str(git_root), *args] + options: dict[str, object] = { + "text": True, + "capture_output": True, + "check": False, + } + if args[:1] == ["ls-remote"]: + options["timeout"] = 10 + options["env"] = {**os.environ, "GIT_TERMINAL_PROMPT": "0"} + try: + return run_process(command, **options) + except subprocess.TimeoutExpired as exc: + return subprocess.CompletedProcess( + command, + 124, + stdout=str(exc.stdout or ""), + stderr="Timed out while querying remote branches.", + ) + + def git_output( git_root: Path, args: list[str], diff --git a/python/envctl_engine/actions/action_pr_git_support.py b/python/envctl_engine/actions/action_pr_git_support.py new file mode 100644 index 00000000..bf20826a --- /dev/null +++ b/python/envctl_engine/actions/action_pr_git_support.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from pathlib import Path +import shutil +import subprocess + +from envctl_engine.actions.action_git_state_support import ( + ExistingPullRequest, + existing_pr_url as lookup_existing_pr_url, + existing_pull_request as lookup_existing_pull_request, + update_pull_request_base as update_existing_pull_request_base, +) + + +def existing_pr_url(git_root: Path, branch: str) -> str: + return lookup_existing_pr_url( + git_root, + branch, + gh_path=shutil.which("gh"), + run_process=subprocess.run, + ) + + +def existing_pull_request(git_root: Path, branch: str) -> ExistingPullRequest | None: + return lookup_existing_pull_request( + git_root, + branch, + gh_path=shutil.which("gh"), + run_process=subprocess.run, + ) + + +def update_pull_request_base( + git_root: Path, + pull_request: ExistingPullRequest, + base_branch: str, +) -> subprocess.CompletedProcess[str]: + return update_existing_pull_request_base( + git_root, + pull_request, + base_branch, + gh_path=shutil.which("gh"), + run_process=subprocess.run, + ) + + +__all__ = ["existing_pr_url", "existing_pull_request", "update_pull_request_base"] diff --git a/python/envctl_engine/actions/action_pr_workflow_support.py b/python/envctl_engine/actions/action_pr_workflow_support.py index 523b7897..da6ecf87 100644 --- a/python/envctl_engine/actions/action_pr_workflow_support.py +++ b/python/envctl_engine/actions/action_pr_workflow_support.py @@ -5,6 +5,12 @@ import subprocess from typing import Any, Callable +from envctl_engine.actions.action_git_state_support import ExistingPullRequest + + +ExistingPullRequestLookup = Callable[[Path, str], ExistingPullRequest | None] +PullRequestBaseUpdater = Callable[[Path, ExistingPullRequest, str], subprocess.CompletedProcess[str]] + def run_pr_workflow( context: Any, @@ -13,7 +19,8 @@ def run_pr_workflow( git_available: bool, git_output_fn: Callable[[Path, list[str]], str], resolve_base_branch_fn: Callable[[Any, Path], str], - existing_pr_url_fn: Callable[[Path, str], str], + existing_pull_request_fn: ExistingPullRequestLookup, + update_pull_request_base_fn: PullRequestBaseUpdater, probe_dirty_worktree_fn: Callable[[Path, Path, str], Any], run_commit_action_fn: Callable[[Any], int], pr_title_fn: Callable[[Any, Path, str], str], @@ -32,13 +39,31 @@ def run_pr_workflow( if head_branch in {"HEAD", "unknown"}: print(f"Skipping {context.project_name} (detached HEAD).") return 0 - base_branch = resolve_base_branch_fn(context, git_root) - existing_url = existing_pr_url_fn(git_root, head_branch) - if existing_url: - print(f"PR already exists: {existing_url}") + explicit_base = _explicit_pr_base(context) + existing = existing_pull_request_fn(git_root, head_branch) + if existing is not None: + if explicit_base and existing.base_branch != explicit_base: + if not _update_existing_pull_request_base( + git_root=git_root, + head_branch=head_branch, + existing=existing, + desired_base=explicit_base, + existing_pull_request_fn=existing_pull_request_fn, + update_pull_request_base_fn=update_pull_request_base_fn, + print_process_output_fn=print_process_output_fn, + ): + return 1 + print(f"PR already exists: {existing.url}") return 0 + base_branch = resolve_base_branch_fn(context, git_root) + if base_branch and head_branch == base_branch: + print( + f"Cannot open a pull request from {head_branch!r} into itself. Create or switch to a feature branch first." + ) + return 1 + dirty_report = probe_dirty_worktree_fn(context.project_root, context.repo_root, context.project_name) if dirty_report.dirty: print(f"Dirty worktree detected for {context.project_name}; committing and pushing before PR creation.") @@ -54,6 +79,7 @@ def run_pr_workflow( git_root=git_root, head_branch=head_branch, base_branch=base_branch, + existing_pull_request_fn=existing_pull_request_fn, print_process_output_fn=print_process_output_fn, run_process_fn=run_process_fn, ) @@ -70,11 +96,58 @@ def run_pr_workflow( pr_title_fn=pr_title_fn, pr_body_fn=pr_body_fn, write_pr_body_file_fn=write_pr_body_file_fn, + existing_pull_request_fn=existing_pull_request_fn, print_process_output_fn=print_process_output_fn, run_process_fn=run_process_fn, ) +def _explicit_pr_base(context: Any) -> str: + env = getattr(context, "env", {}) + try: + return str(env.get("ENVCTL_PR_BASE", "")).strip() + except AttributeError: + return "" + + +def _update_existing_pull_request_base( + *, + git_root: Path, + head_branch: str, + existing: ExistingPullRequest, + desired_base: str, + existing_pull_request_fn: ExistingPullRequestLookup, + update_pull_request_base_fn: PullRequestBaseUpdater, + print_process_output_fn: Callable[[subprocess.CompletedProcess[str]], None], +) -> bool: + updated = update_pull_request_base_fn(git_root, existing, desired_base) + if updated.returncode != 0: + refreshed = existing_pull_request_fn(git_root, head_branch) + if _pull_request_targets(refreshed, desired_base): + return True + print_process_output_fn(updated) + print( + f"Unable to update existing PR {existing.url} from base " + f"{existing.base_branch or ''!r} to {desired_base!r}." + ) + return False + + refreshed = existing_pull_request_fn(git_root, head_branch) + if not _pull_request_targets(refreshed, desired_base): + actual_base = refreshed.base_branch if refreshed is not None else "" + print( + f"Updated existing PR {existing.url}, but could not verify base {desired_base!r} " + f"(reported {actual_base!r})." + ) + return False + print(f"Updated PR base to {desired_base}: {refreshed.url}") + return True + + +def _pull_request_targets(pull_request: ExistingPullRequest | None, base_branch: str) -> bool: + return pull_request is not None and pull_request.base_branch == base_branch + + def _run_create_pr_helper( context: Any, *, @@ -82,6 +155,7 @@ def _run_create_pr_helper( git_root: Path, head_branch: str, base_branch: str, + existing_pull_request_fn: ExistingPullRequestLookup, print_process_output_fn: Callable[[subprocess.CompletedProcess[str]], None], run_process_fn: Callable[..., subprocess.CompletedProcess[str]], ) -> int: @@ -96,9 +170,17 @@ def _run_create_pr_helper( capture_output=True, check=False, ) - print_process_output_fn(created) if created.returncode != 0: + if _recover_concurrent_pull_request( + git_root=git_root, + head_branch=head_branch, + desired_base=base_branch, + existing_pull_request_fn=existing_pull_request_fn, + ): + return 0 + print_process_output_fn(created) return 1 + print_process_output_fn(created) return 0 @@ -112,6 +194,7 @@ def _run_gh_pr_create( pr_title_fn: Callable[[Any, Path, str], str], pr_body_fn: Callable[[Any, Path, str, str], str], write_pr_body_file_fn: Callable[[str], Path], + existing_pull_request_fn: ExistingPullRequestLookup, print_process_output_fn: Callable[[subprocess.CompletedProcess[str]], None], run_process_fn: Callable[..., subprocess.CompletedProcess[str]], ) -> int: @@ -123,12 +206,34 @@ def _run_gh_pr_create( args.extend(["--base", base_branch]) try: created = run_process_fn(args, cwd=str(git_root), text=True, capture_output=True, check=False) - print_process_output_fn(created) if created.returncode != 0: + if _recover_concurrent_pull_request( + git_root=git_root, + head_branch=head_branch, + desired_base=base_branch, + existing_pull_request_fn=existing_pull_request_fn, + ): + return 0 + print_process_output_fn(created) return 1 + print_process_output_fn(created) return 0 finally: try: body_file.unlink() except OSError: pass + + +def _recover_concurrent_pull_request( + *, + git_root: Path, + head_branch: str, + desired_base: str, + existing_pull_request_fn: ExistingPullRequestLookup, +) -> bool: + existing = existing_pull_request_fn(git_root, head_branch) + if not _pull_request_targets(existing, desired_base): + return False + print(f"PR already exists: {existing.url}") + return True diff --git a/python/envctl_engine/actions/action_protected_artifacts.py b/python/envctl_engine/actions/action_protected_artifacts.py index 90baf020..90169998 100644 --- a/python/envctl_engine/actions/action_protected_artifacts.py +++ b/python/envctl_engine/actions/action_protected_artifacts.py @@ -63,6 +63,26 @@ def ordered_unique_paths(*path_groups: list[str]) -> list[str]: return paths +def unstaged_stageable_paths(status_output: str) -> list[str]: + paths: list[str] = [] + seen: set[str] = set() + for raw_line in str(status_output or "").splitlines(): + line = raw_line.rstrip("\n") + if len(line) < 4: + continue + is_untracked = line.startswith("?? ") + worktree_status = line[1] + if not is_untracked and worktree_status == " ": + continue + candidates = status_candidate_paths(line, include_rename_source=not is_untracked) + for candidate in candidates: + if is_envctl_local_artifact_path(candidate) or candidate in seen: + continue + paths.append(candidate) + seen.add(candidate) + return paths + + def status_candidate_path(line: str) -> str: if line.startswith("?? "): return line[3:].strip() @@ -72,3 +92,16 @@ def status_candidate_path(line: str) -> str: if " -> " in payload: return payload.split(" -> ", 1)[1].strip() return payload + + +def status_candidate_paths(line: str, *, include_rename_source: bool = False) -> list[str]: + candidate = status_candidate_path(line) + if not candidate: + return [] + if not include_rename_source: + return [candidate] + payload = line[3:].strip() + if " -> " not in payload: + return [candidate] + source = payload.split(" -> ", 1)[0].strip() + return [source, candidate] if source and source != candidate else [candidate] diff --git a/python/envctl_engine/actions/action_ship_checks_phase.py b/python/envctl_engine/actions/action_ship_checks_phase.py new file mode 100644 index 00000000..f9103492 --- /dev/null +++ b/python/envctl_engine/actions/action_ship_checks_phase.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from envctl_engine.actions.action_ship_contract import emit_ship_progress +from envctl_engine.actions.action_ship_phase_status import ( + callable_accepts_keyword, + check_phase_status, + ship_status_is_success, +) + +GithubPrChecks = Callable[..., dict[str, object]] +FinishShip = Callable[..., int] + + +def run_ship_checks_phase( + *, + state: Any, + checks_fn: GithubPrChecks, + finish: FinishShip, +) -> int: + check_kwargs: dict[str, object] = { + "branch": state.branch, + "pr_url": state.pr_url, + "expected_head_sha": state.after_sha, + } + if callable_accepts_keyword(checks_fn, "progress_callback"): + check_kwargs["progress_callback"] = emit_ship_progress + checks = checks_fn(state.git_root, **check_kwargs) + status = check_phase_status(checks) + state.step_statuses.append(status) + return finish( + state, + status=status, + ok=ship_status_is_success(status), + commit_sha=state.after_sha, + pushed=state.pushed, + pr_url=state.pr_url, + pr_created=state.pr_created, + checks=checks, + merge_conflicts=state.merge_conflicts, + ) + + +__all__ = ["run_ship_checks_phase"] diff --git a/python/envctl_engine/actions/action_ship_conflicts.py b/python/envctl_engine/actions/action_ship_conflicts.py index d0047d03..919210dd 100644 --- a/python/envctl_engine/actions/action_ship_conflicts.py +++ b/python/envctl_engine/actions/action_ship_conflicts.py @@ -62,9 +62,10 @@ def predicted_merge_conflict_report( resolve_base_ref: ResolveBaseRef, run_git: RunGit, git_output: GitOutput, + base_branch_override: str = "", ) -> dict[str, object]: del branch - base_branch = resolve_base_branch(context, git_root) + base_branch = base_branch_override.strip() or resolve_base_branch(context, git_root) base_ref = resolve_base_ref(git_root, base_branch) if not base_ref: return { diff --git a/python/envctl_engine/actions/action_ship_pr_context.py b/python/envctl_engine/actions/action_ship_pr_context.py new file mode 100644 index 00000000..fdc36b6f --- /dev/null +++ b/python/envctl_engine/actions/action_ship_pr_context.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import Any, Protocol + +from envctl_engine.actions.action_git_state_support import ExistingPullRequest + +ExistingPullRequestLookup = Callable[[Path, str], ExistingPullRequest | None] + + +class ShipPullRequestState(Protocol): + git_root: Path + branch: str + pr_url: str + pr_base_branch: str + + +def explicit_pr_base(context: Any) -> str: + env = getattr(context, "env", {}) + try: + return str(env.get("ENVCTL_PR_BASE", "")).strip() + except AttributeError: + return "" + + +def refresh_existing_pull_request( + state: ShipPullRequestState, + lookup: ExistingPullRequestLookup | None, +) -> None: + if lookup is None: + return + existing = lookup(state.git_root, state.branch) + if existing is None: + return + state.pr_url = existing.url + state.pr_base_branch = existing.base_branch + + +def conflict_base_branch_override(context: Any, state: ShipPullRequestState) -> str: + return explicit_pr_base(context) or state.pr_base_branch + + +__all__ = [ + "ExistingPullRequestLookup", + "conflict_base_branch_override", + "explicit_pr_base", + "refresh_existing_pull_request", +] diff --git a/python/envctl_engine/actions/action_ship_pr_phase.py b/python/envctl_engine/actions/action_ship_pr_phase.py index f328c360..68b693a4 100644 --- a/python/envctl_engine/actions/action_ship_pr_phase.py +++ b/python/envctl_engine/actions/action_ship_pr_phase.py @@ -4,6 +4,7 @@ from typing import Any, Callable from envctl_engine.actions.action_ship_contract import emit_ship_progress +from envctl_engine.actions.action_ship_pr_context import explicit_pr_base as _explicit_pr_base FinishShip = Callable[..., int] @@ -18,6 +19,19 @@ def run_ship_pr_phase( finish: FinishShip, ) -> int | None: if state.pr_url: + explicit_base = _explicit_pr_base(context) + if explicit_base: + pr_code = run_pr_action(context) + if pr_code != 0: + return finish( + state, + status="pr_failed", + ok=False, + commit_sha=state.after_sha, + pr_url=state.pr_url, + pr_created=False, + ) + state.pr_url = existing_pr_url(state.git_root, state.branch) or state.pr_url state.step_statuses.append("pr_exists") emit_ship_progress(f"ship: PR already exists for {context.project_name}: {state.pr_url}") return None diff --git a/python/envctl_engine/actions/action_ship_support.py b/python/envctl_engine/actions/action_ship_support.py index 87c96088..407eb1dc 100644 --- a/python/envctl_engine/actions/action_ship_support.py +++ b/python/envctl_engine/actions/action_ship_support.py @@ -9,6 +9,7 @@ from envctl_engine.actions.action_ship_check_results import normalize_github_pr_checks # noqa: F401 from envctl_engine.actions.action_ship_checks import github_pr_checks as default_github_pr_checks +from envctl_engine.actions.action_ship_checks_phase import run_ship_checks_phase from envctl_engine.actions.action_ship_conflicts import ( existing_merge_conflict_report, parse_merge_tree_conflicts, # noqa: F401 @@ -17,17 +18,16 @@ ) from envctl_engine.actions.action_ship_contract import ( emit_ship_commit_progress, - emit_ship_progress, parse_ship_json_output, print_ship_result, # noqa: F401 ship_payload, # noqa: F401 ship_protected_paths, ) from envctl_engine.actions.action_ship_finish import finish_ship_workflow -from envctl_engine.actions.action_ship_phase_status import ( - callable_accepts_keyword, - check_phase_status, - ship_status_is_success, +from envctl_engine.actions.action_ship_pr_context import ( + ExistingPullRequestLookup, + conflict_base_branch_override, + refresh_existing_pull_request, ) from envctl_engine.actions.action_ship_pr_phase import run_ship_pr_label_phase, run_ship_pr_phase from envctl_engine.actions.action_ship_push import run_ship_push_phase @@ -39,6 +39,7 @@ GithubPrChecks = Callable[..., dict[str, object]] github_pr_checks = default_github_pr_checks + @dataclass(slots=True) class ShipWorkflowState: git_root: Path @@ -51,6 +52,7 @@ class ShipWorkflowState: committed: bool = False pushed: bool = False pr_url: str = "" + pr_base_branch: str = "" pr_created: bool = False step_statuses: list[str] = field(default_factory=list) merge_conflicts: dict[str, object] = field(default_factory=dict) @@ -72,6 +74,7 @@ class ShipWorkflowDependencies: partition_envctl_protected_paths: Callable[[str], Any] ordered_unique_paths: Callable[..., list[str]] github_pr_checks: GithubPrChecks | None = None + existing_pull_request: ExistingPullRequestLookup | None = None @dataclass(slots=True) @@ -144,7 +147,10 @@ def _run_commit_phase(self, state: ShipWorkflowState) -> int | None: self.context.repo_root, project_name=self.context.project_name, ) - state.pr_url = self.dependencies.existing_pr_url(state.git_root, state.branch) + if self.dependencies.existing_pull_request is None: + state.pr_url = self.dependencies.existing_pr_url(state.git_root, state.branch) + else: + refresh_existing_pull_request(state, self.dependencies.existing_pull_request) commit_code = self.dependencies.run_commit_action(self.context) state.after_sha = self.dependencies.git_output(state.git_root, ["rev-parse", "HEAD"]).strip() @@ -159,13 +165,16 @@ def _run_commit_phase(self, state: ShipWorkflowState) -> int | None: return self._finish(state, status="commit_failed", ok=False, commit_sha=state.after_sha) def _run_pr_phase(self, state: ShipWorkflowState) -> int | None: - return run_ship_pr_phase( + result = run_ship_pr_phase( state=state, context=self.context, run_pr_action=self.dependencies.run_pr_action, existing_pr_url=self.dependencies.existing_pr_url, finish=self._finish, ) + if result is None: + refresh_existing_pull_request(state, self.dependencies.existing_pull_request) + return result def _run_pr_label_phase(self, state: ShipWorkflowState) -> int | None: return run_ship_pr_label_phase( @@ -184,6 +193,7 @@ def _reject_predicted_merge_conflicts(self, state: ShipWorkflowState) -> int | N resolve_base_ref=self.dependencies.resolve_base_ref, run_git=self.dependencies.run_git, git_output=self.dependencies.git_output, + base_branch_override=conflict_base_branch_override(self.context, state), ) if state.merge_conflicts.get("state") != "conflicts": return None @@ -228,27 +238,10 @@ def _run_push_phase(self, state: ShipWorkflowState) -> int | None: return None def _run_checks_phase(self, state: ShipWorkflowState) -> int: - checks_fn = self.dependencies.github_pr_checks or default_github_pr_checks - check_kwargs: dict[str, object] = { - "branch": state.branch, - "pr_url": state.pr_url, - "expected_head_sha": state.after_sha, - } - if callable_accepts_keyword(checks_fn, "progress_callback"): - check_kwargs["progress_callback"] = emit_ship_progress - checks = checks_fn(state.git_root, **check_kwargs) - status = check_phase_status(checks) - state.step_statuses.append(status) - return self._finish( - state, - status=status, - ok=ship_status_is_success(status), - commit_sha=state.after_sha, - pushed=state.pushed, - pr_url=state.pr_url, - pr_created=state.pr_created, - checks=checks, - merge_conflicts=state.merge_conflicts, + return run_ship_checks_phase( + state=state, + checks_fn=self.dependencies.github_pr_checks or default_github_pr_checks, + finish=self._finish, ) def _finish( @@ -277,8 +270,10 @@ def _finish( merge_conflicts=merge_conflicts, ) + def run_ship_workflow( - context: Any, *, + context: Any, + *, resolve_git_root: Callable[[Path, Path], Path], git_available: bool, git_output: GitOutput, @@ -292,7 +287,9 @@ def run_ship_workflow( existing_pr_url: Callable[[Path, str], str], partition_envctl_protected_paths: Callable[[str], Any], ordered_unique_paths: Callable[..., list[str]], - github_pr_checks: GithubPrChecks | None = None) -> int: + github_pr_checks: GithubPrChecks | None = None, + existing_pull_request: ExistingPullRequestLookup | None = None, +) -> int: return ShipWorkflowRunner( context=context, dependencies=ShipWorkflowDependencies( @@ -310,5 +307,6 @@ def run_ship_workflow( partition_envctl_protected_paths=partition_envctl_protected_paths, ordered_unique_paths=ordered_unique_paths, github_pr_checks=github_pr_checks, + existing_pull_request=existing_pull_request, ), ).run() diff --git a/python/envctl_engine/actions/action_worktree_target_resolution.py b/python/envctl_engine/actions/action_worktree_target_resolution.py index 3a0fd638..9345c292 100644 --- a/python/envctl_engine/actions/action_worktree_target_resolution.py +++ b/python/envctl_engine/actions/action_worktree_target_resolution.py @@ -102,7 +102,13 @@ def _linked_worktree_candidate( configured_root_path: Path | None, raw_runtime: Any, ) -> object | None: - """Resolve a Git-linked checkout that lives outside the configured trees directory.""" + """Represent a linked checkout even when it is outside the configured trees directory. + + Source-checkout workflows commonly create sibling worktrees. Git proves + that such a checkout belongs to the configured repository, while the + top-level and configured-root comparisons prevent a tree-mode action + from silently targeting the main checkout. + """ top_level_raw = _git_output(raw_runtime, cwd, ["rev-parse", "--show-toplevel"]) if not top_level_raw: @@ -113,7 +119,6 @@ def _linked_worktree_candidate( return None if not _path_is_at_or_under(cwd, top_level): return None - normalized_repo_root = repo_root.resolve() if top_level == normalized_repo_root: return None @@ -179,9 +184,7 @@ def main_repo_root_for_worktree(*, worktree_root: Path, runtime: Any, trees_dir_ return top_level common_dir_path = Path(common_dir_raw) common_dir = ( - (worktree_root / common_dir_path).resolve() - if not common_dir_path.is_absolute() - else common_dir_path.resolve() + (worktree_root / common_dir_path).resolve() if not common_dir_path.is_absolute() else common_dir_path.resolve() ) if common_dir.name == ".git": return common_dir.parent diff --git a/python/envctl_engine/actions/project_action_domain.py b/python/envctl_engine/actions/project_action_domain.py index 316f75a1..5ff6e920 100644 --- a/python/envctl_engine/actions/project_action_domain.py +++ b/python/envctl_engine/actions/project_action_domain.py @@ -10,6 +10,7 @@ import envctl_engine.actions.action_commit_support as commit_support import envctl_engine.actions.action_pr_label_support as pr_label_support import envctl_engine.actions.action_git_state_support as git_state_support +import envctl_engine.actions.action_pr_git_support as pr_git_support import envctl_engine.actions.action_pr_message_support as pr_message_support import envctl_engine.actions.action_review_artifact_support as review_artifact_support import envctl_engine.actions.action_review_base_support as review_base_support @@ -46,6 +47,7 @@ __all__ = [ "ActionProjectContext", "DirtyWorktreeReport", + "ExistingPullRequest", "EnvctlProtectedPathPartition", "OriginalPlanResolution", "ReviewBaseResolution", @@ -70,6 +72,7 @@ "_unmerged_stage_entries", "_write_pr_body_file", "detect_default_branch", + "existing_pull_request", "existing_pr_url", "probe_dirty_worktree", "resolve_git_root", @@ -78,6 +81,7 @@ "run_pr_action", "run_review_action", "run_ship_action", + "update_pull_request_base", ] @@ -97,6 +101,7 @@ def interactive(self) -> bool: ReviewBaseResolutionError = review_base_support.ReviewBaseResolutionError OriginalPlanResolution = review_original_plan_support.OriginalPlanResolution + def _workflow_runner() -> ProjectActionWorkflowRunner: return workflow_factory.ProjectActionWorkflowFactory( git=workflow_factory.ProjectActionWorkflowGitSources( @@ -115,6 +120,8 @@ def _workflow_runner() -> ProjectActionWorkflowRunner: pull_request=workflow_factory.ProjectActionWorkflowPullRequestSources( resolve_base_branch_fn=_resolve_pr_base_branch, existing_pr_url_fn=existing_pr_url, + existing_pull_request_fn=existing_pull_request, + update_pull_request_base_fn=update_pull_request_base, probe_dirty_worktree_source_fn=probe_dirty_worktree, run_commit_action_fn=run_commit_action, pr_title_fn=_pr_title, @@ -279,8 +286,12 @@ def run_review_action(context: ActionProjectContext) -> int: DirtyWorktreeReport = git_state_support.DirtyWorktreeReport +ExistingPullRequest = git_state_support.ExistingPullRequest resolve_git_root = git_state_support.resolve_git_root _classify_dirty_porcelain = git_state_support.classify_dirty_porcelain +existing_pr_url = pr_git_support.existing_pr_url +existing_pull_request = pr_git_support.existing_pull_request +update_pull_request_base = pr_git_support.update_pull_request_base def probe_dirty_worktree(project_root: Path, repo_root: Path, *, project_name: str = "") -> DirtyWorktreeReport: @@ -296,15 +307,6 @@ def detect_default_branch(git_root: Path) -> str: return git_state_support.detect_default_branch(git_root, git_output=_git_output) -def existing_pr_url(git_root: Path, branch: str) -> str: - return git_state_support.existing_pr_url( - git_root, - branch, - gh_path=shutil.which("gh"), - run_process=subprocess.run, - ) - - _resolve_original_plan = review_original_plan_support.resolve_original_plan _read_worktree_provenance = review_original_plan_support.read_worktree_provenance _resolve_plan_file_from_record = review_original_plan_support.resolve_plan_file_from_record diff --git a/python/envctl_engine/actions/project_action_workflow_factory.py b/python/envctl_engine/actions/project_action_workflow_factory.py index 0b5d2fe4..7258c722 100644 --- a/python/envctl_engine/actions/project_action_workflow_factory.py +++ b/python/envctl_engine/actions/project_action_workflow_factory.py @@ -7,7 +7,7 @@ from typing import Any import envctl_engine.actions.action_ship_support as ship_support -from envctl_engine.actions.action_git_state_support import DirtyWorktreeReport +from envctl_engine.actions.action_git_state_support import DirtyWorktreeReport, ExistingPullRequest from envctl_engine.actions.action_review_context import ReviewActionContext from envctl_engine.actions.action_review_base_support import ReviewBaseResolution from envctl_engine.actions.action_review_original_plan_support import OriginalPlanResolution @@ -41,6 +41,8 @@ class ProjectActionWorkflowCommitSources: class ProjectActionWorkflowPullRequestSources: resolve_base_branch_fn: Callable[[Any, Path], str] existing_pr_url_fn: Callable[[Path, str], str] + existing_pull_request_fn: Callable[[Path, str], ExistingPullRequest | None] + update_pull_request_base_fn: Callable[[Path, ExistingPullRequest, str], subprocess.CompletedProcess[str]] probe_dirty_worktree_source_fn: Callable[..., DirtyWorktreeReport] run_commit_action_fn: Callable[[Any], int] pr_title_fn: Callable[[Any, Path, str], str] @@ -89,6 +91,8 @@ def build(self) -> ProjectActionWorkflowRunner: pull_request=ProjectActionPullRequestWorkflowDependencies( resolve_base_branch_fn=self.pull_request.resolve_base_branch_fn, existing_pr_url_fn=self.pull_request.existing_pr_url_fn, + existing_pull_request_fn=self.pull_request.existing_pull_request_fn, + update_pull_request_base_fn=self.pull_request.update_pull_request_base_fn, probe_dirty_worktree_fn=self.probe_dirty_worktree, run_commit_action_fn=self.pull_request.run_commit_action_fn, pr_title_fn=self.pull_request.pr_title_fn, diff --git a/python/envctl_engine/actions/project_action_workflows.py b/python/envctl_engine/actions/project_action_workflows.py index e516c7d4..9cf3d43e 100644 --- a/python/envctl_engine/actions/project_action_workflows.py +++ b/python/envctl_engine/actions/project_action_workflows.py @@ -7,6 +7,7 @@ from typing import Any import envctl_engine.actions.action_commit_support as commit_support +from envctl_engine.actions.action_git_state_support import ExistingPullRequest import envctl_engine.actions.action_pr_workflow_support as pr_workflow_support import envctl_engine.actions.action_review_workflow_support as review_workflow_support import envctl_engine.actions.action_ship_support as ship_support @@ -45,6 +46,8 @@ class ProjectActionCommitWorkflowDependencies: class ProjectActionPullRequestWorkflowDependencies: resolve_base_branch_fn: Callable[[Any, Path], str] existing_pr_url_fn: Callable[[Path, str], str] + existing_pull_request_fn: Callable[[Path, str], ExistingPullRequest | None] + update_pull_request_base_fn: Callable[[Path, ExistingPullRequest, str], subprocess.CompletedProcess[str]] probe_dirty_worktree_fn: Callable[[Path, Path, str], DirtyWorktreeReport] run_commit_action_fn: Callable[[Any], int] pr_title_fn: Callable[[Any, Path, str], str] @@ -126,7 +129,8 @@ def run_pr_action(self, context: Any) -> int: git_available=self.git_available, git_output_fn=self.git.git_output_fn, resolve_base_branch_fn=self.pull_request.resolve_base_branch_fn, - existing_pr_url_fn=self.pull_request.existing_pr_url_fn, + existing_pull_request_fn=self.pull_request.existing_pull_request_fn, + update_pull_request_base_fn=self.pull_request.update_pull_request_base_fn, probe_dirty_worktree_fn=self.pull_request.probe_dirty_worktree_fn, run_commit_action_fn=self.pull_request.run_commit_action_fn, pr_title_fn=self.pull_request.pr_title_fn, @@ -151,6 +155,7 @@ def run_ship_action(self, context: Any) -> int: add_ship_pr_label=self.pull_request.add_ship_pr_label_fn, probe_dirty_worktree=self._probe_dirty_worktree_for_ship, existing_pr_url=self.pull_request.existing_pr_url_fn, + existing_pull_request=self.pull_request.existing_pull_request_fn, partition_envctl_protected_paths=self.commit.partition_envctl_protected_paths_fn, ordered_unique_paths=self.commit.ordered_unique_paths_fn, github_pr_checks=self.pull_request.github_pr_checks_fn, diff --git a/python/envctl_engine/config/__init__.py b/python/envctl_engine/config/__init__.py index 00399fbe..5f8fb637 100644 --- a/python/envctl_engine/config/__init__.py +++ b/python/envctl_engine/config/__init__.py @@ -220,9 +220,6 @@ def load_config(env: Mapping[str, str] | None = None) -> EngineConfig: runtime_scope_id = _runtime_scope_id(base_dir=base_dir, env=env, resolved=resolved) runtime_scope_dir = runtime_dir / "python-engine" / runtime_scope_id - runtime_dir.mkdir(parents=True, exist_ok=True) - runtime_scope_dir.mkdir(parents=True, exist_ok=True) - dependency_ports: dict[str, dict[str, int]] = {} for definition in _dependency_definitions(): resources: dict[str, int] = {} @@ -283,9 +280,11 @@ def load_config(env: Mapping[str, str] | None = None) -> EngineConfig: plan_agent_cmux_workspace = str(resolved.get("ENVCTL_PLAN_AGENT_CMUX_WORKSPACE", "") or "").strip() plan_agent_superset_project = str(resolved.get("ENVCTL_PLAN_AGENT_SUPERSET_PROJECT", "") or "").strip() plan_agent_superset_workspace = str(resolved.get("ENVCTL_PLAN_AGENT_SUPERSET_WORKSPACE", "") or "").strip() - plan_agent_terminals_enable = parse_bool(resolved.get("ENVCTL_PLAN_AGENT_TERMINALS_ENABLE"), False) or bool( - plan_agent_cmux_workspace - ) or bool(plan_agent_superset_project or plan_agent_superset_workspace) + plan_agent_terminals_enable = ( + parse_bool(resolved.get("ENVCTL_PLAN_AGENT_TERMINALS_ENABLE"), False) + or bool(plan_agent_cmux_workspace) + or bool(plan_agent_superset_project or plan_agent_superset_workspace) + ) additional_services, additional_service_errors = _parse_additional_services(resolved) supabase_auth_users, supabase_auth_user_errors = _parse_supabase_auth_users(resolved) @@ -358,8 +357,7 @@ def load_config(env: Mapping[str, str] | None = None) -> EngineConfig: plan_agent_cli_cmd=str(resolved.get("ENVCTL_PLAN_AGENT_CLI_CMD", "") or "").strip(), plan_agent_cmux_workspace=plan_agent_cmux_workspace, plan_agent_surface_transport=( - str(resolved.get("ENVCTL_PLAN_AGENT_SURFACE_TRANSPORT", "cmux") or "cmux").strip().lower() - or "cmux" + str(resolved.get("ENVCTL_PLAN_AGENT_SURFACE_TRANSPORT", "cmux") or "cmux").strip().lower() or "cmux" ), plan_agent_superset_project=plan_agent_superset_project, plan_agent_superset_workspace=plan_agent_superset_workspace, diff --git a/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py b/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py index 70343d00..43621fe4 100644 --- a/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py +++ b/python/envctl_engine/planning/plan_agent/workflow_runtime_addresses.py @@ -7,6 +7,7 @@ from envctl_engine.planning.plan_agent.models import CreatedPlanWorktree from envctl_engine.runtime.runtime_context import optional_state_repository +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RunState @@ -16,7 +17,7 @@ class RuntimeAddressPromptBuilder: worktree: CreatedPlanWorktree | None = None def prompt_section(self) -> str: - state = _latest_runtime_state(self.runtime) + state = _latest_runtime_state(self.runtime, worktree=self.worktree) if state is None: return "" lines = [ @@ -56,27 +57,44 @@ def _runtime_addresses_prompt_section(runtime: Any, *, worktree: CreatedPlanWork return RuntimeAddressPromptBuilder(runtime, worktree=worktree).prompt_section() -def _latest_runtime_state(runtime: Any) -> RunState | None: +def _latest_runtime_state(runtime: Any, *, worktree: CreatedPlanWorktree | None = None) -> RunState | None: + project_names = [worktree.name] if worktree is not None and str(worktree.name).strip() else None state_repository = optional_state_repository(runtime) if state_repository is not None and hasattr(state_repository, "load_latest"): - try: - state = state_repository.load_latest() - except Exception: - state = None - if isinstance(state, RunState): - return state - try_loader = getattr(runtime, "_try_load_existing_state", None) - if callable(try_loader): - for mode in ("trees", "main"): + for selected_projects in _state_lookup_project_attempts(project_names): try: - state = try_loader(mode=mode, strict_mode_match=True) + state = call_state_loader( + state_repository.load_latest, + project_names=selected_projects, + ) except Exception: state = None if isinstance(state, RunState): return state + try_loader = getattr(runtime, "_try_load_existing_state", None) + if callable(try_loader): + for mode in ("trees", "main"): + for selected_projects in _state_lookup_project_attempts(project_names): + try: + state = call_state_loader( + try_loader, + mode=mode, + strict_mode_match=True, + project_names=selected_projects, + ) + except Exception: + state = None + if isinstance(state, RunState): + return state return None +def _state_lookup_project_attempts(project_names: list[str] | None) -> tuple[list[str] | None, ...]: + if project_names is None: + return (None,) + return (project_names, None) + + def _dependency_address_lines(state: RunState, *, worktree: CreatedPlanWorktree | None = None) -> list[str]: rows: list[str] = [] seen: set[tuple[str, int]] = set() diff --git a/python/envctl_engine/planning/worktree_provenance.py b/python/envctl_engine/planning/worktree_provenance.py index aaf59fce..837aa142 100644 --- a/python/envctl_engine/planning/worktree_provenance.py +++ b/python/envctl_engine/planning/worktree_provenance.py @@ -8,6 +8,7 @@ from envctl_engine.planning.worktree_code_intelligence_files import ensure_worktree_git_excludes from envctl_engine.runtime.codex_tmux_support import _tmux_session_exists from envctl_engine.runtime.runtime_context import resolve_process_runtime +from envctl_engine.shared.git_branch_support import detect_preferred_default_branch from envctl_engine.shared.parsing import parse_bool @@ -165,13 +166,10 @@ def resolve_branch_ref(self: Any, *, source_branch: str) -> str: def detect_default_branch(self: Any) -> str: - ref = git_command_output(self, ["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]).strip() - if ref.startswith("origin/"): - return ref.split("origin/", 1)[1] - for candidate in ("main", "master"): - if git_command_output(self, ["rev-parse", "--verify", candidate]).strip(): - return candidate - return "main" + return detect_preferred_default_branch( + lambda args: git_command_output(self, list(args)), + probe_remote_heads=True, + ) def git_command_output(self: Any, args: list[str]) -> str: @@ -179,11 +177,16 @@ def git_command_output(self: Any, args: list[str]) -> str: def git_command_output_at(self: Any, cwd: Path, args: list[str]) -> str: + command_env = dict(self._command_env(port=0)) + timeout = 30.0 + if args[:1] == ["ls-remote"]: + command_env["GIT_TERMINAL_PROMPT"] = "0" + timeout = 10.0 result = resolve_process_runtime(self).run( ["git", "-C", str(cwd), *args], cwd=cwd, - env=self._command_env(port=0), - timeout=30.0, + env=command_env, + timeout=timeout, ) if getattr(result, "returncode", 1) != 0: return "" diff --git a/python/envctl_engine/pr_preview_controller.py b/python/envctl_engine/pr_preview_controller.py index 95df9c72..e86fe714 100644 --- a/python/envctl_engine/pr_preview_controller.py +++ b/python/envctl_engine/pr_preview_controller.py @@ -99,6 +99,7 @@ PUBLIC_ROUTE_HEALTH_DEADLINE_SECONDS = 30.0 PUBLIC_ROUTE_HEALTH_INTERVAL_SECONDS = 2.0 PUBLIC_ROUTE_REQUEST_TIMEOUT_SECONDS = 5.0 +PUBLIC_ROUTE_RESPONSE_BODY_LIMIT_BYTES = 64 * 1024 PLAN_AGENT_BOOLEAN_CONFIG_KEYS = { "ENVCTL_PLAN_AGENT_TERMINALS_ENABLE", "CMUX", @@ -652,6 +653,8 @@ def probe_public_route_url(url: str) -> str | None: timeout=PUBLIC_ROUTE_REQUEST_TIMEOUT_SECONDS, ) as response: status = int(response.status) + read = getattr(response, "read", None) + body = read(PUBLIC_ROUTE_RESPONSE_BODY_LIMIT_BYTES) if callable(read) else b"" except urllib.error.HTTPError as exc: return f"{url} returned HTTP {exc.code}" except urllib.error.URLError as exc: @@ -661,10 +664,35 @@ def probe_public_route_url(url: str) -> str | None: except OSError as exc: return f"{url} failed: {exc}" if 200 <= status < 400: + readiness_failure = semantic_readiness_failure(url, body) + if readiness_failure: + return readiness_failure return None return f"{url} returned HTTP {status}" +def semantic_readiness_failure(url: str, body: object) -> str | None: + if not isinstance(body, (bytes, bytearray)) or not body: + return None + try: + payload = json.loads(bytes(body).decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return None + if not isinstance(payload, dict) or "ready" not in payload: + return None + if payload["ready"] is True: + return None + + reason = payload.get("reason") + safe_reason = "" + if isinstance(reason, str): + safe_reason = re.sub(r"[^A-Za-z0-9_.:-]+", "_", reason).strip("_")[:80] + suffix = f" (reason={safe_reason})" if safe_reason else "" + if payload["ready"] is False: + return f"{url} reported ready=false{suffix}" + return f"{url} returned a non-boolean readiness value{suffix}" + + def public_route_health(endpoints: dict[str, Any]) -> PublicRouteHealth: routed_services = tuple( name @@ -1623,7 +1651,12 @@ def run_command(self, command: str, pr_number: int | None) -> int: raise SystemExit(f"command {command!r} requires --pr-number") pr = self.get_pr(pr_number) if command == "start": - return self.start(pr, reason="manual workflow dispatch", refresh_ttl=True) + return self.start( + pr, + reason="manual workflow dispatch", + refresh_ttl=True, + force_restart=True, + ) if command == "stop": return self.stop(pr, reason="manual workflow dispatch", comment=True) if command == "delete": @@ -1742,13 +1775,17 @@ def start( *, reason: str, refresh_ttl: bool = False, + force_restart: bool = False, ) -> int: self.validate_same_repo(pr) label_added_at = self.label_active_since(pr.number) or utc_now() if refresh_ttl: label_added_at = utc_now() existing_state = self.load_state(pr.number) - if self.current_head_preview_is_running(pr, existing_state): + if not force_restart and self.current_head_preview_is_running( + pr, + existing_state, + ): assert existing_state is not None updated_state = self.refresh_running_preview_state( existing_state, diff --git a/python/envctl_engine/runtime/engine_runtime_artifacts.py b/python/envctl_engine/runtime/engine_runtime_artifacts.py index aaf7d931..349d2801 100644 --- a/python/envctl_engine/runtime/engine_runtime_artifacts.py +++ b/python/envctl_engine/runtime/engine_runtime_artifacts.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Any, Protocol +from envctl_engine.runtime.runtime_context import optional_state_repository from envctl_engine.runtime.runtime_readiness import ( FEATURE_MATRIX_RELATIVE_PATH, GAP_REPORT_RELATIVE_PATH, @@ -60,36 +61,37 @@ def write_runtime_readiness_report( *, run_dir: Path | None = None, readiness_result: RuntimeReadinessResult | None = None, -) -> None: +) -> bool: started = time.monotonic() emit = getattr(runtime, "_emit", None) cached = _cached_runtime_readiness_payload(runtime) if readiness_result is None and cached is not None: - _write_runtime_readiness_payload(runtime, report_text=cached, run_dir=run_dir) + persisted = _write_runtime_readiness_payload(runtime, report_text=cached, run_dir=run_dir) if callable(emit): emit( "artifacts.runtime_readiness_report", duration_ms=round((time.monotonic() - started) * 1000.0, 2), used_cached_contract=True, run_dir=str(run_dir) if run_dir is not None else None, + persisted=persisted, ) - return + return persisted readiness = ( - readiness_result - if readiness_result is not None - else evaluate_runtime_readiness(runtime.config.base_dir) + readiness_result if readiness_result is not None else evaluate_runtime_readiness(runtime.config.base_dir) ) report_payload = build_runtime_readiness_report(readiness) report_text = json.dumps(report_payload, indent=2, sort_keys=True) - _write_runtime_readiness_payload(runtime, report_text=report_text, run_dir=run_dir) + persisted = _write_runtime_readiness_payload(runtime, report_text=report_text, run_dir=run_dir) if callable(emit): emit( "artifacts.runtime_readiness_report", duration_ms=round((time.monotonic() - started) * 1000.0, 2), used_cached_contract=readiness_result is not None, run_dir=str(run_dir) if run_dir is not None else None, + persisted=persisted, ) + return persisted def _write_runtime_readiness_payload( @@ -97,12 +99,24 @@ def _write_runtime_readiness_payload( *, report_text: str, run_dir: Path | None, -) -> None: +) -> bool: + repository = optional_state_repository(runtime) + repository_writer = getattr(repository, "write_runtime_artifact", None) + if callable(repository_writer): + return bool( + repository_writer( + run_id=run_dir.name if run_dir is not None else None, + artifact_name="runtime_readiness_report.json", + text=report_text, + ) + ) + (runtime.runtime_root / "runtime_readiness_report.json").write_text(report_text, encoding="utf-8") runtime.runtime_legacy_root.mkdir(parents=True, exist_ok=True) (runtime.runtime_legacy_root / "runtime_readiness_report.json").write_text(report_text, encoding="utf-8") if run_dir is not None: (run_dir / "runtime_readiness_report.json").write_text(report_text, encoding="utf-8") + return True def _write_cached_runtime_readiness_payload( @@ -110,9 +124,9 @@ def _write_cached_runtime_readiness_payload( *, run_dir: Path, cached: str, -) -> None: +) -> bool: started = time.monotonic() - _write_runtime_readiness_payload(runtime, report_text=cached, run_dir=run_dir) + persisted = _write_runtime_readiness_payload(runtime, report_text=cached, run_dir=run_dir) emit = getattr(runtime, "_emit", None) if callable(emit): emit( @@ -120,10 +134,12 @@ def _write_cached_runtime_readiness_payload( duration_ms=round((time.monotonic() - started) * 1000.0, 2), used_cached_contract=True, run_dir=str(run_dir), + persisted=persisted, ) + return persisted -def _write_pending_runtime_readiness_payload(runtime: Any, *, run_dir: Path) -> None: +def _write_pending_runtime_readiness_payload(runtime: Any, *, run_dir: Path) -> bool: report = { "passed": None, "pending": True, @@ -139,23 +155,29 @@ def _write_pending_runtime_readiness_payload(runtime: Any, *, run_dir: Path) -> "errors": [], "warnings": [], } - _write_runtime_readiness_payload( + persisted = _write_runtime_readiness_payload( runtime, report_text=json.dumps(report, indent=2, sort_keys=True), run_dir=run_dir, ) emit = getattr(runtime, "_emit", None) if callable(emit): - emit("artifacts.runtime_readiness_report.pending", run_dir=str(run_dir)) + emit("artifacts.runtime_readiness_report.pending", run_dir=str(run_dir), persisted=persisted) + return persisted def _start_background_runtime_readiness_report(runtime: Any, *, run_dir: Path) -> None: def _worker() -> None: try: - write_runtime_readiness_report(runtime, run_dir=run_dir) + persisted = write_runtime_readiness_report(runtime, run_dir=run_dir) emit = getattr(runtime, "_emit", None) if callable(emit): - emit("artifacts.runtime_readiness_report.complete", run_dir=str(run_dir)) + event = ( + "artifacts.runtime_readiness_report.complete" + if persisted + else "artifacts.runtime_readiness_report.discarded" + ) + emit(event, run_dir=str(run_dir)) except Exception as exc: # pragma: no cover emit = getattr(runtime, "_emit", None) if callable(emit): diff --git a/python/envctl_engine/runtime/engine_runtime_commands.py b/python/envctl_engine/runtime/engine_runtime_commands.py index 49fbddf5..a5043763 100644 --- a/python/envctl_engine/runtime/engine_runtime_commands.py +++ b/python/envctl_engine/runtime/engine_runtime_commands.py @@ -4,6 +4,7 @@ from pathlib import Path import shlex import shutil +import sys from typing import Any, Mapping from envctl_engine.runtime.command_resolution import resolve_requirement_start_command, resolve_service_start_command @@ -150,14 +151,14 @@ def split_command( def _command_exists_for_cwd(runtime: Any, executable: str, *, cwd: str | Path | None) -> bool: - if runtime._command_exists(executable): - return True - if cwd is None or ("/" not in executable and "\\" not in executable): - return False + if "/" not in executable and "\\" not in executable: + return bool(runtime._command_exists(executable)) candidate = Path(executable).expanduser() - if candidate.is_absolute(): - return candidate.exists() - return (Path(cwd) / candidate).exists() + if not candidate.is_absolute(): + if cwd is None: + return False + candidate = Path(cwd) / candidate + return candidate.is_file() and os.access(candidate, os.X_OK) def command_env(runtime: Any, *, port: int, extra: Mapping[str, str] | None = None) -> dict[str, str]: @@ -166,6 +167,15 @@ def command_env(runtime: Any, *, port: int, extra: Mapping[str, str] | None = No env["PORT"] = str(port) if extra: env.update(extra) + # Every envctl process necessarily has one known-good Python executable, + # even on minimal hosts that intentionally provide no ``python`` shim. + # Repository-owned launchers can use this explicit path to bootstrap their + # own isolated environments without guessing which command name exists. + python_executable = str(sys.executable or "").strip() + if python_executable: + env["ENVCTL_PYTHON_EXECUTABLE"] = python_executable + else: + env.pop("ENVCTL_PYTHON_EXECUTABLE", None) for key in SERVICE_LAUNCH_ENV_REMOVALS: env.pop(key, None) return env diff --git a/python/envctl_engine/runtime/engine_runtime_construction.py b/python/envctl_engine/runtime/engine_runtime_construction.py index 70531d22..88177330 100644 --- a/python/envctl_engine/runtime/engine_runtime_construction.py +++ b/python/envctl_engine/runtime/engine_runtime_construction.py @@ -30,9 +30,23 @@ def initialize_runtime_construction(runtime: Any, config: EngineConfig, *, env: runtime.env = dict(env or {}) runtime.runtime_legacy_root = config.runtime_dir / "python-engine" runtime.runtime_root = config.runtime_scope_dir - runtime.runtime_legacy_root.mkdir(parents=True, exist_ok=True) - runtime.runtime_root.mkdir(parents=True, exist_ok=True) - runtime._ensure_legacy_lock_view() + compat_mode = runtime._state_compat_mode() + runtime.state_repository = RuntimeStateRepository( + runtime_root=runtime.runtime_root, + runtime_legacy_root=runtime.runtime_legacy_root, + runtime_dir=runtime.config.runtime_dir, + runtime_scope_id=runtime.config.runtime_scope_id, + compat_mode=compat_mode, + ) + # The repository canonicalizes ancestor symlinks before applying its + # containment and ownership checks. Keep every runtime consumer on that + # same canonical identity so persisted artifact paths and caller-visible + # paths never disagree (for example, macOS /var versus /private/var). + runtime.runtime_root = runtime.state_repository.runtime_root + runtime.runtime_legacy_root = runtime.state_repository.runtime_legacy_root + runtime.state_repository.ensure_runtime_roots() + if compat_mode == RuntimeStateRepository.COMPAT_READ_WRITE: + runtime._ensure_legacy_lock_view() runtime.port_planner = PortPlanner( backend_base=config.backend_port_base, frontend_base=config.frontend_port_base, @@ -61,7 +75,6 @@ def initialize_runtime_construction(runtime: Any, config: EngineConfig, *, env: ), ) runtime.requirements = RequirementsOrchestrator() - runtime.services = ServiceManager() runtime.events = [] runtime._emit_lock = threading.Lock() runtime._emit_listeners = [] @@ -69,9 +82,11 @@ def initialize_runtime_construction(runtime: Any, config: EngineConfig, *, env: runtime._startup_warnings_by_project = {} runtime._debug_hash_salt = uuid.uuid4().hex runtime._debug_recorder = None + runtime._debug_recorder_failure = None runtime._active_command_id = None runtime._last_debug_bundle_path = None runtime.process_runner = ProcessRunner(emit=runtime._emit) + runtime.services = ServiceManager(process_runner=runtime.process_runner) probe_backend_name = "psutil" if runtime._probe_psutil_enabled() else "shell" probe_backend = runtime._build_process_probe_backend() runtime.process_probe = ProcessProbe(probe_backend) @@ -89,13 +104,6 @@ def initialize_runtime_construction(runtime: Any, config: EngineConfig, *, env: "backend": runtime._conflict_count("BACKEND"), "frontend": runtime._conflict_count("FRONTEND"), } - runtime.state_repository = RuntimeStateRepository( - runtime_root=runtime.runtime_root, - runtime_legacy_root=runtime.runtime_legacy_root, - runtime_dir=runtime.config.runtime_dir, - runtime_scope_id=runtime.config.runtime_scope_id, - compat_mode=runtime._state_compat_mode(), - ) runtime.runtime_context = RuntimeContext( config=runtime.config, env=runtime.env, diff --git a/python/envctl_engine/runtime/engine_runtime_dispatch.py b/python/envctl_engine/runtime/engine_runtime_dispatch.py index 3254549a..7d36cf34 100644 --- a/python/envctl_engine/runtime/engine_runtime_dispatch.py +++ b/python/envctl_engine/runtime/engine_runtime_dispatch.py @@ -2,13 +2,35 @@ from typing import Any -from .command_policy import dispatch_family_for_command +from .command_policy import ( + LIFECYCLE_CLEANUP_COMMANDS, + STARTUP_COMMANDS, + dispatch_family_for_command, +) from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.inspection_support import dispatch_direct_inspection +from envctl_engine.runtime.lifecycle_operation_lease import run_exclusive_lifecycle_operation from envctl_engine.runtime.utility_command_support import dispatch_utility_command from envctl_engine.shared.process_probe import ProcessProbe +_EXCLUSIVE_OPERATION_COMMANDS = frozenset( + { + *LIFECYCLE_CLEANUP_COMMANDS, + *STARTUP_COMMANDS, + "blast-worktree", + "delete-worktree", + "ensure-worktree", + "resume", + "self-destruct-worktree", + } +) + + +def _requires_exclusive_operation(command: str) -> bool: + return command in _EXCLUSIVE_OPERATION_COMMANDS + + def dispatch(runtime: Any, route: Route) -> int: runtime.process_probe = ProcessProbe(runtime._build_process_probe_backend()) effective_mode = getattr(route, "mode", "") @@ -28,7 +50,14 @@ def dispatch(runtime: Any, route: Route) -> int: effective_mode=effective_mode, command=getattr(route, "command", ""), ) - return dispatch_command(runtime, route) + command = str(getattr(route, "command", "")).strip() + if not _requires_exclusive_operation(command): + return dispatch_command(runtime, route) + return run_exclusive_lifecycle_operation( + runtime, + command=command, + operation=lambda: dispatch_command(runtime, route), + ) def dispatch_command(runtime: Any, route: Route) -> int: diff --git a/python/envctl_engine/runtime/engine_runtime_event_support.py b/python/envctl_engine/runtime/engine_runtime_event_support.py index 53e2a968..7c8bbce0 100644 --- a/python/envctl_engine/runtime/engine_runtime_event_support.py +++ b/python/envctl_engine/runtime/engine_runtime_event_support.py @@ -6,7 +6,7 @@ from typing import Any, Mapping from envctl_engine.runtime.command_router import Route -from envctl_engine.runtime.runtime_context import run_dir_path +from envctl_engine.runtime.runtime_context import optional_state_repository, run_dir_path from envctl_engine.debug.debug_contract import apply_debug_event_contract from envctl_engine.debug.debug_utils import debug_env_value, hash_command, hash_value from envctl_engine.shared.parsing import parse_bool, parse_int @@ -70,7 +70,14 @@ def emit(runtime: Any, event_name: str, **payload: object) -> None: recorder_payload = dict(safe_payload) component_value = recorder_payload.pop("component", None) component: str | None = component_value if isinstance(component_value, str) else None - runtime._debug_recorder.record(event_name, component=component, **recorder_payload) + try: + runtime._debug_recorder.record(event_name, component=component, **recorder_payload) + except Exception as exc: # noqa: BLE001 + # Debug telemetry must never interrupt lifecycle ownership changes. + # Disable a failed recorder so a read-only diagnostic sink cannot + # repeatedly destabilize the command that it is observing. + runtime._debug_recorder_failure = str(exc) + runtime._debug_recorder = None for listener in list(runtime._emit_listeners): try: listener(event_name, dict(safe_payload)) @@ -112,6 +119,16 @@ def auto_debug_pack(runtime: Any, *, reason: str) -> None: def persist_events_snapshot(runtime: Any) -> None: events_text = "".join(json.dumps(event, sort_keys=True) + "\n" for event in runtime.events) + repository = optional_state_repository(runtime) + repository_writer = getattr(repository, "write_runtime_artifact", None) + if callable(repository_writer): + repository_writer( + run_id=_current_run_id(runtime), + artifact_name="events.jsonl", + text=events_text, + ) + return + (runtime.runtime_root / "events.jsonl").write_text(events_text, encoding="utf-8") runtime.runtime_legacy_root.mkdir(parents=True, exist_ok=True) (runtime.runtime_legacy_root / "events.jsonl").write_text(events_text, encoding="utf-8") @@ -121,6 +138,17 @@ def persist_events_snapshot(runtime: Any) -> None: def _current_run_dir(runtime: Any) -> Path | None: + run_id = _current_run_id(runtime) + if run_id is None: + return None + try: + candidate = run_dir_path(runtime, run_id) + except Exception: + return None + return candidate + + +def _current_run_id(runtime: Any) -> str | None: env = getattr(runtime, "env", None) if not isinstance(env, dict): return None @@ -130,11 +158,7 @@ def _current_run_dir(runtime: Any) -> Path | None: run_id = raw_run_id.strip() if not run_id: return None - try: - candidate = run_dir_path(runtime, run_id) - except Exception: - return None - return candidate + return run_id def current_session_id(runtime: Any) -> str | None: diff --git a/python/envctl_engine/runtime/engine_runtime_lifecycle_facade.py b/python/envctl_engine/runtime/engine_runtime_lifecycle_facade.py index 2764266c..44f0984a 100644 --- a/python/envctl_engine/runtime/engine_runtime_lifecycle_facade.py +++ b/python/envctl_engine/runtime/engine_runtime_lifecycle_facade.py @@ -27,8 +27,8 @@ class RuntimeLifecycleFacadeMixin: def _release_port_session(self) -> None: runtime_release_port_session(self) - def _clear_runtime_state(self, *, command: str, aggressive: bool = False, route: Route | None = None) -> None: - self.lifecycle_cleanup_orchestrator.clear_runtime_state( + def _clear_runtime_state(self, *, command: str, aggressive: bool = False, route: Route | None = None) -> bool: + return self.lifecycle_cleanup_orchestrator.clear_runtime_state( command=command, aggressive=aggressive, route=route, @@ -54,8 +54,8 @@ def _blast_all_kill_orchestrator_processes(self) -> None: def _blast_all_docker_cleanup(self, *, route: Route | None) -> int: return self.lifecycle_cleanup_orchestrator.blast_all_docker_cleanup(route=route) - def _terminate_started_services(self, services: dict[str, object]) -> None: - runtime_terminate_started_services(self, services) + def _terminate_started_services(self, services: dict[str, object]) -> set[str]: + return runtime_terminate_started_services(self, services) def _terminate_services_from_state( self, @@ -64,8 +64,8 @@ def _terminate_services_from_state( selected_services: set[str] | None, aggressive: bool, verify_ownership: bool, - ) -> None: - runtime_terminate_services_from_state( + ) -> set[str]: + return runtime_terminate_services_from_state( self, state, selected_services=selected_services, diff --git a/python/envctl_engine/runtime/engine_runtime_service_truth.py b/python/envctl_engine/runtime/engine_runtime_service_truth.py index 1ada806a..0df85d2d 100644 --- a/python/envctl_engine/runtime/engine_runtime_service_truth.py +++ b/python/envctl_engine/runtime/engine_runtime_service_truth.py @@ -28,6 +28,7 @@ STARTUP_FAILURE_TOKENS, STARTUP_PROGRESS_TOKENS, command_result_error_text, + service_listener_failure_class, service_listener_failure_detail, tail_log_error_line, tail_log_startup_progress_line, @@ -50,6 +51,7 @@ "rebind_stale_service_pid", "refresh_service_listener_pids", "service_display_name", + "service_listener_failure_class", "service_listener_failure_detail", "service_startup_progress_timeout", "service_truth_discovery", diff --git a/python/envctl_engine/runtime/engine_runtime_startup_facade.py b/python/envctl_engine/runtime/engine_runtime_startup_facade.py index bd84ea62..2ed209e1 100644 --- a/python/envctl_engine/runtime/engine_runtime_startup_facade.py +++ b/python/envctl_engine/runtime/engine_runtime_startup_facade.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Sequence from pathlib import Path from typing import Any, Mapping, cast @@ -208,10 +209,21 @@ def _write_runtime_readiness_report( ) -> None: runtime_write_runtime_readiness_report(self, run_dir=run_dir, readiness_result=readiness_result) - def _try_load_existing_state(self, *, mode: str | None = None, strict_mode_match: bool = False) -> RunState | None: + def _try_load_existing_state( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, + ) -> RunState | None: return cast( RunState | None, - runtime_try_load_existing_state(self, mode=mode, strict_mode_match=strict_mode_match), + runtime_try_load_existing_state( + self, + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ), ) def _state_matches_scope(self, state: RunState) -> bool: diff --git a/python/envctl_engine/runtime/engine_runtime_startup_support.py b/python/envctl_engine/runtime/engine_runtime_startup_support.py index 9a4abb52..0cddda46 100644 --- a/python/envctl_engine/runtime/engine_runtime_startup_support.py +++ b/python/envctl_engine/runtime/engine_runtime_startup_support.py @@ -1,8 +1,10 @@ from __future__ import annotations +import threading from pathlib import Path from typing import Any, Callable, Mapping +from envctl_engine.requirements.core import dependency_definitions from envctl_engine.runtime.command_router import Route from envctl_engine.startup.run_reuse_support import ( build_startup_identity_metadata as build_startup_identity_metadata_impl, @@ -11,13 +13,24 @@ state_has_resumable_services as state_has_resumable_services_impl, ) from envctl_engine.startup.protocols import ProjectContextLike +from envctl_engine.startup.service_execution_environment import configured_service_types_for_mode +from envctl_engine.startup.service_execution_policy import additional_service_enabled_for_context +from envctl_engine.startup.startup_selection_support import _restart_service_types_for_project from envctl_engine.state.models import PortPlan, RunState from envctl_engine.shared.parsing import parse_bool, parse_int from envctl_engine.planning import discover_tree_projects +_PORT_REAP_LOCK = threading.Lock() + + def effective_start_mode(runtime: Any, route: Route) -> str: - if route.mode == "main" and runtime._setup_worktree_requested(route): + setup_worktree_requested = getattr(runtime, "_setup_worktree_requested", None) + if ( + route.mode == "main" + and callable(setup_worktree_requested) + and setup_worktree_requested(route) + ): return "trees" return route.mode @@ -197,7 +210,22 @@ def discover_projects( def reserve_project_ports(runtime: Any, context: ProjectContextLike, route: Route | None = None) -> None: + _reap_stale_port_locks_once(runtime) + selected_app_services = _selected_app_services_for_port_reservation(runtime, context=context, route=route) for service_name, plan in context.ports.items(): + if not _port_reservation_enabled( + runtime, + service_name=service_name, + route=route, + selected_app_services=selected_app_services, + ): + runtime._emit( + "port.reservation.skipped", + project=context.name, + service=service_name, + reason="startup_surface_disabled", + ) + continue owner = f"{context.name}:{service_name}" restart_preferred_port = plan.assigned if getattr(plan, "source", "") == "restart" else None restart_conflict_detail = None @@ -245,6 +273,85 @@ def reserve_project_ports(runtime: Any, context: ProjectContextLike, route: Rout runtime._emit("port.reserved", project=context.name, service=service_name, port=reserved) +def _reap_stale_port_locks_once(runtime: Any) -> None: + reaper = getattr(runtime.port_planner, "reap_stale_locks", None) + if not callable(reaper) or bool(getattr(runtime, "_startup_stale_port_locks_reaped", False)): + return + with _PORT_REAP_LOCK: + if bool(getattr(runtime, "_startup_stale_port_locks_reaped", False)): + return + reaped = int(reaper() or 0) + setattr(runtime, "_startup_stale_port_locks_reaped", True) + runtime._emit("port.lock.reap_stale", count=reaped) + + +def _port_reservation_enabled( + runtime: Any, + *, + service_name: str, + route: Route | None, + selected_app_services: set[str] | None, +) -> bool: + if route is None: + return True + runtime_mode = effective_start_mode(runtime, route) + + dependency_ids = { + definition.id + for definition in dependency_definitions() + if any(resource.legacy_port_key == service_name for resource in definition.resources) + } + if dependency_ids: + if route.flags.get("launch_dependencies") is False: + return False + enabled = getattr(runtime, "_requirement_enabled", None) + if not callable(enabled): + return True + return any( + bool(enabled(dependency_id, mode=runtime_mode, route=route)) + for dependency_id in dependency_ids + ) + + return selected_app_services is None or service_name in selected_app_services + + +def _selected_app_services_for_port_reservation( + runtime: Any, + *, + context: ProjectContextLike, + route: Route | None, +) -> set[str] | None: + if route is None: + return None + runtime_mode = effective_start_mode(runtime, route) + try: + configured_service_types = configured_service_types_for_mode( + runtime, + runtime_mode, + Path(context.root), + ) + except AttributeError: + dependency_port_names = { + resource.legacy_port_key for definition in dependency_definitions() for resource in definition.resources + } + configured_service_types = set(context.ports).difference(dependency_port_names) + configured_additional_services = tuple( + service + for service in getattr(getattr(runtime, "config", None), "additional_services", ()) + if additional_service_enabled_for_context( + service, + mode=runtime_mode, + project_root=Path(context.root), + ) + ) + return _restart_service_types_for_project( + route=route, + project_name=context.name, + default_service_types=configured_service_types, + additional_services=configured_additional_services, + ) + + def _release_restart_preferred_lock(port_planner: Any, port: int, *, owner: str) -> None: release = getattr(port_planner, "release", None) if not callable(release): diff --git a/python/envctl_engine/runtime/engine_runtime_state_lookup.py b/python/envctl_engine/runtime/engine_runtime_state_lookup.py index 6aa0a371..5f1f8219 100644 --- a/python/envctl_engine/runtime/engine_runtime_state_lookup.py +++ b/python/envctl_engine/runtime/engine_runtime_state_lookup.py @@ -1,15 +1,24 @@ from __future__ import annotations +from collections.abc import Sequence from typing import Any +from envctl_engine.state.lookup import call_state_loader + def try_load_existing_state( runtime: Any, *, mode: str | None = None, strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, ) -> object | None: - state = runtime.state_repository.load_latest(mode=mode, strict_mode_match=strict_mode_match) + state = call_state_loader( + runtime.state_repository.load_latest, + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ) if state is not None: runtime._emit( "state.fingerprint.after_reload", diff --git a/python/envctl_engine/runtime/help_topic_actions.py b/python/envctl_engine/runtime/help_topic_actions.py index 84b358a6..74e8f7e3 100644 --- a/python/envctl_engine/runtime/help_topic_actions.py +++ b/python/envctl_engine/runtime/help_topic_actions.py @@ -68,6 +68,7 @@ usage=("envctl pr [--project ] [--pr-base ]",), what_it_does=( "resolves the current branch, checks for an existing PR, and creates one with gh or repo helper scripts", + "defaults the PR base to dev, then main, then master, while --pr-base remains an explicit override", "commits and pushes dirty worktree changes first using envctl commit behavior when needed", "uses MAIN_TASK.md or commit history to build a useful PR title/body unless explicit env is provided", ), @@ -76,7 +77,7 @@ "--pr-base set the target base branch", "--interactive opt into prompts/selection instead of default headless action mode", ), - examples=("envctl pr --project feature-a-1 --pr-base main", "envctl pr --main --pr-base main"), + examples=("envctl pr --project feature-a-1", "envctl pr --main --pr-base main"), aliases=("prs", "p", "--pr", "--prs"), related=("commit", "review", "test"), ), diff --git a/python/envctl_engine/runtime/help_topic_lifecycle.py b/python/envctl_engine/runtime/help_topic_lifecycle.py index 4de80036..9d6b2c8a 100644 --- a/python/envctl_engine/runtime/help_topic_lifecycle.py +++ b/python/envctl_engine/runtime/help_topic_lifecycle.py @@ -39,9 +39,7 @@ "restart": CommandHelpTopic( command="restart", summary="restart selected envctl-managed services", - usage=( - "envctl restart [--project |--service |--all] [runtime scope] [--headless]", - ), + usage=("envctl restart [--project |--service |--all] [runtime scope] [--headless]",), what_it_does=( "loads saved runtime state, stops the selected services, and starts them again", "keeps the same target model as startup while using existing envctl state as the source of truth", @@ -95,12 +93,17 @@ "stop": CommandHelpTopic( command="stop", summary="stop selected envctl-managed services and update runtime state", - usage=("envctl stop [--project |--service |--all] [runtime scope]",), + usage=("envctl stop [--main|--trees] [--project |--service |--all] [runtime scope]",), what_it_does=( "loads saved runtime state and terminates selected services", - "supports service-scope flags so you can stop backend, frontend, dependencies, or the entire system", + ( + "dependency scope releases saved dependency records and port locks; " + "managed Docker stacks and storage remain reusable" + ), + "use stop-all --remove-volumes or blast-all when managed Docker containers must be removed", ), flags=( + "--main | --trees choose the saved runtime mode; an explicit mode leaves the other mode active", "--project stop services for selected project/worktree", "--service stop one saved service", "--all stop all matching saved services", @@ -117,14 +120,19 @@ "stop-all": CommandHelpTopic( command="stop-all", summary="stop every saved envctl-managed service in the selected scope", - usage=("envctl stop-all [runtime scope]", "envctl kill-all [runtime scope]"), + usage=( + "envctl stop-all [--main|--trees] [runtime scope]", + "envctl kill-all [--main|--trees] [runtime scope]", + ), what_it_does=( - "terminates all saved services for the selected runtime state", - "can optionally remove volumes when supported by the selected cleanup flags", + "terminates all saved app services and releases dependency records and port locks", + "keeps managed Docker dependency stacks and storage reusable unless explicit removal cleanup is requested", + "use --remove-volumes for supported Docker removal, or blast-all for aggressive cleanup", ), flags=( + "--main | --trees choose the saved runtime mode; an explicit mode leaves the other mode active", "--backend|--frontend|--fullstack|--dependencies|--entire-system choose cleanup scope", - "--stop-all-remove-volumes / --remove-volumes remove supported volumes too", + "--stop-all-remove-volumes / --remove-volumes remove supported managed containers and volumes", ), examples=("envctl stop-all", "envctl kill-all", "envctl stop-all --entire-system --remove-volumes"), aliases=("kill-all", "killall", "stopall", "--stop-all", "--kill-all"), diff --git a/python/envctl_engine/runtime/inspection_support.py b/python/envctl_engine/runtime/inspection_support.py index a83f6c33..08d6f88b 100644 --- a/python/envctl_engine/runtime/inspection_support.py +++ b/python/envctl_engine/runtime/inspection_support.py @@ -10,6 +10,11 @@ from envctl_engine.config.persistence import managed_values_from_local_state, managed_values_to_payload from envctl_engine.planning import PlanProjectPrediction from envctl_engine.runtime.command_router import Route, list_supported_commands +from envctl_engine.runtime.state_inspection_support import ( + emit_cwd_runtime_mismatch_warnings as _emit_cwd_runtime_mismatch_warnings_impl, + print_state as _print_state_impl, + try_load_existing_state as _try_load_existing_state_impl, +) from envctl_engine.startup.startup_selection_support import ( _state_project_names as _state_project_names_impl, _tree_preselected_projects_from_state as _tree_preselected_projects_from_state_impl, @@ -20,15 +25,7 @@ print_attach_command, kill_session, ) -from envctl_engine.state import state_to_dict from envctl_engine.state.models import RunState -from envctl_engine.state.project_runtime import ( - cwd_runtime_warnings, - dependency_mode_summary, - project_resolution_event_payload, - resolve_requested_project_state, -) -from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.runtime.endpoints_command_support import run_endpoints_command from envctl_engine.runtime.startup_inspection_support import ( additional_service_contexts as _additional_service_contexts_impl, @@ -87,14 +84,35 @@ def _print_targets(runtime: Any, route: Route, *, trees_only: bool) -> int: preselected: set[str] = set() state_present: set[str] = set() services_running: set[str] = set() + state_error: dict[str, str] | None = None if mode == "trees": - startup = getattr(runtime, "startup_orchestrator", None) - if startup is not None: - preselected = set(_tree_preselected_projects_from_state_impl(runtime=runtime, project_contexts=projects)) - state = _try_load_existing_state(runtime, mode="trees", strict_mode_match=True) - if state is not None: - state_present = set(_state_project_names_impl(runtime=runtime, state=state)) - services_running = _running_tree_project_names(runtime=runtime, state=state) + states, load_error = _load_all_existing_states(runtime, mode="trees") + if load_error is not None: + state_error = { + "code": "runtime_state_integrity_error", + "message": load_error, + } + else: + startup = getattr(runtime, "startup_orchestrator", None) + if startup is not None: + try: + preselected = set( + _tree_preselected_projects_from_state_impl(runtime=runtime, project_contexts=projects) + ) + except RuntimeError as exc: + # The selection helper performs its own strict state load. If + # the repository changes between the two reads, keep this + # inspection command useful without presenting partial state + # as trustworthy. + states = [] + preselected.clear() + state_error = { + "code": "runtime_state_integrity_error", + "message": str(exc), + } + for state in states: + state_present.update(_state_project_names_impl(runtime=runtime, state=state)) + services_running.update(_running_tree_project_names(runtime=runtime, state=state)) payload = { "mode": mode, @@ -112,6 +130,8 @@ def _print_targets(runtime: Any, route: Route, *, trees_only: bool) -> int: "running": project.name in services_running, }, ), + "state_error": state_error, + "warnings": [state_error] if state_error is not None else [], } print(json.dumps(payload, indent=2, sort_keys=True)) return 0 @@ -317,87 +337,42 @@ def _apply_runtime_effective_config_values(values: Any, config: Any) -> None: def _print_state(runtime: Any, route: Route, *, json_output: bool) -> int: - strict = runtime._state_lookup_strict_mode_match(route) - state = _try_load_existing_state(runtime, mode=route.mode, strict_mode_match=strict) - if state is None: - if json_output: - print(json.dumps({"found": False, "mode": getattr(route, "mode", None)}, indent=2, sort_keys=True)) - else: - print("No previous state found.") - return 1 - requested_projects = list(getattr(route, "projects", []) or []) - if requested_projects: - resolution = resolve_requested_project_state( - state, - requested_projects, - command="show-state", - runtime=runtime, - allow_multi=False, - ) - if not resolution.ok: - emitter = getattr(runtime, "_emit", None) - if callable(emitter): - emitter( - "state.project_resolution.failed", - **project_resolution_event_payload(resolution, state, runtime=runtime), - ) - if json_output: - print(json.dumps(resolution.payload(), indent=2, sort_keys=True)) - else: - print(f"Requested project is not running: {resolution.requested_project}") - print("Active runtime projects: " + (", ".join(resolution.active_projects) or "none")) - return 1 - emitter = getattr(runtime, "_emit", None) - if callable(emitter): - emitter( - "state.project_resolution.ok", - **project_resolution_event_payload(resolution, state, runtime=runtime), - ) - if resolution.state is not None and not bool(getattr(route, "flags", {}).get("all")): - state = resolution.state - - dependency_summary = dependency_mode_summary(state) - cwd_project, warnings = cwd_runtime_warnings( - state, - runtime=runtime, - requested_projects=list(getattr(route, "projects", []) or []), + return _print_state_impl( + runtime, + route, + json_output=json_output, + state_loader=_try_load_existing_state, + warning_emitter=_emit_cwd_runtime_mismatch_warnings, ) - _emit_cwd_runtime_mismatch_warnings(runtime, command="show-state", state=state, warnings=warnings) - payload: dict[str, Any] = { - "found": True, - "runtime_root": str(runtime.runtime_root), - "run_state_path": str(runtime._run_state_path()), - "state": state_to_dict(state), - "runtime_map": build_runtime_map(state), - "dependency_mode": dependency_summary["dependency_mode"], - "shared_dependencies": dependency_summary["shared_dependencies"], - "cwd_project": cwd_project, - "warnings": warnings, - } - if json_output: - print(json.dumps(payload, indent=2, sort_keys=True)) - return 0 - - state_payload = payload["state"] - print(f"run_id: {state_payload['run_id']}") - print(f"mode: {state_payload['mode']}") - print(f"services: {len(state_payload['services'])}") - print(f"requirements: {len(state_payload['requirements'])}") - for warning in warnings: - print(f"Warning: {warning['message']}") - print("run_state_path:") - print(render_path_for_terminal(payload["run_state_path"], env=getattr(runtime, "env", {}))) - return 0 +def _try_load_existing_state( + runtime: Any, + *, + mode: str | None, + strict_mode_match: bool, + project_names: list[str] | None = None, +) -> RunState | None: + return _try_load_existing_state_impl( + runtime, + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ) -def _try_load_existing_state(runtime: Any, *, mode: str | None, strict_mode_match: bool) -> RunState | None: - loader = getattr(runtime, "_try_load_existing_state", None) - if not callable(loader): - return None - state = loader(mode=mode, strict_mode_match=strict_mode_match) - return state if isinstance(state, RunState) else None +def _load_all_existing_states(runtime: Any, *, mode: str) -> tuple[list[RunState], str | None]: + repository = getattr(runtime, "state_repository", None) + load_all = getattr(repository, "load_all", None) + try: + if callable(load_all): + states = [state for state in load_all(mode=mode) if isinstance(state, RunState)] + if states: + return states, None + state = _try_load_existing_state(runtime, mode=mode, strict_mode_match=True) + except RuntimeError as exc: + return [], str(exc) + return ([state] if state is not None else []), None def _emit_cwd_runtime_mismatch_warnings( @@ -407,20 +382,12 @@ def _emit_cwd_runtime_mismatch_warnings( state: RunState, warnings: list[dict[str, object]], ) -> None: - emitter = getattr(runtime, "_emit", None) - if not callable(emitter): - return - for warning in warnings: - if str(warning.get("code") or "") != "cwd_runtime_mismatch": - continue - emitter( - "state.cwd_runtime_mismatch", - run_id=getattr(state, "run_id", ""), - mode=getattr(state, "mode", ""), - command=command, - cwd_project=warning.get("cwd_project"), - active_projects=warning.get("active_projects"), - ) + _emit_cwd_runtime_mismatch_warnings_impl( + runtime, + command=command, + state=state, + warnings=warnings, + ) def _additional_service_enabled_for_context(runtime: Any, mode: str, service: object, context: object) -> bool: diff --git a/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py b/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py index b2680e98..122df92c 100644 --- a/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py +++ b/python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py @@ -11,6 +11,7 @@ requirements_have_enabled_components, select_dependency_components_for_stop, ) +from envctl_engine.runtime.lifecycle_requirement_ports import requirement_component_port_owners from envctl_engine.runtime.lifecycle_service_stop_selection import ( select_services_for_stop, service_matches_runtime_scope, @@ -23,15 +24,17 @@ service_type_from_stopped_service, should_preserve_stopped_dashboard_state, ) +from envctl_engine.runtime.lifecycle_targeted_state import LifecycleTargetedStateSupport from envctl_engine.runtime.runtime_context import ( resolve_port_allocator, resolve_process_runtime, resolve_state_repository, ) from envctl_engine.shared.protocols import PortAllocator, ProcessRuntime, StateRepository -from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.runtime.command_router import Route +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RequirementsResult, RunState +from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.ui.selection_support import ( interactive_selection_allowed, project_names_from_state, @@ -43,7 +46,7 @@ from envctl_engine.ui.spinner_service import emit_spinner_policy, resolve_spinner_policy -class LifecycleCleanupOrchestrator(LifecycleBlastCleanupSupport): +class LifecycleCleanupOrchestrator(LifecycleTargetedStateSupport, LifecycleBlastCleanupSupport): def __init__(self, runtime: Any) -> None: self.runtime = runtime @@ -54,13 +57,16 @@ def execute(self, route: Route) -> int: def _execute_full_cleanup(self, route: Route) -> int: def operation() -> int: - self.clear_runtime_state( + completed = self.clear_runtime_state( command=route.command, aggressive=(route.command == "blast-all"), route=route, ) - print("Stopped runtime state.") - return 0 + if completed: + print("Stopped runtime state.") + return 0 + print("Runtime cleanup is incomplete; failed services remain tracked.") + return 1 message = ( "Stopping all services and runtime state..." @@ -75,77 +81,197 @@ def operation() -> int: ) def _execute_stop(self, route: Route) -> int: - state = self.runtime._try_load_existing_state( # type: ignore[attr-defined] + targeted_states = self._targeted_active_states(route) + if targeted_states: + result = 0 + for state in targeted_states: + result = max(result, self._execute_stop_state(route, state)) + return result + + state = call_state_loader( + self.runtime._try_load_existing_state, # type: ignore[attr-defined] mode=route.mode, strict_mode_match=self.runtime._state_lookup_strict_mode_match(route), # type: ignore[attr-defined] + project_names=route.projects or None, ) if state is None: print("No active runtime state found.") return 0 + return self._execute_stop_state(route, state) + + def _targeted_active_states(self, route: Route) -> list[RunState]: + selected_projects = {str(project).strip().casefold() for project in route.projects if str(project).strip()} + if not selected_projects: + return [] + repository = self._state_repository(self.runtime) + load_all = getattr(repository, "load_all", None) + if not callable(load_all): + return [] + matching_states = self._states_matching_projects( + [state for state in load_all(mode=route.mode) if isinstance(state, RunState)], + selected_projects, + ) + if matching_states: + return matching_states + if self.runtime._state_lookup_strict_mode_match(route): # type: ignore[attr-defined] + return [] + + fallback_state = call_state_loader( + self.runtime._try_load_existing_state, # type: ignore[attr-defined] + mode=route.mode, + strict_mode_match=False, + project_names=route.projects, + ) + if not isinstance(fallback_state, RunState) or fallback_state.mode == route.mode: + return [] + return self._states_matching_projects( + [state for state in load_all(mode=fallback_state.mode) if isinstance(state, RunState)], + selected_projects, + ) + + def _states_matching_projects( + self, + states: Iterable[RunState], + selected_projects: set[str], + ) -> list[RunState]: + return [ + state + for state in states + if selected_projects.intersection( + str(getattr(project, "name", project)).strip().casefold() + for project in self._project_names_from_state(state) + ) + ] + + def _execute_stop_state(self, route: Route, state: RunState) -> int: + if route.flags.get("runtime_scope") == "dependencies": return self._execute_stop_dependencies(route, state) + owned_projects = self._owned_project_names(state) + targeted_project_keys = frozenset( + str(project).strip().casefold() + for project in route.projects + if str(project).strip().casefold() in owned_projects + ) preserve_stopped_dashboard_state = self._should_preserve_stopped_dashboard_state(route) selected_dependencies = self._select_dependency_components_for_stop(state, route) selected = self._select_services_for_stop(state, route) if not selected and not selected_dependencies: - if not state.services and not state.requirements: - self.clear_runtime_state(command="stop", aggressive=False) + if targeted_project_keys and self._targeted_projects_have_no_managed_resources( + state, + targeted_project_keys, + ): + self._remove_targeted_requirements(state, targeted_project_keys) + state_remains = self._persist_targeted_project_state( + state, + owned_projects=owned_projects, + targeted_project_keys=targeted_project_keys, + ) + print("Stopped selected project state." if state_remains else "Stopped runtime state.") + return 0 + has_enabled_requirements = any( + self._requirements_have_enabled_components(requirements) for requirements in state.requirements.values() + ) + if not state.services and not has_enabled_requirements: + self._deactivate_runtime_state(state) print("Stopped runtime state.") return 0 print("No matching services selected for stop.") return 1 def operation() -> int: - if selected and preserve_stopped_dashboard_state: - self._remember_dashboard_stopped_services(state, selected) - + self.runtime._emit( # type: ignore[attr-defined] + "cleanup.stop", + aggressive=False, + run_id=state.run_id, + ) + failed_services: set[str] = set() if selected: - self.runtime._terminate_services_from_state( # type: ignore[attr-defined] + termination_result = self.runtime._terminate_services_from_state( # type: ignore[attr-defined] state, selected_services=selected, aggressive=False, verify_ownership=True, ) + if isinstance(termination_result, set): + failed_services = {str(name) for name in termination_result} - for service_name in list(selected): + stopped_services = selected.difference(failed_services) + if stopped_services and preserve_stopped_dashboard_state: + self._remember_dashboard_stopped_services(state, stopped_services) + + for service_name in stopped_services: state.services.pop(service_name, None) - if selected_dependencies: + if selected_dependencies and not failed_services: self._release_selected_dependency_components(state, selected_dependencies) - if not bool(route.flags.get("stop_preserve_requirements")): + if not failed_services and not bool(route.flags.get("stop_preserve_requirements")): remaining_projects = { - self.runtime._project_name_from_service(name) # type: ignore[attr-defined] - for name in state.services + self._project_name_for_service(name, service) for name, service in state.services.items() } - removed_projects = [project for project in state.requirements if project not in remaining_projects] + removed_projects = [ + project + for project, requirements in state.requirements.items() + if self._requirement_project_key(project, requirements) not in remaining_projects + and ( + not targeted_project_keys + or self._requirement_project_key(project, requirements) in targeted_project_keys + ) + ] for project in removed_projects: self.runtime._release_requirement_ports(state.requirements[project]) # type: ignore[attr-defined] state.requirements.pop(project, None) + if targeted_project_keys: + self._persist_targeted_project_state( + state, + owned_projects=owned_projects, + targeted_project_keys=targeted_project_keys, + ) + if failed_services: + print("Could not safely stop: " + ", ".join(sorted(failed_services))) + return 1 + if stopped_services and selected_dependencies: + print("Stopped selected services and dependencies.") + elif selected_dependencies: + print("Stopped selected dependencies.") + elif stopped_services: + print("Stopped selected services.") + else: + print("Stopped selected project state.") + return 0 + if not state.services and not state.requirements: if preserve_stopped_dashboard_state and self._has_dashboard_stopped_services(state): self._save_selected_stop_state(state) - if selected and selected_dependencies: + if stopped_services and selected_dependencies: print("Stopped selected services and dependencies.") elif selected_dependencies: print("Stopped selected dependencies.") - else: + elif stopped_services: print("Stopped selected services.") + else: + print("No selected service could be stopped.") return 0 - self.clear_runtime_state(command="stop", aggressive=False) + self._deactivate_runtime_state(state) print("Stopped runtime state.") return 0 self._save_selected_stop_state(state) - if selected and selected_dependencies: + if failed_services: + print("Could not safely stop: " + ", ".join(sorted(failed_services))) + return 1 + if stopped_services and selected_dependencies: print("Stopped selected services and dependencies.") elif selected_dependencies: print("Stopped selected dependencies.") - else: + elif stopped_services: print("Stopped selected services.") + else: + print("Stopped selected dependencies.") return 0 selected_count = len(selected) + sum(len(items) for items in selected_dependencies.values()) @@ -218,7 +344,7 @@ def _execute_stop_dependencies(self, route: Route, state: RunState) -> int: def operation() -> int: if not state.requirements: if not state.services: - self._purge_runtime_state(aggressive=False) + self._deactivate_runtime_state(state) print("No dependencies found.") return 0 @@ -229,7 +355,7 @@ def operation() -> int: if state.services: self._save_selected_stop_state(state) else: - self._purge_runtime_state(aggressive=False) + self._deactivate_runtime_state(state) print("Stopped dependencies.") return 0 @@ -255,9 +381,18 @@ def _release_selected_dependency_components( release_component_ports_fn=self._release_requirement_component_ports, ) - def _release_requirement_component_ports(self, component: Mapping[str, object]) -> None: + def _release_requirement_component_ports( + self, + requirements: RequirementsResult, + dependency_id: str, + component: Mapping[str, object], + ) -> None: port_allocator = self._port_allocator(self.runtime) - release_requirement_component_ports(component, release_port_fn=port_allocator.release) + release_requirement_component_ports( + component, + port_planner=port_allocator, + owner_candidates=requirement_component_port_owners(requirements, dependency_id), + ) @staticmethod def _requirements_have_enabled_components(requirements: object) -> bool: @@ -296,7 +431,7 @@ def _project_names_from_state(self, state: RunState) -> list[object]: def _services_from_selection(self, selection: TargetSelection, state: RunState) -> set[str]: return services_from_selection(self.runtime, selection, state) - def clear_runtime_state(self, *, command: str, aggressive: bool = False, route: Route | None = None) -> None: + def clear_runtime_state(self, *, command: str, aggressive: bool = False, route: Route | None = None) -> bool: rt = self.runtime if command == "stop-all": rt._emit("cleanup.stop_all", aggressive=aggressive) # type: ignore[attr-defined] @@ -304,14 +439,77 @@ def clear_runtime_state(self, *, command: str, aggressive: bool = False, route: rt._emit("cleanup.blast", aggressive=aggressive) # type: ignore[attr-defined] else: rt._emit("cleanup.stop", aggressive=aggressive) # type: ignore[attr-defined] - state = rt._try_load_existing_state() # type: ignore[attr-defined] - if state is not None: - rt._terminate_services_from_state( # type: ignore[attr-defined] + state_repository = self._state_repository(rt) + load_all = getattr(state_repository, "load_all", None) + mode_filter = None + if ( + command == "stop-all" + and route is not None + and rt._state_lookup_strict_mode_match(route) # type: ignore[attr-defined] + ): + mode_filter = route.mode + if callable(load_all): + states = [state for state in load_all(mode=mode_filter) if isinstance(state, RunState)] + else: + states = [] + if not states: + state = call_state_loader( + rt._try_load_existing_state, # type: ignore[attr-defined] + mode=mode_filter, + strict_mode_match=mode_filter is not None, + ) + states = [state] if isinstance(state, RunState) else [] + seen_run_ids: set[str] = set() + failed_by_run: dict[str, set[str]] = {} + for state in states: + if state.run_id in seen_run_ids: + continue + seen_run_ids.add(state.run_id) + termination_result = rt._terminate_services_from_state( # type: ignore[attr-defined] state, selected_services=None, aggressive=aggressive, verify_ownership=not aggressive, ) + failed_services = ( + {str(name) for name in termination_result} if isinstance(termination_result, set) else set() + ) + if not failed_services: + requirements_to_release = list(state.requirements.values()) + self._deactivate_runtime_state(state, release_session_when_empty=False) + for requirements in requirements_to_release: + self._release_requirement_ports(requirements) + continue + failed_by_run[state.run_id] = failed_services + state.services = {name: service for name, service in state.services.items() if name in failed_services} + failed_projects = { + project_name + for name, service in state.services.items() + if (project_name := self._project_name_for_service(name, service)) + } + requirements_to_release: list[RequirementsResult] = [] + retained_requirements: dict[str, RequirementsResult] = {} + for project, requirements in state.requirements.items(): + requirement_project = str(getattr(requirements, "project", "") or project).strip().casefold() + if not failed_projects or requirement_project in failed_projects: + retained_requirements[project] = requirements + else: + requirements_to_release.append(requirements) + state.requirements = {project: requirements for project, requirements in retained_requirements.items()} + state_repository.save_selected_stop_state( + state=state, + emit=rt._emit, # type: ignore[attr-defined] + runtime_map_builder=lambda raw_state: build_runtime_map(cast(RunState, raw_state)), + ) + for requirements in requirements_to_release: + self._release_requirement_ports(requirements) + + if failed_by_run: + rt._emit( # type: ignore[attr-defined] + "cleanup.incomplete", + failed_services={run_id: sorted(names) for run_id, names in failed_by_run.items()}, + ) + return False if aggressive and self.blast_all_ecosystem_enabled(): self.blast_all_ecosystem_cleanup(route=route) @@ -332,25 +530,68 @@ def clear_runtime_state(self, *, command: str, aggressive: bool = False, route: removed = self.blast_all_docker_cleanup(route=volume_route) rt._emit("cleanup.stop_all.remove_volumes.finish", removed=removed) # type: ignore[attr-defined] - self._purge_runtime_state(aggressive=aggressive) - if aggressive: + if aggressive or command != "stop-all" or not self._has_active_runs(state_repository): + self._purge_runtime_state(aggressive=aggressive) + repository = self._state_repository(rt) + if aggressive and getattr(repository, "compat_mode", None) == getattr( + repository, + "COMPAT_READ_WRITE", + "compat_read_write", + ): self.blast_all_purge_legacy_state_artifacts() + return True - def _save_selected_stop_state(self, state: RunState) -> None: - self._state_repository(self.runtime).save_selected_stop_state( - state=state, - emit=self.runtime._emit, # type: ignore[attr-defined] - runtime_map_builder=lambda raw_state: build_runtime_map(cast(RunState, raw_state)), - ) + @staticmethod + def _has_active_runs(repository: object) -> bool: + has_active_runs = getattr(repository, "has_active_runs", None) + if callable(has_active_runs): + return bool(has_active_runs()) + load_all = getattr(repository, "load_all", None) + return bool(load_all()) if callable(load_all) else True def _purge_runtime_state(self, *, aggressive: bool) -> None: - if aggressive: - self.release_all_runtime_ports() - else: - release_port_session = getattr(self.runtime, "_release_port_session", None) - if callable(release_port_session): - release_port_session() self._state_repository(self.runtime).purge(aggressive=aggressive) + # Purging means no tracked run remains in this repository scope. Port + # locks were created by the original startup process (and therefore a + # different PortPlanner session), so releasing only this command's + # session leaves unused/disabled stack locks behind indefinitely. The + # registry commit happens first so a failed purge cannot erase locks + # that still protect an authoritative active state. + self.release_all_runtime_ports() + + def _deactivate_runtime_state( + self, + state: RunState, + *, + release_session_when_empty: bool = True, + ) -> None: + repository = self._state_repository(self.runtime) + source_run_ids = [ + str(run_id).strip() + for run_id in ( + state.metadata.get("state_source_run_ids") + if isinstance(state.metadata.get("state_source_run_ids"), list) + else [] + ) + if str(run_id).strip() + ] + run_ids = sorted({state.run_id, *source_run_ids}) + deactivate_many = getattr(repository, "deactivate_runs", None) + deactivate = getattr(repository, "deactivate_run", None) + if callable(deactivate_many) or callable(deactivate): + if callable(deactivate_many): + deactivate_many(run_ids) + elif callable(deactivate): + for run_id in run_ids: + deactivate(run_id) + if release_session_when_empty and not self._has_active_runs(repository): + # The final run may have been started by an earlier process and + # therefore a different planner session. At this point the + # repository proves the whole scope inactive, so every scoped + # lock is stale ownership rather than another live run's lock. + self.release_all_runtime_ports() + return + self._purge_runtime_state(aggressive=False) def _run_spinner_operation( self, diff --git a/python/envctl_engine/runtime/lifecycle_dependency_stop.py b/python/envctl_engine/runtime/lifecycle_dependency_stop.py index aedc9743..909958e9 100644 --- a/python/envctl_engine/runtime/lifecycle_dependency_stop.py +++ b/python/envctl_engine/runtime/lifecycle_dependency_stop.py @@ -1,11 +1,12 @@ from __future__ import annotations from collections.abc import Callable, Mapping +from typing import Any from envctl_engine.requirements.core import dependency_definitions from envctl_engine.runtime.command_router import Route -from envctl_engine.runtime.lifecycle_requirement_ports import component_port_values -from envctl_engine.state.models import RunState +from envctl_engine.runtime.lifecycle_requirement_ports import release_requirement_component_port_values +from envctl_engine.state.models import RequirementsResult, RunState def select_dependency_components_for_stop(state: RunState, route: Route) -> dict[str, set[str]]: @@ -37,7 +38,7 @@ def release_selected_dependency_components( state: RunState, selected_dependencies: dict[str, set[str]], *, - release_component_ports_fn: Callable[[Mapping[str, object]], None], + release_component_ports_fn: Callable[[RequirementsResult, str, Mapping[str, object]], None], ) -> None: for project_name, dependency_ids in selected_dependencies.items(): requirements = state.requirements.get(project_name) @@ -48,7 +49,7 @@ def release_selected_dependency_components( if not bool(component.get("enabled", False)): continue if not bool(component.get("external")): - release_component_ports_fn(component) + release_component_ports_fn(requirements, dependency_id, component) requirements.components[dependency_id] = {} if not requirements_have_enabled_components(requirements): state.requirements.pop(project_name, None) @@ -57,10 +58,14 @@ def release_selected_dependency_components( def release_requirement_component_ports( component: Mapping[str, object], *, - release_port_fn: Callable[[int], None], + port_planner: Any, + owner_candidates: tuple[str, ...] = (), ) -> None: - for port in sorted(component_port_values(component)): - release_port_fn(port) + release_requirement_component_port_values( + port_planner, + component, + owner_candidates=owner_candidates, + ) def requirements_have_enabled_components(requirements: object) -> bool: @@ -68,7 +73,5 @@ def requirements_have_enabled_components(requirements: object) -> bool: if not isinstance(components, Mapping): return False return any( - bool(component.get("enabled", False)) - for component in components.values() - if isinstance(component, Mapping) + bool(component.get("enabled", False)) for component in components.values() if isinstance(component, Mapping) ) diff --git a/python/envctl_engine/runtime/lifecycle_operation_lease.py b/python/envctl_engine/runtime/lifecycle_operation_lease.py new file mode 100644 index 00000000..8992961d --- /dev/null +++ b/python/envctl_engine/runtime/lifecycle_operation_lease.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import os +import threading +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from envctl_engine.state.persistence import advisory_file_lock + + +_OPERATION_OWNER_ATTR = "_lifecycle_operation_owner" +_OPERATION_DEPTH_ATTR = "_lifecycle_operation_depth" +_OPERATION_LEASE_ATTR = "_lifecycle_operation_lease" + + +class _LifecycleOperationLease: + def __init__(self, manager: Any) -> None: + self._manager = manager + self.released = False + + def release(self) -> None: + if self.released: + return + self.released = True + self._manager.__exit__(None, None, None) + + +def run_exclusive_lifecycle_operation( + runtime: Any, + *, + command: str, + operation: Callable[[], int], +) -> int: + owner = os.getpid(), threading.get_ident() + current_owner = getattr(runtime, _OPERATION_OWNER_ATTR, None) + current_depth = int(getattr(runtime, _OPERATION_DEPTH_ATTR, 0) or 0) + if current_owner == owner and current_depth > 0: + setattr(runtime, _OPERATION_DEPTH_ATTR, current_depth + 1) + try: + return operation() + finally: + if getattr(runtime, _OPERATION_OWNER_ATTR, None) == owner: + setattr(runtime, _OPERATION_DEPTH_ATTR, current_depth) + + lock_path = _operation_lock_path(runtime) + manager = advisory_file_lock(lock_path, exclusive=True, timeout_seconds=0.0) + try: + manager.__enter__() + except TimeoutError: + runtime._emit("lifecycle.operation.busy", command=command, lock_path=str(lock_path)) + print("Another envctl lifecycle operation is already active for this repository.") + return 1 + + lease = _LifecycleOperationLease(manager) + setattr(runtime, _OPERATION_OWNER_ATTR, owner) + setattr(runtime, _OPERATION_DEPTH_ATTR, 1) + setattr(runtime, _OPERATION_LEASE_ATTR, lease) + try: + return operation() + finally: + if getattr(runtime, _OPERATION_LEASE_ATTR, None) is lease: + _clear_runtime_lease(runtime) + lease.release() + + +def release_lifecycle_operation(runtime: Any) -> bool: + """Release a committed lifecycle mutation before entering an interactive wait.""" + + owner = os.getpid(), threading.get_ident() + if getattr(runtime, _OPERATION_OWNER_ATTR, None) != owner: + return False + lease = getattr(runtime, _OPERATION_LEASE_ATTR, None) + if not isinstance(lease, _LifecycleOperationLease): + return False + _clear_runtime_lease(runtime) + lease.release() + return True + + +def lifecycle_operation_active(runtime: Any) -> bool: + """Return whether this thread still owns an unreleased lifecycle lease.""" + + owner = os.getpid(), threading.get_ident() + lease = getattr(runtime, _OPERATION_LEASE_ATTR, None) + return ( + getattr(runtime, _OPERATION_OWNER_ATTR, None) == owner + and int(getattr(runtime, _OPERATION_DEPTH_ATTR, 0) or 0) > 0 + and isinstance(lease, _LifecycleOperationLease) + and not lease.released + ) + + +def _clear_runtime_lease(runtime: Any) -> None: + setattr(runtime, _OPERATION_DEPTH_ATTR, 0) + setattr(runtime, _OPERATION_OWNER_ATTR, None) + setattr(runtime, _OPERATION_LEASE_ATTR, None) + + +def _operation_lock_path(runtime: Any) -> Path: + repository = getattr(runtime, "state_repository", None) + repository_lock_path = getattr(repository, "lifecycle_operation_lock_path", None) + if callable(repository_lock_path): + return Path(repository_lock_path()) + root = getattr(repository, "runtime_root", None) + if root is None: + root = getattr(runtime, "runtime_root") + root_path = Path(root).expanduser().absolute() + if root_path.is_symlink(): + raise RuntimeError(f"runtime_root changed identity or became a symlink: {root_path}") + return root_path.resolve(strict=False) / ".lifecycle_operation.lock" + + +__all__ = [ + "lifecycle_operation_active", + "release_lifecycle_operation", + "run_exclusive_lifecycle_operation", +] diff --git a/python/envctl_engine/runtime/lifecycle_requirement_ports.py b/python/envctl_engine/runtime/lifecycle_requirement_ports.py index 213ef615..5d51d01b 100644 --- a/python/envctl_engine/runtime/lifecycle_requirement_ports.py +++ b/python/envctl_engine/runtime/lifecycle_requirement_ports.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Mapping +import inspect from typing import Any from envctl_engine.requirements.core import dependency_definitions @@ -8,8 +9,122 @@ def release_requirement_ports(runtime: Any, requirements: RequirementsResult) -> None: - for port in sorted(requirement_port_values(requirements)): - runtime.port_planner.release(port) + reservations: list[tuple[int, tuple[str, ...], str | None]] = [] + for definition in dependency_definitions(): + component = requirements.component(definition.id) + if not bool(component.get("enabled", False)) or bool(component.get("external")): + continue + owners = requirement_component_port_owners(requirements, definition.id) + expected_session = _port_lock_session(component.get("port_lock_session")) + reservations.extend((port, owners, expected_session) for port in component_port_values(component)) + for port, owners, expected_session in sorted(reservations, key=lambda item: item[0]): + release_port_reservation( + runtime.port_planner, + port, + owner_candidates=owners, + expected_session=expected_session, + ) + + +def release_requirement_component_port_values( + port_planner: Any, + component: Mapping[str, object], + *, + owner_candidates: tuple[str, ...], +) -> None: + expected_session = _port_lock_session(component.get("port_lock_session")) + for port in sorted(component_port_values(component)): + release_port_reservation( + port_planner, + port, + owner_candidates=owner_candidates, + expected_session=expected_session, + ) + + +def release_port_reservation( + port_planner: Any, + port: int, + *, + owner_candidates: tuple[str, ...], + expected_session: str | None, +) -> bool: + """Release one persisted reservation without crossing planner sessions.""" + + owners = tuple(dict.fromkeys(owner for owner in owner_candidates if owner)) + normalized_session = _port_lock_session(expected_session) + release_owned = getattr(port_planner, "release_owned", None) + if normalized_session and callable(release_owned): + for owner in owners: + if bool(release_owned(port, owner, expected_session=normalized_session)): + return True + + # A legacy record has no session identity. Its only safe eager release is + # through the current planner session, whose release method checks its own + # session before unlinking. For a persisted session, use this fallback only + # when that exact planner session is still active. + current_session = _port_lock_session(getattr(port_planner, "session_id", None)) + if normalized_session is None or current_session == normalized_session: + _release_current_session(port_planner, port, owners) + + # Reaping is the compatibility path for dead foreign sessions. It is + # deliberately attempted only through the planner's liveness proof. + reap_stale = getattr(port_planner, "reap_stale", None) + if callable(reap_stale): + if owners: + for owner in owners: + if bool(reap_stale(port, owner=owner)): + return True + elif bool(reap_stale(port)): + return True + return False + + +def _release_current_session(port_planner: Any, port: int, owners: tuple[str, ...]) -> None: + release = getattr(port_planner, "release", None) + if not callable(release): + return + if not owners: + release(port) + return + try: + parameters = inspect.signature(release).parameters.values() + except (TypeError, ValueError): + return + supports_owner = any( + parameter.name == "owner" or parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters + ) + if not supports_owner: + # Minimal test doubles and legacy adapters may expose release(port). + # Production PortPlanner always takes owner and enforces its session. + release(port) + return + for owner in owners: + release(port, owner=owner) + + +def _port_lock_session(value: object) -> str | None: + normalized = value.strip() if isinstance(value, str) else "" + return normalized or None + + +def requirement_component_port_owners( + requirements: RequirementsResult, + dependency_id: str, +) -> tuple[str, ...]: + """Return every owner form used while reserving one dependency's ports.""" + + project = str(requirements.project).strip() + normalized_dependency = str(dependency_id).strip().lower() + if not project or not normalized_dependency: + return () + owner_suffixes = {"requirements"} + for definition in dependency_definitions(): + if definition.id != normalized_dependency: + continue + owner_suffixes.update(resource.legacy_port_key for resource in definition.resources) + break + return tuple(f"{project}:{suffix}" for suffix in sorted(owner_suffixes)) def requirement_port_values(requirements: RequirementsResult) -> set[int]: @@ -20,9 +135,7 @@ def requirement_port_values(requirements: RequirementsResult) -> set[int]: continue if bool(component.get("external")): continue - final = component.get("final") - if isinstance(final, int) and final > 0: - ports.add(final) + ports.update(component_port_values(component)) return ports diff --git a/python/envctl_engine/runtime/lifecycle_service_stop_selection.py b/python/envctl_engine/runtime/lifecycle_service_stop_selection.py index 917c5950..273c5823 100644 --- a/python/envctl_engine/runtime/lifecycle_service_stop_selection.py +++ b/python/envctl_engine/runtime/lifecycle_service_stop_selection.py @@ -22,24 +22,26 @@ def select_services_for_stop( if bool(route.flags.get("all")): return set(state.services.keys()) + selected, has_selectors = _selected_services_from_route( + state, + route, + project_name_from_service_fn=project_name_from_service_fn, + selectors_from_passthrough_fn=selectors_from_passthrough_fn, + ) + runtime_scope = route.flags.get("runtime_scope") if runtime_scope in {"backend", "frontend"}: - return { + scoped = { name for name, service in state.services.items() if service_matches_runtime_scope(name, service, str(runtime_scope)) } + return scoped.intersection(selected) if has_selectors else scoped if runtime_scope in {"fullstack", "entire-system"}: - return set(state.services.keys()) + return selected if has_selectors else set(state.services.keys()) if runtime_scope == "dependencies": return set() - selected, has_selectors = _selected_services_from_route( - state, - route, - project_name_from_service_fn=project_name_from_service_fn, - selectors_from_passthrough_fn=selectors_from_passthrough_fn, - ) if not selected and has_selectors: return set() if not selected and select_dependency_components_for_stop(state, route): diff --git a/python/envctl_engine/runtime/lifecycle_service_termination.py b/python/envctl_engine/runtime/lifecycle_service_termination.py index 0c8ff984..aac50d64 100644 --- a/python/envctl_engine/runtime/lifecycle_service_termination.py +++ b/python/envctl_engine/runtime/lifecycle_service_termination.py @@ -2,13 +2,39 @@ import concurrent.futures import os -import signal +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path from typing import Any +from envctl_engine.shared.process_termination import ( + pid_is_running, + terminate_pid, + wait_for_pid_exit, +) +from envctl_engine.shared.services import service_project_name +from envctl_engine.runtime.lifecycle_requirement_ports import release_port_reservation -def terminate_started_services(runtime: Any, services: dict[str, object]) -> None: - for service in services.values(): - runtime._terminate_service_record(service, aggressive=False, verify_ownership=False) + +def terminate_started_services(runtime: Any, services: dict[str, object]) -> set[str]: + """Terminate startup-owned services and report every unconfirmed exit.""" + + failed: set[str] = set() + for name, service in services.items(): + try: + terminated = runtime._terminate_service_record( + service, + aggressive=False, + verify_ownership=False, + ) + except Exception as exc: # noqa: BLE001 + _emit_cleanup_error(runtime, service=name, error=exc) + terminated = False + if not terminated: + failed.add(name) + else: + release_service_ports(runtime, name=name, service=service) + return failed def terminate_services_from_state( @@ -18,19 +44,25 @@ def terminate_services_from_state( selected_services: set[str] | None, aggressive: bool, verify_ownership: bool, -) -> None: +) -> set[str]: work_items: list[tuple[str, object]] = [] for name, service in getattr(state, "services", {}).items(): if selected_services is not None and name not in selected_services: continue work_items.append((name, service)) - def terminate_one(item: tuple[str, object]) -> tuple[str, bool, int | None]: + def terminate_one(item: tuple[str, object]) -> tuple[str, object, bool, int | None]: name, service = item - terminated = runtime._terminate_service_record( - service, aggressive=aggressive, verify_ownership=verify_ownership - ) - return name, terminated, service_port(service) + try: + terminated = runtime._terminate_service_record( + service, + aggressive=aggressive, + verify_ownership=verify_ownership, + ) + except Exception as exc: # noqa: BLE001 + _emit_cleanup_error(runtime, service=name, error=exc) + terminated = False + return name, service, terminated, service_port(service) if len(work_items) <= 1: results = [terminate_one(item) for item in work_items] @@ -39,47 +71,120 @@ def terminate_one(item: tuple[str, object]) -> tuple[str, bool, int | None]: with concurrent.futures.ThreadPoolExecutor(max_workers=worker_count) as pool: results = list(pool.map(terminate_one, work_items)) - for _name, terminated, port in results: - if terminated and port is not None: - runtime.port_planner.release(port) + for name, service, terminated, _port in results: + if terminated: + release_service_ports(runtime, name=name, service=service) + return {name for name, _service, terminated, _port in results if not terminated} + + +def release_service_ports(runtime: Any, *, name: str, service: object) -> None: + port_planner = getattr(runtime, "port_planner", None) + if port_planner is None: + return + project = str(service_project_name(service)).strip() + if not project: + project_resolver = getattr(runtime, "_project_name_from_service", None) + if callable(project_resolver): + project = str(project_resolver(name) or "").strip() + service_type = str(getattr(service, "type", "") or "").strip().lower() + owners: tuple[str, ...] = () + if project: + owner_candidates = [f"{project}:services"] + if service_type: + owner_candidates = [ + f"{project}:{service_type}", + f"{project}:services:{service_type}-launch", + *owner_candidates, + ] + owners = tuple(owner_candidates) + + raw_expected_session = getattr(service, "port_lock_session", None) + expected_session = raw_expected_session.strip() if isinstance(raw_expected_session, str) else None + for port in service_ports(service): + release_port_reservation( + port_planner, + port, + owner_candidates=owners, + expected_session=expected_session, + ) + + +def service_ports(service: object) -> tuple[int, ...]: + """Return every distinct port reservation represented by a service record.""" + + ports: list[int] = [] + for attribute in ("requested_port", "actual_port"): + value = getattr(service, attribute, None) + if isinstance(value, int) and not isinstance(value, bool) and value > 0 and value not in ports: + ports.append(value) + return tuple(ports) def terminate_service_record(runtime: Any, service: object, *, aggressive: bool, verify_ownership: bool) -> bool: - pid = getattr(service, "pid", None) - if not isinstance(pid, int) or pid <= 0: - return True - if pid in {os.getpid(), os.getppid()}: - runtime._emit("cleanup.skip", service=getattr(service, "name", "unknown"), pid=pid, reason="self_or_parent") + recorded_pids = _recorded_service_pids(service) + if not recorded_pids and str(getattr(service, "status", "") or "") == "termination_failed": + return False + protected_pids = {os.getpid(), os.getppid()} + protected = [pid for pid in recorded_pids if pid in protected_pids] + if protected: + runtime._emit( + "cleanup.skip", + service=getattr(service, "name", "unknown"), + pid=protected[0], + reason="self_or_parent", + ) return False + live_pids = [pid for pid in recorded_pids if _pid_is_running(runtime, pid)] port = service_port(service) + listener_probe = _live_listener_pids_for_port(runtime, port) if port is not None else _ListenerProbe() + if not live_pids: + return not listener_probe.failed and not listener_probe.pids if verify_ownership: if port is None: - runtime._emit( - "cleanup.skip", - service=getattr(service, "name", "unknown"), - pid=pid, - reason="missing_port_for_ownership", - ) - return False - is_owner = _service_pid_or_listener_owns_port(runtime, service, pid=pid, port=port) - if not is_owner: - runtime._emit("cleanup.skip", service=getattr(service, "name", "unknown"), pid=pid, port=port) - return False + if not all(_non_listener_service_matches_recorded_cwd(runtime, service, pid=pid) for pid in live_pids): + runtime._emit( + "cleanup.skip", + service=getattr(service, "name", "unknown"), + pid=live_pids[0], + reason="missing_port_for_ownership", + ) + return False + termination_pids = live_pids + else: + owners = [pid for pid in live_pids if _pid_owns_port(runtime, pid=pid, port=port)] + if not owners: + runtime._emit( + "cleanup.skip", + service=getattr(service, "name", "unknown"), + pid=live_pids[0], + port=port, + reason="ownership_mismatch", + ) + return False + primary_pid = getattr(service, "pid", None) + termination_pids = list(owners) + if isinstance(primary_pid, int) and primary_pid in live_pids and primary_pid not in termination_pids: + termination_pids.insert(0, primary_pid) + else: + termination_pids = live_pids - try: - terminate_group = getattr(runtime.process_runner, "terminate_process_group", None) - if callable(terminate_group): - return bool(terminate_group(pid, term_timeout=0.5 if aggressive else 2.0, kill_timeout=1.0)) - except Exception: # noqa: BLE001 - pass - try: - return bool(runtime.process_runner.terminate(pid, term_timeout=0.5 if aggressive else 2.0, kill_timeout=1.0)) - except Exception: # noqa: BLE001 - pass - try: - os.kill(pid, signal.SIGTERM) - except OSError: - return True + failed = False + for pid in termination_pids: + if not _pid_is_running(runtime, pid): + continue + if not terminate_pid( + pid, + process_runner=runtime.process_runner, + term_timeout=0.5 if aggressive else 2.0, + kill_timeout=1.0, + ): + failed = True + if failed: + return False + if port is not None: + listener_probe = _live_listener_pids_for_port(runtime, port) + if listener_probe.failed or listener_probe.pids: + return False return True @@ -93,21 +198,92 @@ def service_port(service: object) -> int | None: return None -def _service_pid_or_listener_owns_port(runtime: Any, service: object, *, pid: int, port: int) -> bool: +def _emit_cleanup_error(runtime: Any, *, service: str, error: BaseException) -> None: + emit = getattr(runtime, "_emit", None) + if callable(emit): + emit("cleanup.error", service=service, error=str(error)) + + +def _pid_owns_port(runtime: Any, *, pid: int, port: int) -> bool: try: - if bool(runtime.process_runner.pid_owns_port(pid, port)): - return True + return bool(runtime.process_runner.pid_owns_port(pid, port)) except Exception: # noqa: BLE001 - pass - listener_pids = getattr(service, "listener_pids", None) - if not isinstance(listener_pids, list): return False - for listener_pid in listener_pids: - if not isinstance(listener_pid, int) or listener_pid <= 0: - continue - try: - if bool(runtime.process_runner.pid_owns_port(listener_pid, port)): - return True - except Exception: # noqa: BLE001 - continue - return False + + +def _recorded_service_pids(service: object) -> list[int]: + candidates = [getattr(service, "pid", None), *(getattr(service, "listener_pids", None) or [])] + return list( + dict.fromkeys(pid for pid in candidates if isinstance(pid, int) and not isinstance(pid, bool) and pid > 0) + ) + + +@dataclass(frozen=True, slots=True) +class _ListenerProbe: + available: bool = False + failed: bool = False + pids: frozenset[int] = frozenset() + + +def _live_listener_pids_for_port(runtime: Any, port: int) -> _ListenerProbe: + listener_reader = getattr(runtime, "_listener_pids_for_port", None) + if not callable(listener_reader): + return _ListenerProbe() + try: + listeners = listener_reader(port) + except Exception: # noqa: BLE001 + return _ListenerProbe(available=True, failed=True) + if listeners is None or isinstance(listeners, (str, bytes, Mapping)): + return _ListenerProbe(available=True, failed=True) + try: + listener_items = iter(listeners) + except TypeError: + return _ListenerProbe(available=True, failed=True) + return _ListenerProbe( + available=True, + pids=frozenset( + pid + for pid in listener_items + if isinstance(pid, int) and not isinstance(pid, bool) and pid > 0 and _pid_is_running(runtime, pid) + ), + ) + + +def _non_listener_service_matches_recorded_cwd(runtime: Any, service: object, *, pid: int) -> bool: + if bool(getattr(service, "listener_expected", True)): + return False + recorded_cwd = str(getattr(service, "cwd", "") or "").strip() + if not recorded_cwd: + return False + cwd_reader = getattr(runtime.process_runner, "process_cwd", None) + if not callable(cwd_reader): + return False + try: + actual_cwd = str(cwd_reader(pid) or "").strip() + except Exception: # noqa: BLE001 + return False + if not actual_cwd: + return False + try: + return Path(actual_cwd).resolve(strict=False) == Path(recorded_cwd).resolve(strict=False) + except OSError: + return actual_cwd == recorded_cwd + + +def _wait_for_pid_exit( + runtime: Any, + pid: int, + *, + timeout: float, + initial_identity: str | None, +) -> bool: + return wait_for_pid_exit( + runtime.process_runner, + pid, + timeout=timeout, + initial_identity=initial_identity, + ) + + +def _pid_is_running(runtime: Any, pid: int) -> bool: + return pid_is_running(runtime.process_runner, pid) diff --git a/python/envctl_engine/runtime/lifecycle_targeted_state.py b/python/envctl_engine/runtime/lifecycle_targeted_state.py new file mode 100644 index 00000000..1d7cf805 --- /dev/null +++ b/python/envctl_engine/runtime/lifecycle_targeted_state.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from typing import Any, cast + +from envctl_engine.state.models import RequirementsResult, RunState +from envctl_engine.state.runtime_map import build_runtime_map +from envctl_engine.state.state_aggregation import filter_state_to_owned_projects + + +class LifecycleTargetedStateSupport: + """Own project-scoped state reduction after targeted lifecycle stops.""" + + runtime: Any = cast(Any, None) + + def _project_name_for_service(self, name: str, service: object) -> str: + explicit_project = str(getattr(service, "project", "") or "").strip() + if explicit_project: + return explicit_project.casefold() + return str(self.runtime._project_name_from_service(name) or "").strip().casefold() + + def _save_selected_stop_state( + self, + state: RunState, + *, + authoritative_project_names: list[str] | None = None, + ) -> None: + owner = cast(Any, self) + repository = owner._state_repository(self.runtime) + + def runtime_map_builder(raw_state: object) -> dict[str, object]: + return build_runtime_map(cast(RunState, raw_state)) + + if authoritative_project_names is None: + repository.save_selected_stop_state( + state=state, + emit=self.runtime._emit, + runtime_map_builder=runtime_map_builder, + ) + return + repository.save_selected_stop_state( + state=state, + emit=self.runtime._emit, + runtime_map_builder=runtime_map_builder, + authoritative_project_names=authoritative_project_names, + ) + + def _persist_targeted_project_state( + self, + state: RunState, + *, + owned_projects: dict[str, str], + targeted_project_keys: frozenset[str], + ) -> bool: + owner = cast(Any, self) + active_project_keys = self._active_project_keys(state) + remaining_keys = (set(owned_projects).difference(targeted_project_keys)).union( + targeted_project_keys.intersection(active_project_keys) + ) + if not remaining_keys: + for requirements in state.requirements.values(): + owner._release_requirement_ports(requirements) + owner._deactivate_runtime_state(state) + return False + + remaining_names = [owned_projects[key] for key in sorted(remaining_keys)] + filtered = filter_state_to_owned_projects( + state, + frozenset(remaining_keys), + service_project_name=self._project_name_for_service, + ) + filtered.metadata["project_names"] = remaining_names + self._save_selected_stop_state( + filtered, + authoritative_project_names=remaining_names, + ) + return True + + def _owned_project_names(self, state: RunState) -> dict[str, str]: + projects: dict[str, str] = {} + + def add(raw_name: object) -> None: + name = str(raw_name or "").strip() + if name: + projects.setdefault(name.casefold(), name) + + metadata_names = state.metadata.get("project_names") + if isinstance(metadata_names, Iterable) and not isinstance(metadata_names, (str, bytes, Mapping)): + for project in metadata_names: + add(project) + metadata_roots = state.metadata.get("project_roots") + if isinstance(metadata_roots, Mapping): + for project in metadata_roots: + add(project) + for project, requirements in state.requirements.items(): + add(getattr(requirements, "project", "") or project) + for name, service in state.services.items(): + add(self._project_name_for_service(name, service)) + if state.mode == "trees" and any(project != "main" for project in projects): + projects.pop("main", None) + return projects + + def _active_project_keys(self, state: RunState) -> set[str]: + active = { + self._project_name_for_service(name, service) + for name, service in state.services.items() + if self._project_name_for_service(name, service) + } + active.update( + self._requirement_project_key(project, requirements) for project, requirements in state.requirements.items() + ) + raw_stopped = state.metadata.get("dashboard_stopped_services") + if isinstance(raw_stopped, list): + active.update( + str(item.get("project", "")).strip().casefold() + for item in raw_stopped + if isinstance(item, Mapping) and str(item.get("project", "")).strip() + ) + return {project for project in active if project} + + def _targeted_projects_have_no_managed_resources( + self, + state: RunState, + targeted_project_keys: frozenset[str], + ) -> bool: + if any( + self._project_name_for_service(name, service) in targeted_project_keys + for name, service in state.services.items() + ): + return False + return not any( + self._requirement_project_key(project, requirements) in targeted_project_keys + and cast(Any, self)._requirements_have_enabled_components(requirements) + for project, requirements in state.requirements.items() + ) + + def _remove_targeted_requirements( + self, + state: RunState, + targeted_project_keys: frozenset[str], + ) -> None: + owner = cast(Any, self) + for project, requirements in list(state.requirements.items()): + if self._requirement_project_key(project, requirements) not in targeted_project_keys: + continue + owner._release_requirement_ports(requirements) + state.requirements.pop(project, None) + + @staticmethod + def _requirement_project_key(project: str, requirements: RequirementsResult) -> str: + return str(getattr(requirements, "project", "") or project).strip().casefold() diff --git a/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py b/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py index 771073db..07f726b8 100644 --- a/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py +++ b/python/envctl_engine/runtime/lifecycle_worktree_cleanup.py @@ -30,6 +30,7 @@ from envctl_engine.runtime.lifecycle_worktree_processes import ( blast_tree_listener_ports as _blast_tree_listener_ports, ) +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RequirementsResult, RunState from envctl_engine.state.runtime_map import build_runtime_map @@ -136,7 +137,11 @@ def blast_worktree_before_delete( ) for mode in ("trees", "main"): - state = runtime._try_load_existing_state(mode=mode, strict_mode_match=True) + state = call_state_loader( + runtime._try_load_existing_state, + mode=mode, + strict_mode_match=True, + ) if state is None: continue @@ -159,33 +164,27 @@ def blast_worktree_before_delete( if not selected_services and requirement_key is None: continue - runtime._terminate_services_from_state( + termination_result = runtime._terminate_services_from_state( state, selected_services=selected_services, aggressive=True, verify_ownership=False, ) - for service_name in selected_services: + failed_services = ( + {str(name) for name in termination_result} if isinstance(termination_result, (set, frozenset)) else set() + ) + stopped_services = selected_services.difference(failed_services) + for service_name in stopped_services: state.services.pop(service_name, None) - if requirement_key is not None: + if requirement_key is not None and not failed_services: requirement_entry = state.requirements.pop(requirement_key, None) if requirement_entry is not None: target_ports.update(cleanup.collect_requirement_ports(requirement_entry)) release_requirement_ports(runtime, requirement_entry) - remaining_projects: set[str] = set() - for service_name in state.services: - project = runtime._project_name_from_service(service_name) - if project: - remaining_projects.add(project) - for project in list(state.requirements.keys()): - if project in remaining_projects: - continue - requirement_entry = state.requirements.pop(project) - release_requirement_ports(runtime, requirement_entry) - - artifact_paths.update(cleanup.prune_project_metadata(state, project_name=normalized_project)) + if not failed_services: + artifact_paths.update(cleanup.prune_project_metadata(state, project_name=normalized_project)) try: runtime.state_repository.save_selected_stop_state( @@ -202,15 +201,27 @@ def blast_worktree_before_delete( mode=mode, warning=warning, ) - continue + raise RuntimeError(warning) from exc runtime._emit( "cleanup.worktree.state.updated", project=normalized_project, mode=mode, - services_removed=len(selected_services), - requirements_removed=(requirement_key is not None), + services_removed=len(stopped_services), + requirements_removed=(requirement_key is not None and not failed_services), ) + if failed_services: + warning = ( + f"worktree cleanup could not confirm service exit for {normalized_project} ({mode}): " + + ", ".join(sorted(failed_services)) + ) + runtime._emit( + "cleanup.worktree.warning", + project=normalized_project, + mode=mode, + warning=warning, + ) + raise RuntimeError(warning) if blast_mode: cleanup.blast_tree_listener_ports( diff --git a/python/envctl_engine/runtime/lifecycle_worktree_metadata.py b/python/envctl_engine/runtime/lifecycle_worktree_metadata.py index 297a5143..dcc87fa0 100644 --- a/python/envctl_engine/runtime/lifecycle_worktree_metadata.py +++ b/python/envctl_engine/runtime/lifecycle_worktree_metadata.py @@ -8,6 +8,18 @@ def prune_project_metadata(state: RunState, *, project_name: str) -> list[Path]: removed_paths: list[Path] = [] + normalized_project = str(project_name).strip().casefold() + raw_project_names = state.metadata.get("project_names") + if isinstance(raw_project_names, list): + remaining_project_names = [ + name + for name in raw_project_names + if str(name).strip().casefold() != normalized_project + ] + if remaining_project_names: + state.metadata["project_names"] = remaining_project_names + else: + state.metadata.pop("project_names", None) for metadata_key in ("project_pr_links", "project_roots"): raw = state.metadata.get(metadata_key) if not isinstance(raw, dict): diff --git a/python/envctl_engine/runtime/service_listener_truth.py b/python/envctl_engine/runtime/service_listener_truth.py index f6f6c137..8cdc4390 100644 --- a/python/envctl_engine/runtime/service_listener_truth.py +++ b/python/envctl_engine/runtime/service_listener_truth.py @@ -35,6 +35,12 @@ def wait_for_service_listener( return True except Exception: # noqa: BLE001 pass + # A generic port probe is only a safe fallback while the process envctl + # launched is still alive. Once that PID has exited, waiting on the port can + # hide the real startup error for minutes and can adopt an unrelated process + # that happens to bind the same port later. + if _launch_pid_definitively_exited(runtime, pid): + return False if ( debug_listener_group in {"", "port_fallback"} and service_truth_fallback_enabled(runtime) @@ -51,6 +57,16 @@ def wait_for_service_listener( return False +def _launch_pid_definitively_exited(runtime: Any, pid: int) -> bool: + checker = getattr(runtime.process_runner, "is_pid_running", None) + if not callable(checker): + return False + try: + return not bool(checker(pid)) + except Exception: # noqa: BLE001 + return False + + def process_tree_probe_supported(runtime: Any) -> bool: try: return bool(runtime.process_runner.supports_process_tree_probe()) diff --git a/python/envctl_engine/runtime/service_manager.py b/python/envctl_engine/runtime/service_manager.py index a2683337..20384feb 100644 --- a/python/envctl_engine/runtime/service_manager.py +++ b/python/envctl_engine/runtime/service_manager.py @@ -1,13 +1,13 @@ from __future__ import annotations import concurrent.futures -import os -import signal import time +from collections.abc import Iterable from dataclasses import dataclass from typing import Callable from envctl_engine.requirements.common_contracts import is_bind_conflict +from envctl_engine.shared.process_termination import terminate_pid from envctl_engine.shared.services import service_display_name from envctl_engine.state.models import ServiceRecord @@ -16,6 +16,14 @@ class ServiceStartError(RuntimeError): """Raised when a service fails to start after retry policy.""" +class ServiceCleanupError(ServiceStartError): + """Raised when a failed startup leaves processes whose exit is unconfirmed.""" + + def __init__(self, message: str, unterminated_services: dict[str, ServiceRecord]) -> None: + super().__init__(message) + self.unterminated_services = dict(unterminated_services) + + @dataclass(slots=True) class ServiceStartOutcome: record: ServiceRecord @@ -38,6 +46,9 @@ class ServiceStartDescriptor: class ServiceManager: + def __init__(self, *, process_runner: object | None = None) -> None: + self.process_runner = process_runner + def start_service_with_retry( self, *, @@ -58,6 +69,20 @@ def start_service_with_retry( while True: success, error, pid = start(current_port) if success: + if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0: + record = _failed_start_record( + project=project, + service_type=service_type, + cwd=cwd, + pid=None, + requested_port=current_port if listener_expected else None, + listener_expected=listener_expected, + failure_detail="service launcher reported success without a valid PID", + ) + raise ServiceCleanupError( + f"Service launcher for {record.name} reported success without a valid PID", + {record.name: record}, + ) try: actual_port: int | None = current_port if listener_expected else None if detect_actual is not None: @@ -73,9 +98,37 @@ def start_service_with_retry( started_at=time.time(), listener_expected=listener_expected, ) - except RuntimeError as exc: + except Exception as exc: # noqa: BLE001 error = str(exc) - _terminate_pid(pid, process_runner=self) + if not _terminate_pid(pid, process_runner=self.process_runner): + record = _failed_start_record( + project=project, + service_type=service_type, + cwd=cwd, + pid=pid, + requested_port=current_port if listener_expected else None, + listener_expected=listener_expected, + failure_detail=error, + ) + raise ServiceCleanupError( + f"{error}; cleanup could not confirm exit of PID {pid}", + {record.name: record}, + ) from exc + elif isinstance(pid, int) and pid > 0: + if not _terminate_pid(pid, process_runner=self.process_runner): + record = _failed_start_record( + project=project, + service_type=service_type, + cwd=cwd, + pid=pid, + requested_port=current_port if listener_expected else None, + listener_expected=listener_expected, + failure_detail=error or "service launcher reported failure", + ) + raise ServiceCleanupError( + f"{error or 'service launcher reported failure'}; cleanup could not confirm exit of PID {pid}", + {record.name: record}, + ) if retries >= max_retries or not _is_retryable_error(error): raise ServiceStartError( @@ -160,6 +213,8 @@ def start_record(descriptor: ServiceStartDescriptor) -> ServiceRecord: on_retry=on_retry, ) except Exception as exc: + if isinstance(exc, ServiceCleanupError): + raise if descriptor.critical: raise record = ServiceRecord( @@ -189,8 +244,7 @@ def start_record(descriptor: ServiceStartDescriptor) -> ServiceRecord: worker_count = max_workers or len(descriptors) with concurrent.futures.ThreadPoolExecutor(max_workers=min(len(descriptors), worker_count)) as executor: future_map = { - executor.submit(start_record, descriptor): index - for index, descriptor in enumerate(descriptors) + executor.submit(start_record, descriptor): index for index, descriptor in enumerate(descriptors) } records_by_index: dict[int, ServiceRecord] = {} failures_by_index: dict[int, Exception] = {} @@ -201,17 +255,27 @@ def start_record(descriptor: ServiceStartDescriptor) -> ServiceRecord: except Exception as exc: # noqa: BLE001 failures_by_index[index] = exc - partial_records.extend( - records_by_index[index] for index in sorted(records_by_index) - ) + partial_records.extend(records_by_index[index] for index in sorted(records_by_index)) if failures_by_index: - raise failures_by_index[min(failures_by_index)] + primary_failure = failures_by_index[min(failures_by_index)] + unterminated = _unterminated_services_from_failures(failures_by_index.values()) + if unterminated: + raise ServiceCleanupError(str(primary_failure), unterminated) from primary_failure + raise primary_failure else: for descriptor in descriptors: partial_records.append(start_record(descriptor)) - except Exception: + except Exception as exc: + unterminated = _unterminated_services_from_failures((exc,)) for record in partial_records: - _terminate_pid(record.pid, process_runner=self) + if not _terminate_pid(record.pid, process_runner=self.process_runner): + unterminated[record.name] = record + if unterminated: + names = ", ".join(sorted(unterminated)) + raise ServiceCleanupError( + f"{exc}; cleanup could not confirm exit for: {names}", + unterminated, + ) from exc raise return {record.name: record for record in partial_records} @@ -225,17 +289,52 @@ def _is_retryable_error(error: str | None) -> bool: return False -def _terminate_pid(pid: int | None, *, process_runner: object | None = None) -> None: - if not isinstance(pid, int) or pid <= 0: - return - terminator = getattr(process_runner, "terminate_process_group", None) if process_runner is not None else None - if callable(terminator): - try: - terminator(pid, term_timeout=2.0, kill_timeout=1.0) - return - except Exception: # noqa: BLE001 - pass - try: - os.kill(pid, signal.SIGTERM) - except OSError: - return +def _terminate_pid(pid: int | None, *, process_runner: object | None = None) -> bool: + return terminate_pid( + pid, + process_runner=process_runner, + term_timeout=2.0, + kill_timeout=1.0, + ) + + +def _failed_start_record( + *, + project: str, + service_type: str, + cwd: str, + pid: int | None, + requested_port: int | None, + listener_expected: bool, + failure_detail: str, +) -> ServiceRecord: + record = ServiceRecord( + name=f"{project} {service_display_name(service_type)}", + type=service_type, + cwd=cwd, + pid=pid, + requested_port=requested_port, + actual_port=None, + status="termination_failed", + started_at=time.time(), + listener_expected=listener_expected, + failure_detail=failure_detail, + degraded=True, + ) + record.project = project + record.service_slug = service_type + return record + + +def _unterminated_services_from_failures( + failures: Iterable[object], +) -> dict[str, ServiceRecord]: + unterminated: dict[str, ServiceRecord] = {} + for failure in failures: + services = getattr(failure, "unterminated_services", None) + if not isinstance(services, dict): + continue + for name, service in services.items(): + if isinstance(name, str) and isinstance(service, ServiceRecord): + unterminated[name] = service + return unterminated diff --git a/python/envctl_engine/runtime/service_truth_diagnostics.py b/python/envctl_engine/runtime/service_truth_diagnostics.py index 094fb23f..7fcf7b4b 100644 --- a/python/envctl_engine/runtime/service_truth_diagnostics.py +++ b/python/envctl_engine/runtime/service_truth_diagnostics.py @@ -1,8 +1,11 @@ from __future__ import annotations from pathlib import Path +import re from typing import Any +from envctl_engine.shared.reason_codes import ServiceFailureReason + STARTUP_PROGRESS_TOKENS = ( "Waiting for application startup", @@ -28,6 +31,9 @@ "No module named", "Address already in use", "address already in use", + "command not found", + "exec:", + "No such file or directory", ) @@ -64,6 +70,25 @@ def service_listener_failure_detail(runtime: Any, *, log_path: str | None, pid: return "; ".join(parts) +def service_listener_failure_class(detail: str | None) -> str: + """Classify listener failures using process and log evidence.""" + + text = str(detail or "") + normalized = text.casefold() + missing_executable = bool( + "command not found" in normalized + or "modulenotfounderror" in normalized + or "no module named" in normalized + or re.search(r"\bexec:\s+[^:;\s]+:\s+not found\b", normalized) + or ("/usr/bin/env:" in normalized and "no such file or directory" in normalized) + ) + if missing_executable: + return ServiceFailureReason.DEPENDENCY_MISSING.value + if re.search(r"\bprocess(?:\s+\d+)?\s+exited\b", normalized): + return ServiceFailureReason.PROCESS_EXITED.value + return "listener_not_detected" + + def tail_log_error_line(log_path: str | None, *, max_chars: int = 800) -> str | None: lines = _tail_candidate_lines(log_path) if not lines: @@ -77,6 +102,9 @@ def tail_log_error_line(log_path: str | None, *, max_chars: int = 800) -> str | "Error", "Exception", "No module named", + "command not found", + "exec:", + "No such file or directory", ) selected = lines[-1] selected_index = len(lines) - 1 @@ -115,9 +143,7 @@ def _tail_candidate_lines(log_path: str | None) -> list[str]: return [] try: return [ - line.strip() - for line in path.read_text(encoding="utf-8", errors="replace").splitlines() - if line.strip() + line.strip() for line in path.read_text(encoding="utf-8", errors="replace").splitlines() if line.strip() ] except OSError: return [] diff --git a/python/envctl_engine/runtime/state_fingerprint_support.py b/python/envctl_engine/runtime/state_fingerprint_support.py index 3c16058b..cb06bea1 100644 --- a/python/envctl_engine/runtime/state_fingerprint_support.py +++ b/python/envctl_engine/runtime/state_fingerprint_support.py @@ -1,12 +1,8 @@ from __future__ import annotations -import hashlib -import json - -from envctl_engine.state import state_to_dict +from envctl_engine.state.fingerprints import state_fingerprint as _state_fingerprint from envctl_engine.state.models import RunState def state_fingerprint(state: RunState) -> str: - payload = json.dumps(state_to_dict(state), sort_keys=True) - return hashlib.sha256(payload.encode("utf-8")).hexdigest() + return _state_fingerprint(state) diff --git a/python/envctl_engine/runtime/state_inspection_support.py b/python/envctl_engine/runtime/state_inspection_support.py new file mode 100644 index 00000000..260989fc --- /dev/null +++ b/python/envctl_engine/runtime/state_inspection_support.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +from envctl_engine.runtime.command_router import Route +from envctl_engine.state import state_to_dict +from envctl_engine.state.lookup import call_state_loader +from envctl_engine.state.models import RunState +from envctl_engine.state.project_runtime import ( + cwd_runtime_warnings, + dependency_mode_summary, + project_resolution_event_payload, + resolve_requested_project_state, +) +from envctl_engine.state.runtime_map import build_runtime_map +from envctl_engine.ui.path_links import render_path_for_terminal + +StateLoader = Callable[..., RunState | None] +WarningEmitter = Callable[..., None] + + +def print_state( + runtime: Any, + route: Route, + *, + json_output: bool, + state_loader: StateLoader, + warning_emitter: WarningEmitter, +) -> int: + strict = runtime._state_lookup_strict_mode_match(route) + requested_projects = list(getattr(route, "projects", []) or []) + state = state_loader( + runtime, + mode=route.mode, + strict_mode_match=strict, + project_names=requested_projects or None, + ) + if state is None and requested_projects: + # A targeted repository lookup correctly returns no owner for an + # unknown project. Load the mode-wide aggregate once so the command can + # distinguish that case from an entirely missing runtime and return the + # structured requested_project_not_running contract. + state = state_loader( + runtime, + mode=route.mode, + strict_mode_match=strict, + project_names=None, + ) + if state is None: + if json_output: + print(json.dumps({"found": False, "mode": getattr(route, "mode", None)}, indent=2, sort_keys=True)) + else: + print("No previous state found.") + return 1 + if requested_projects: + resolution = resolve_requested_project_state( + state, + requested_projects, + command="show-state", + runtime=runtime, + allow_multi=False, + ) + if not resolution.ok: + _emit_project_resolution(runtime, "state.project_resolution.failed", resolution, state) + if json_output: + print(json.dumps(resolution.payload(), indent=2, sort_keys=True)) + else: + print(f"Requested project is not running: {resolution.requested_project}") + print("Active runtime projects: " + (", ".join(resolution.active_projects) or "none")) + return 1 + _emit_project_resolution(runtime, "state.project_resolution.ok", resolution, state) + if resolution.state is not None and not bool(getattr(route, "flags", {}).get("all")): + state = resolution.state + + dependency_summary = dependency_mode_summary(state) + cwd_project, warnings = cwd_runtime_warnings( + state, + runtime=runtime, + requested_projects=list(getattr(route, "projects", []) or []), + ) + warning_emitter(runtime, command="show-state", state=state, warnings=warnings) + payload: dict[str, Any] = { + "found": True, + "runtime_root": str(runtime.runtime_root), + "run_state_path": str(runtime._run_state_path()), + "state": state_to_dict(state), + "runtime_map": build_runtime_map(state), + "dependency_mode": dependency_summary["dependency_mode"], + "shared_dependencies": dependency_summary["shared_dependencies"], + "cwd_project": cwd_project, + "warnings": warnings, + } + if json_output: + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + + state_payload = payload["state"] + print(f"run_id: {state_payload['run_id']}") + print(f"mode: {state_payload['mode']}") + print(f"services: {len(state_payload['services'])}") + print(f"requirements: {len(state_payload['requirements'])}") + for warning in warnings: + print(f"Warning: {warning['message']}") + print("run_state_path:") + print(render_path_for_terminal(payload["run_state_path"], env=getattr(runtime, "env", {}))) + return 0 + + +def try_load_existing_state( + runtime: Any, + *, + mode: str | None, + strict_mode_match: bool, + project_names: list[str] | None = None, +) -> RunState | None: + loader = getattr(runtime, "_try_load_existing_state", None) + if not callable(loader): + return None + state = call_state_loader( + loader, + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ) + return state if isinstance(state, RunState) else None + + +def emit_cwd_runtime_mismatch_warnings( + runtime: Any, + *, + command: str, + state: RunState, + warnings: list[dict[str, object]], +) -> None: + emitter = getattr(runtime, "_emit", None) + if not callable(emitter): + return + for warning in warnings: + if str(warning.get("code") or "") != "cwd_runtime_mismatch": + continue + emitter( + "state.cwd_runtime_mismatch", + run_id=getattr(state, "run_id", ""), + mode=getattr(state, "mode", ""), + command=command, + cwd_project=warning.get("cwd_project"), + active_projects=warning.get("active_projects"), + ) + + +def _emit_project_resolution(runtime: Any, event: str, resolution: Any, state: RunState) -> None: + emitter = getattr(runtime, "_emit", None) + if callable(emitter): + emitter( + event, + **project_resolution_event_payload(resolution, state, runtime=runtime), + ) diff --git a/python/envctl_engine/shared/git_branch_support.py b/python/envctl_engine/shared/git_branch_support.py new file mode 100644 index 00000000..0b4efe60 --- /dev/null +++ b/python/envctl_engine/shared/git_branch_support.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import dataclass + + +PREFERRED_PR_BASE_BRANCHES = ("dev", "main", "master") + + +@dataclass(frozen=True, slots=True) +class _RemoteBranchInventory: + preferred_heads: frozenset[str] + default_branch: str | None + reachable: bool + + +def detect_preferred_default_branch( + git_output: Callable[[Sequence[str]], str], + *, + probe_remote_heads: bool = False, +) -> str: + """Resolve the PR base contract before consulting a repository's remote HEAD.""" + + tracked_remote_branches: set[str] = set() + for branch in PREFERRED_PR_BASE_BRANCHES: + if git_output(("rev-parse", "--verify", f"refs/remotes/origin/{branch}")).strip(): + tracked_remote_branches.add(branch) + + remote_inventory = _remote_branch_inventory(git_output) if probe_remote_heads else None + # A successful live inventory is authoritative over local tracking refs, + # including when none of the preferred branches exist. When the probe is + # unavailable or times out, retain the offline tracking-ref fallback. + available_branches = ( + remote_inventory.preferred_heads + if remote_inventory is not None and remote_inventory.reachable + else tracked_remote_branches + ) + for branch in PREFERRED_PR_BASE_BRANCHES: + if branch in available_branches: + return branch + if remote_inventory is not None and remote_inventory.reachable and remote_inventory.default_branch: + return remote_inventory.default_branch + + remote_head = git_output(("symbolic-ref", "--short", "refs/remotes/origin/HEAD")).strip() + if remote_head.startswith("origin/"): + return remote_head.removeprefix("origin/") + for branch in PREFERRED_PR_BASE_BRANCHES: + if git_output(("rev-parse", "--verify", f"refs/heads/{branch}")).strip(): + return branch + return "main" + + +def _remote_branch_inventory( + git_output: Callable[[Sequence[str]], str], +) -> _RemoteBranchInventory: + output = git_output( + ( + "ls-remote", + "--symref", + "origin", + "HEAD", + *(f"refs/heads/{branch}" for branch in PREFERRED_PR_BASE_BRANCHES), + ) + ) + heads: set[str] = set() + default_branch: str | None = None + for raw_line in output.splitlines(): + fields = raw_line.split() + if len(fields) == 3 and fields[0] == "ref:" and fields[2] == "HEAD": + ref = fields[1] + if ref.startswith("refs/heads/"): + default_branch = ref.removeprefix("refs/heads/") or None + continue + if len(fields) != 2 or not fields[1].startswith("refs/heads/"): + continue + branch = fields[1].removeprefix("refs/heads/") + if branch in PREFERRED_PR_BASE_BRANCHES: + heads.add(branch) + return _RemoteBranchInventory( + preferred_heads=frozenset(heads), + default_branch=default_branch, + reachable=bool(output.strip()), + ) + + +__all__ = ["PREFERRED_PR_BASE_BRANCHES", "detect_preferred_default_branch"] diff --git a/python/envctl_engine/shared/ports.py b/python/envctl_engine/shared/ports.py index 76f5f4fa..36552d9e 100644 --- a/python/envctl_engine/shared/ports.py +++ b/python/envctl_engine/shared/ports.py @@ -1,18 +1,24 @@ from __future__ import annotations +import errno import hashlib import json +import math import os import shutil import socket import subprocess +import threading import time import uuid +from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path from typing import Final from typing import Callable from envctl_engine.state.models import PortPlan +from envctl_engine.state.persistence import advisory_file_lock, fsync_directory _CORE_SERVICE_NAMES: Final[frozenset[str]] = frozenset( @@ -26,6 +32,12 @@ } ) +_PORT_GUARD_SHARDS: Final[int] = 64 +# ``flock`` behavior for separate descriptors in one process differs across +# platforms. A bounded in-process stripe set makes the contract explicit while +# the on-disk guard shards serialize independent processes. +_PORT_THREAD_GUARDS: Final[tuple[threading.RLock, ...]] = tuple(threading.RLock() for _ in range(_PORT_GUARD_SHARDS)) + class PortPlanner: def __init__( @@ -41,6 +53,7 @@ def __init__( lock_dir: str | None = None, session_id: str | None = None, stale_lock_seconds: int = 3600, + corrupt_lock_grace_seconds: float = 1.0, availability_checker: Callable[[int], bool] | None = None, pid_checker: Callable[[int], bool] | None = None, time_provider: Callable[[], float] | None = None, @@ -59,10 +72,11 @@ def __init__( self.supabase_api_base = supabase_api_base self.additional_service_bases = self._normalized_additional_service_bases(additional_service_bases or {}) self.lock_dir = Path(lock_dir or "/tmp/envctl-python-locks") - self.lock_dir.mkdir(parents=True, exist_ok=True) + self._ensure_real_directory(self.lock_dir) self.session_id = session_id or f"session-{uuid.uuid4().hex[:8]}" self.session_pid = os.getpid() self.stale_lock_seconds = max(stale_lock_seconds, 0) + self.corrupt_lock_grace_seconds = max(float(corrupt_lock_grace_seconds), 0.0) self.availability_checker = availability_checker self.pid_checker = pid_checker self.time_provider = time_provider or time.time @@ -75,6 +89,8 @@ def __init__( self.scope_key = str(scope_key or "global").strip() or "global" self.dynamic_main_dependency_ports = bool(dynamic_main_dependency_ports) self.max_port: Final[int] = 65000 + self._guard_dir = self.lock_dir / ".port-guards" + self._thread_guard_namespace = str(self.lock_dir.resolve(strict=False)) def plan_project( self, @@ -196,57 +212,83 @@ def attach_existing_port(self, plan: PortPlan, existing_port: int) -> PortPlan: def release(self, port: int, owner: str | None = None) -> None: lock_path = self._lock_path(port) - if not lock_path.exists(): - return - if owner is None: - lock_path.unlink(missing_ok=True) - self._emit("port.lock.release", port=port, owner=None, session=self.session_id) - return - try: - payload = json.loads(lock_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): + with self._port_guard(port): + payload = self._read_lock_payload(lock_path) + if payload is None or payload.get("session") != self.session_id: + return + if owner is not None and payload.get("owner") != owner: + return lock_path.unlink(missing_ok=True) self._emit("port.lock.release", port=port, owner=owner, session=self.session_id) - return - if payload.get("owner") == owner: + + def release_owned(self, port: int, owner: str, *, expected_session: str) -> bool: + """Release an exact logical owner from one persisted planner session.""" + + normalized_session = expected_session.strip() if isinstance(expected_session, str) else "" + if not normalized_session: + return False + + lock_path = self._lock_path(port) + with self._port_guard(port): + payload = self._read_lock_payload(lock_path) + if payload is None or payload.get("owner") != owner or payload.get("session") != normalized_session: + return False lock_path.unlink(missing_ok=True) - self._emit("port.lock.release", port=port, owner=owner, session=self.session_id) + self._emit("port.lock.release", port=port, owner=owner, session=normalized_session) + return True + + def reap_stale(self, port: int, owner: str | None = None) -> bool: + """Remove one proven-stale reservation without touching a live foreign session.""" + + lock_path = self._lock_path(port) + with self._port_guard(port): + if not lock_path.exists() or not self._lock_is_stale(lock_path): + return False + payload = self._read_lock_payload(lock_path) + if owner is not None and payload is not None and payload.get("owner") != owner: + return False + lock_path.unlink(missing_ok=True) + self._emit_reclaim(port=port, owner=owner, payload=payload) + return True + + def reap_stale_locks(self) -> int: + """Reap every proven-stale numeric reservation in this repository scope.""" + + reaped = 0 + for lock_path in tuple(self.lock_dir.glob("*.lock")): + port = self._port_from_lock_path(lock_path) + if port is not None and self.reap_stale(port): + reaped += 1 + return reaped def release_session(self) -> None: for lock_path in self.lock_dir.glob("*.lock"): - try: - payload = json.loads(lock_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): + port = self._port_from_lock_path(lock_path) + if port is None: + continue + with self._port_guard(port): + payload = self._read_lock_payload(lock_path) + if payload is None or payload.get("session") != self.session_id: + continue + lock_path.unlink(missing_ok=True) + self._emit("port.lock.release", port=port, owner=payload.get("owner"), session=self.session_id) + + def release_all(self) -> None: + for lock_path in self.lock_dir.glob("*.lock"): + port = self._port_from_lock_path(lock_path) + if port is None: lock_path.unlink(missing_ok=True) - self._emit( - "port.lock.release", port=self._port_from_lock_path(lock_path), owner=None, session=self.session_id - ) continue - if payload.get("session") == self.session_id: + with self._port_guard(port): + payload = self._read_lock_payload(lock_path) lock_path.unlink(missing_ok=True) self._emit( "port.lock.release", - port=self._port_from_lock_path(lock_path), - owner=payload.get("owner"), + port=port, + owner=payload.get("owner") if payload is not None else None, session=self.session_id, ) - def release_all(self) -> None: - for lock_path in self.lock_dir.glob("*.lock"): - owner = None - try: - payload = json.loads(lock_path.read_text(encoding="utf-8")) - owner = payload.get("owner") - except (OSError, json.JSONDecodeError): - owner = None - lock_path.unlink(missing_ok=True) - self._emit( - "port.lock.release", - port=self._port_from_lock_path(lock_path), - owner=owner, - session=self.session_id, - ) - def _lock_path(self, port: int) -> Path: return self.lock_dir / f"{port}.lock" @@ -370,87 +412,147 @@ def _normalize_project_identity(project: str) -> str: def _reserve_port(self, port: int, owner: str) -> bool: lock_path = self._lock_path(port) - if lock_path.exists() and self._lock_is_stale(lock_path): - stale_owner = None - stale_session = None - stale_pid = None - try: - stale_payload = json.loads(lock_path.read_text(encoding="utf-8")) - if isinstance(stale_payload, dict): - stale_owner = stale_payload.get("owner") - stale_session = stale_payload.get("session") - stale_pid = stale_payload.get("pid") - except (OSError, json.JSONDecodeError): - stale_owner = None - stale_session = None - stale_pid = None - lock_path.unlink(missing_ok=True) - self._emit( - "port.lock.reclaim", - port=port, - owner=owner, - session=self.session_id, - reclaimed_owner=stale_owner, - reclaimed_session=stale_session, - reclaimed_pid=stale_pid, - ) - if lock_path.exists(): - return False + with self._port_guard(port): + if lock_path.exists(): + if not self._lock_is_stale(lock_path): + return False + stale_payload = self._read_lock_payload(lock_path) + lock_path.unlink(missing_ok=True) + self._emit_reclaim(port=port, owner=owner, payload=stale_payload) - if not self._is_port_available(port): - return False + if not self._is_port_available(port): + return False - payload = { - "owner": owner, - "session": self.session_id, - "pid": self.session_pid, - "created_at": self.time_provider(), - } - try: - fd = lock_path.open("x", encoding="utf-8") - except FileExistsError: - return False - with fd: - fd.write(json.dumps(payload, sort_keys=True)) - self._emit("port.lock.acquire", port=port, owner=owner, session=self.session_id) - return True + payload = { + "owner": owner, + "session": self.session_id, + "pid": self.session_pid, + "created_at": self.time_provider(), + } + if not self._publish_lock(lock_path, payload, port=port): + return False + self._emit("port.lock.acquire", port=port, owner=owner, session=self.session_id) + return True def _lock_is_stale(self, lock_path: Path) -> bool: - try: - payload = json.loads(lock_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return True + payload = self._read_lock_payload(lock_path) + if payload is None: + return self._corrupt_lock_is_stale(lock_path) pid = payload.get("pid") - if isinstance(pid, int) and pid > 0: + if isinstance(pid, int) and not isinstance(pid, bool) and pid > 0: if self._pid_is_running(pid): return False return True created_at_raw = payload.get("created_at") if created_at_raw is None: - return False + return self._corrupt_lock_is_stale(lock_path) created_at = self._parse_created_at(created_at_raw) if created_at is None: - return True + return self._corrupt_lock_is_stale(lock_path) return (self.time_provider() - created_at) > self.stale_lock_seconds def _pid_is_running(self, pid: int) -> bool: if self.pid_checker is not None: - return self.pid_checker(pid) + try: + return bool(self.pid_checker(pid)) + except Exception: # noqa: BLE001 - an inconclusive probe must preserve a foreign lock + return True try: os.kill(pid, 0) - except OSError: + except ProcessLookupError: return False + except PermissionError: + return True + except OSError as exc: + return exc.errno != errno.ESRCH return True + def _corrupt_lock_is_stale(self, lock_path: Path) -> bool: + try: + modified_at = lock_path.stat().st_mtime + except FileNotFoundError: + return True + except OSError: + return False + return (self.time_provider() - modified_at) >= self.corrupt_lock_grace_seconds + + @staticmethod + def _read_lock_payload(lock_path: Path) -> dict[str, object] | None: + try: + payload = json.loads(lock_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + return payload if isinstance(payload, dict) else None + + def _publish_lock(self, lock_path: Path, payload: dict[str, object], *, port: int) -> bool: + """Publish a complete payload without ever exposing an empty lock path.""" + + self._ensure_real_directory(self._guard_dir) + pending_path = self._guard_dir / f"{self._guard_shard(port):02x}.pending" + pending_path.unlink(missing_ok=True) + flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(pending_path, flags, 0o600) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as pending_file: + pending_file.write(json.dumps(payload, sort_keys=True)) + pending_file.flush() + os.fsync(pending_file.fileno()) + try: + os.link(pending_path, lock_path) + except FileExistsError: + return False + fsync_directory(self.lock_dir) + return True + finally: + pending_path.unlink(missing_ok=True) + + def _emit_reclaim(self, *, port: int, owner: str | None, payload: dict[str, object] | None) -> None: + payload = payload or {} + self._emit( + "port.lock.reclaim", + port=port, + owner=owner, + session=self.session_id, + reclaimed_owner=payload.get("owner"), + reclaimed_session=payload.get("session"), + reclaimed_pid=payload.get("pid"), + ) + + @contextmanager + def _port_guard(self, port: int) -> Iterator[None]: + shard = self._guard_shard(port) + self._ensure_real_directory(self._guard_dir) + guard_path = self._guard_dir / f"{shard:02x}.guard" + thread_key = f"{self._thread_guard_namespace}:{shard}".encode("utf-8") + thread_index = int(hashlib.sha1(thread_key).hexdigest()[:8], 16) % len(_PORT_THREAD_GUARDS) + with _PORT_THREAD_GUARDS[thread_index]: + with advisory_file_lock(guard_path, exclusive=True): + yield + + @staticmethod + def _guard_shard(port: int) -> int: + return int(port) % _PORT_GUARD_SHARDS + + @staticmethod + def _ensure_real_directory(path: Path) -> None: + try: + path.mkdir(parents=True, exist_ok=False) + except FileExistsError: + pass + if path.is_symlink() or not path.is_dir(): + raise OSError(f"Port reservation directory is not a real directory: {path}") + def _parse_created_at(self, value: object) -> float | None: - if isinstance(value, (int, float)): - return float(value) + if isinstance(value, (int, float)) and not isinstance(value, bool): + parsed = float(value) + return parsed if math.isfinite(parsed) else None if not isinstance(value, str): return None try: - return float(value) + parsed = float(value) + return parsed if math.isfinite(parsed) else None except ValueError: pass iso = value @@ -459,8 +561,9 @@ def _parse_created_at(self, value: object) -> float | None: try: from datetime import datetime - return datetime.fromisoformat(iso).timestamp() - except ValueError: + parsed = datetime.fromisoformat(iso).timestamp() + return parsed if math.isfinite(parsed) else None + except (OverflowError, ValueError): return None def _is_port_available(self, port: int) -> bool: @@ -545,7 +648,10 @@ def _is_port_available_via_docker_publish_filter(self, port: int) -> bool: def _emit(self, event_name: str, **payload: object) -> None: if self.event_handler is None: return - self.event_handler(event_name, payload) + try: + self.event_handler(event_name, payload) + except Exception: # noqa: BLE001 - telemetry must never change lock ownership + return @staticmethod def _port_from_lock_path(lock_path: Path) -> int | None: diff --git a/python/envctl_engine/shared/process_cwd.py b/python/envctl_engine/shared/process_cwd.py new file mode 100644 index 00000000..5b2668a4 --- /dev/null +++ b/python/envctl_engine/shared/process_cwd.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import os +import subprocess +from collections.abc import Callable, Sequence +from typing import Any + + +def process_cwd( + pid: int, + *, + run_probe: Callable[[Sequence[str]], Any] | None = None, +) -> str | None: + """Read a process CWD portably, returning only explicit probe evidence.""" + + if pid <= 0: + return None + try: + return os.readlink(f"/proc/{pid}/cwd") + except OSError: + pass + + command = ["lsof", "-a", "-p", str(pid), "-d", "cwd", "-Fn"] + try: + completed = ( + run_probe(command) + if run_probe is not None + else subprocess.run( # noqa: S603 + command, + text=True, + encoding="utf-8", + errors="replace", + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=1.0, + check=False, + ) + ) + except (OSError, subprocess.TimeoutExpired): + return None + try: + returncode = int(getattr(completed, "returncode", 1)) + except (TypeError, ValueError): + return None + if returncode != 0: + return None + return parse_lsof_cwd(str(getattr(completed, "stdout", "") or "")) + + +def parse_lsof_cwd(output: str) -> str | None: + for line in output.splitlines(): + if line.startswith("n") and len(line) > 1: + return line[1:] + return None diff --git a/python/envctl_engine/shared/process_lifecycle_probe.py b/python/envctl_engine/shared/process_lifecycle_probe.py index f72a7c4e..018d5a23 100644 --- a/python/envctl_engine/shared/process_lifecycle_probe.py +++ b/python/envctl_engine/shared/process_lifecycle_probe.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable +import errno import os import re import shutil @@ -8,6 +9,8 @@ import socket import time +from envctl_engine.shared.process_cwd import process_cwd as read_process_cwd + class ProcessLifecycleProbeMixin: def is_pid_running(self, pid: int) -> bool: @@ -15,8 +18,12 @@ def is_pid_running(self, pid: int) -> bool: return False try: os.kill(pid, 0) - except OSError: + except ProcessLookupError: return False + except PermissionError: + return True + except OSError as exc: + return exc.errno != errno.ESRCH identity = self._pid_identity(pid) if identity is not None: try: @@ -29,6 +36,9 @@ def is_pid_running(self, pid: int) -> bool: return False return True + def process_cwd(self, pid: int) -> str | None: + return read_process_cwd(pid, run_probe=self.run_probe) + def wait_for_port(self, port: int, *, host: str = "127.0.0.1", timeout: float = 30.0) -> bool: if port <= 0: return False @@ -311,7 +321,7 @@ def _terminate_with_signal_sender( try: signal_sender(pid, signal.SIGTERM) except OSError: - return True + return not self.is_pid_running(pid) deadline = time.monotonic() + max(term_timeout, 0.0) while time.monotonic() < deadline: @@ -330,7 +340,7 @@ def _terminate_with_signal_sender( return True signal_sender(pid, signal.SIGKILL) except OSError: - return True + return not self.is_pid_running(pid) kill_deadline = time.monotonic() + max(kill_timeout, 0.0) while time.monotonic() < kill_deadline: @@ -356,12 +366,100 @@ def terminate(self, pid: int, *, term_timeout: float = 2.0, kill_timeout: float ) def terminate_process_group(self, pid: int, *, term_timeout: float = 2.0, kill_timeout: float = 1.0) -> bool: - return self._terminate_with_signal_sender( - pid, - term_timeout=term_timeout, - kill_timeout=kill_timeout, - signal_sender=os.killpg, - ) + if pid <= 0: + return True + try: + process_group_id = os.getpgid(pid) + except ProcessLookupError: + return not self.is_pid_running(pid) + except (PermissionError, OSError): + return False + if process_group_id <= 0: + return False + if process_group_id == os.getpgrp(): + return False + + initial_members = self._process_group_member_pids(process_group_id) + initial_identities = {member_pid: self._pid_identity(member_pid) for member_pid in initial_members or ()} + try: + os.killpg(process_group_id, signal.SIGTERM) + except OSError: + return not self._process_group_is_running(process_group_id) and not self.is_pid_running(pid) + + if self._wait_for_process_group_exit(process_group_id, timeout=term_timeout): + return True + if not self._process_group_escalation_is_safe( + process_group_id, + initial_members=initial_members, + initial_identities=initial_identities, + ): + return False + try: + os.killpg(process_group_id, signal.SIGKILL) + except OSError: + return not self._process_group_is_running(process_group_id) and not self.is_pid_running(pid) + return self._wait_for_process_group_exit(process_group_id, timeout=kill_timeout) + + def _process_group_member_pids(self, process_group_id: int) -> set[int] | None: + try: + completed = self.run_probe(["ps", "-axo", "pid=,pgid=,stat="]) + except OSError: + return None + if completed.returncode != 0: + return None + + members: set[int] = set() + for line in completed.stdout.splitlines(): + parts = line.strip().split(maxsplit=2) + if len(parts) != 3 or not parts[0].isdigit() or not parts[1].isdigit(): + continue + if int(parts[1]) != process_group_id or parts[2].upper().startswith("Z"): + continue + members.add(int(parts[0])) + return members + + def _process_group_is_running(self, process_group_id: int) -> bool: + members = self._process_group_member_pids(process_group_id) + if members is not None: + return bool(members) + try: + os.killpg(process_group_id, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError as exc: + return exc.errno != errno.ESRCH + return True + + def _wait_for_process_group_exit(self, process_group_id: int, *, timeout: float) -> bool: + deadline = time.monotonic() + max(timeout, 0.0) + while True: + if not self._process_group_is_running(process_group_id): + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + time.sleep(min(0.05, remaining)) + + def _process_group_escalation_is_safe( + self, + process_group_id: int, + *, + initial_members: set[int] | None, + initial_identities: dict[int, str | None], + ) -> bool: + if process_group_id == os.getpgrp(): + return False + current_members = self._process_group_member_pids(process_group_id) + if current_members is None or initial_members is None: + return self._process_group_is_running(process_group_id) + for member_pid in initial_members.intersection(current_members): + initial_identity = initial_identities.get(member_pid) + current_identity = self._pid_identity(member_pid) + if initial_identity is None or current_identity is None or current_identity == initial_identity: + return True + return False def _pid_identity(self, pid: int) -> str | None: if pid <= 0: diff --git a/python/envctl_engine/shared/process_termination.py b/python/envctl_engine/shared/process_termination.py new file mode 100644 index 00000000..3ff0d9bb --- /dev/null +++ b/python/envctl_engine/shared/process_termination.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import errno +import os +import signal +import time +from collections.abc import Callable +from typing import Any + +_IDENTITY_UNSET = object() + + +def terminate_pid( + pid: int | None, + *, + process_runner: object | None, + term_timeout: float, + kill_timeout: float, +) -> bool: + """Terminate a PID without treating an attempted signal as proof of exit. + + Process-runner implementations get the first opportunity to terminate the + process group and then the individual process. A false result (or an + exception) is not success: the raw-signal fallback confirms exit and + escalates to SIGKILL when necessary. + """ + + if not isinstance(pid, int) or pid <= 0: + return True + if pid in {os.getpid(), os.getppid()}: + return False + if not pid_is_running(process_runner, pid): + return True + initial_identity = pid_identity(process_runner, pid) + + group_terminator = getattr(process_runner, "terminate_process_group", None) + if callable(group_terminator): + try: + terminated = bool(group_terminator(pid, term_timeout=term_timeout, kill_timeout=kill_timeout)) + except Exception: # noqa: BLE001 + if callable(getattr(process_runner, "_process_group_is_running", None)): + return False + if _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity): + return True + else: + if not terminated: + # Killing only the recorded leader after group cleanup failed + # can orphan descendants and must never be reported as success. + return False + return _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity) + + terminator = getattr(process_runner, "terminate", None) + if callable(terminator): + try: + terminated = bool(terminator(pid, term_timeout=term_timeout, kill_timeout=kill_timeout)) + except Exception: # noqa: BLE001 + if _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity): + return True + else: + if _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity): + return True + if terminated: + # A capable runner that claims success while the same PID remains + # alive is not proof. Do not signal again: the PID may have raced + # with reuse between the runner and this verification. + return False + + return terminate_with_raw_signals( + process_runner, + pid, + term_timeout=term_timeout, + kill_timeout=kill_timeout, + initial_identity=initial_identity, + ) + + +def terminate_with_raw_signals( + process_runner: object | None, + pid: int, + *, + term_timeout: float, + kill_timeout: float, + initial_identity: str | None | object = _IDENTITY_UNSET, +) -> bool: + """Last-resort termination that proves the recorded process exited.""" + + if initial_identity is _IDENTITY_UNSET: + initial_identity = pid_identity(process_runner, pid) + assert isinstance(initial_identity, str) or initial_identity is None + if _pid_gone_or_replaced(process_runner, pid, initial_identity=initial_identity): + return True + term_sent = _send_signal(pid, signal.SIGTERM) + if term_sent is None: + return True + if not term_sent: + return not pid_is_running(process_runner, pid) + if wait_for_pid_exit( + process_runner, + pid, + timeout=term_timeout, + initial_identity=initial_identity, + ): + return True + kill_sent = _send_signal(pid, signal.SIGKILL) + if kill_sent is None: + return True + if not kill_sent: + return not pid_is_running(process_runner, pid) + return wait_for_pid_exit( + process_runner, + pid, + timeout=kill_timeout, + initial_identity=initial_identity, + ) + + +def _pid_gone_or_replaced( + process_runner: object | None, + pid: int, + *, + initial_identity: str | None, +) -> bool: + if not pid_is_running(process_runner, pid): + return True + if initial_identity is None: + return False + current_identity = pid_identity(process_runner, pid) + return current_identity is None or current_identity != initial_identity + + +def wait_for_pid_exit( + process_runner: object | None, + pid: int, + *, + timeout: float, + initial_identity: str | None, +) -> bool: + deadline = time.monotonic() + max(timeout, 0.0) + while True: + if not pid_is_running(process_runner, pid): + return True + if initial_identity is not None: + current_identity = pid_identity(process_runner, pid) + if current_identity is None or current_identity != initial_identity: + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + time.sleep(min(0.05, remaining)) + + +def pid_is_running(process_runner: object | None, pid: int) -> bool: + checker = getattr(process_runner, "is_pid_running", None) + if callable(checker): + try: + return bool(checker(pid)) + except Exception: # noqa: BLE001 + pass + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError as exc: + return exc.errno != errno.ESRCH + return True + + +def pid_identity(process_runner: object | None, pid: int) -> str | None: + identity_reader = getattr(process_runner, "_pid_identity", None) + if not callable(identity_reader): + return None + try: + identity = identity_reader(pid) + except Exception: # noqa: BLE001 + return None + if not isinstance(identity, str): + return None + normalized = identity.strip() + return normalized or None + + +def _send_signal(pid: int, requested_signal: signal.Signals) -> bool | None: + sender: Callable[[int, signal.Signals], Any] = os.kill + try: + sender(pid, requested_signal) + except ProcessLookupError: + return None + except OSError: + return False + return True diff --git a/python/envctl_engine/shared/protocols.py b/python/envctl_engine/shared/protocols.py index 07facf1b..fd2d8fa0 100644 --- a/python/envctl_engine/shared/protocols.py +++ b/python/envctl_engine/shared/protocols.py @@ -72,6 +72,8 @@ def update_final_port(self, plan: object, final_port: int, source: str = "retry" def release(self, port: int, owner: str | None = None) -> None: ... + def release_owned(self, port: int, owner: str, *, expected_session: str) -> bool: ... + def release_session(self) -> None: ... def release_all(self) -> None: ... @@ -108,11 +110,32 @@ def save_selected_stop_state( state: object, emit: Callable[..., None], runtime_map_builder: Callable[[object], dict[str, object]], + authoritative_project_names: Sequence[str] | None = None, ) -> dict[str, object]: ... - def load_latest(self, *, mode: str | None = None, strict_mode_match: bool = False) -> object | None: ... + def load_latest( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, + ) -> object | None: ... + + def load_all(self, *, mode: str | None = None) -> Sequence[object]: ... + + def has_active_runs(self) -> bool: ... + + def write_runtime_artifact( + self, + *, + run_id: str | None, + artifact_name: str, + text: str, + ) -> bool: ... + + def deactivate_run(self, run_id: str) -> bool: ... - def load_by_pointer(self, pointer_path: str) -> object: ... + def deactivate_runs(self, run_ids: Sequence[str]) -> bool: ... def purge(self, *, aggressive: bool = False) -> None: ... diff --git a/python/envctl_engine/startup/disabled_startup_resolution.py b/python/envctl_engine/startup/disabled_startup_resolution.py index 6f4801f5..ccd03fdf 100644 --- a/python/envctl_engine/startup/disabled_startup_resolution.py +++ b/python/envctl_engine/startup/disabled_startup_resolution.py @@ -6,6 +6,7 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_env import route_is_implicit_start +from envctl_engine.runtime.lifecycle_operation_lease import release_lifecycle_operation from envctl_engine.startup.finalization import ( build_planning_dashboard_state, headless_plan_output_only, @@ -68,6 +69,7 @@ def resolve_disabled_startup_mode( artifacts_started = time.monotonic() runtime._write_artifacts(run_state, session.selected_contexts, errors=[]) emit_phase(session, "artifacts_write", artifacts_started, status="ok") + release_lifecycle_operation(runtime) if route.command == "plan": validate_plan_agent_handoff(session, phase="disabled_startup_finalization") print_plan_dry_run_preview(session) @@ -139,13 +141,11 @@ def validate_plan_agent_handoff(session: StartupSession, *, phase: str) -> None: session=session, validate_plan_agent_handoff=validate_plan_agent_handoff, attach_plan_agent_terminal=attach_plan_agent_terminal, - print_headless_plan_session_summary=lambda session, *, attach_target: ( - print_headless_plan_session_summary( - session, - validate_plan_agent_handoff=validate_plan_agent_handoff, - print_fn=print_fn, - attach_target=attach_target, - ) + print_headless_plan_session_summary=lambda session, *, attach_target: print_headless_plan_session_summary( + session, + validate_plan_agent_handoff=validate_plan_agent_handoff, + print_fn=print_fn, + attach_target=attach_target, ), ), print_fn=print_fn, diff --git a/python/envctl_engine/startup/finalization.py b/python/envctl_engine/startup/finalization.py index 77ce8611..0796de92 100644 --- a/python/envctl_engine/startup/finalization.py +++ b/python/envctl_engine/startup/finalization.py @@ -6,6 +6,7 @@ from envctl_engine.planning.plan_agent.tmux_transport import attach_plan_agent_terminal from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.lifecycle_operation_lease import release_lifecycle_operation from envctl_engine.startup.finalization_plan_output import ( format_degraded_handoff_text_for_terminal, headless_plan_output_only, @@ -106,6 +107,7 @@ def finalize_successful_startup( artifacts_started = time.monotonic() runtime._write_artifacts(run_state, session.selected_contexts, errors=session.errors) emit_phase(session, "artifacts_write", artifacts_started, status="ok") + release_lifecycle_operation(runtime) if requirements_timing_enabled(session.effective_route) and not suppress_timing_output(session.effective_route): runtime._emit( "startup.debug_tty_group", @@ -213,13 +215,11 @@ def finalize_successful_startup_with_runtime( session=session, validate_plan_agent_handoff=validate_plan_agent_handoff, attach_plan_agent_terminal=attach_plan_agent_terminal, - print_headless_plan_session_summary=lambda session, *, attach_target: ( - print_headless_plan_session_summary( - session, - validate_plan_agent_handoff=validate_plan_agent_handoff, - print_fn=print_fn, - attach_target=attach_target, - ) + print_headless_plan_session_summary=lambda session, *, attach_target: print_headless_plan_session_summary( + session, + validate_plan_agent_handoff=validate_plan_agent_handoff, + print_fn=print_fn, + attach_target=attach_target, ), ), finalize_plan_agent_degraded_handoff=lambda session: finalize_plan_agent_degraded_handoff_with_runtime( @@ -257,13 +257,11 @@ def finalize_plan_agent_degraded_handoff_with_runtime( session=session, validate_plan_agent_handoff=validate_plan_agent_handoff, attach_plan_agent_terminal=attach_plan_agent_terminal, - print_headless_plan_session_summary=lambda session, *, attach_target: ( - print_headless_plan_session_summary( - session, - validate_plan_agent_handoff=validate_plan_agent_handoff, - print_fn=print_fn, - attach_target=attach_target, - ) + print_headless_plan_session_summary=lambda session, *, attach_target: print_headless_plan_session_summary( + session, + validate_plan_agent_handoff=validate_plan_agent_handoff, + print_fn=print_fn, + attach_target=attach_target, ), ), ) @@ -289,6 +287,7 @@ def finalize_plan_agent_degraded_handoff( build_success_run_state=build_success_run_state, emit_phase=emit_phase, ) + release_lifecycle_operation(runtime) render_plan_agent_degraded_handoff(session) if headless_plan_output_only(session): return 0 diff --git a/python/envctl_engine/startup/finalization_failure.py b/python/envctl_engine/startup/finalization_failure.py index d5d9928f..75a14cf5 100644 --- a/python/envctl_engine/startup/finalization_failure.py +++ b/python/envctl_engine/startup/finalization_failure.py @@ -8,8 +8,10 @@ from typing import Any from envctl_engine.startup.protocols import ProjectContextLike, StartupRuntime -from envctl_engine.startup.session import StartupSession +from envctl_engine.startup.session import StartupSession, unconfirmed_service_names from envctl_engine.startup.finalization_run_state import _build_run_state +from envctl_engine.requirements.core import dependency_definitions +from envctl_engine.runtime.lifecycle_requirement_ports import component_port_values from envctl_engine.state.models import RunState from envctl_engine.ui.color_policy import colors_enabled from envctl_engine.ui.path_links import local_paths_in_text, render_paths_in_terminal_text @@ -62,43 +64,105 @@ def _emit_failure(self, final_error: str) -> None: } if self.session.strict_truth_failed: failure_payload["services"] = sorted(self.session.merged_services) - self.runtime._emit("startup.failed", **failure_payload) + try: + self.runtime._emit("startup.failed", **failure_payload) + except Exception: # noqa: BLE001 + pass def _terminate_started_services(self) -> None: - started_services = { - service_name: service - for project_name in self.session.started_context_names - for service_name, service in self.session.services_by_project.get(project_name, {}).items() - } + started_services = dict(self.session.unterminated_services) + started_services.update( + { + service_name: service + for project_name in self.session.started_context_names + for service_name, service in self.session.services_by_project.get(project_name, {}).items() + } + ) if started_services: - self.runtime._terminate_started_services(started_services) + try: + failed_names = unconfirmed_service_names( + self.runtime._terminate_started_services(started_services), + started_services, + ) + except Exception: # noqa: BLE001 + failed_names = set(started_services) + for project_name in list(self.session.services_by_project): + retained = { + name: service + for name, service in self.session.services_by_project[project_name].items() + if name in failed_names + } + if retained: + self.session.services_by_project[project_name] = retained + else: + self.session.services_by_project.pop(project_name, None) + self.session.unterminated_services = { + name: service for name, service in started_services.items() if name in failed_names + } + for service in self.session.unterminated_services.values(): + service.status = "termination_failed" + service.degraded = True def _release_ports(self) -> None: - self.port_allocator(self.runtime).release_session() + if self.session.unterminated_services or _has_retained_managed_requirements(self.session): + try: + self.runtime._emit( + "port.session_release.skipped", + reason=( + "unterminated_services" + if self.session.unterminated_services + else "started_requirements_remain_tracked" + ), + services=sorted(self.session.unterminated_services), + ) + except Exception: # noqa: BLE001 + pass + return + try: + self.port_allocator(self.runtime).release_session() + except Exception: # noqa: BLE001 + pass def _write_failure_artifacts(self, final_error: str) -> None: - run_state = build_failure_run_state(self.runtime, self.session, final_error) artifacts_started = time.monotonic() + if self.session.preserve_existing_state_on_failure: + try: + self.runtime._emit( + "state.failure_write.skipped", + reason="existing_runtime_state_remains_authoritative", + run_id=self.session.run_id, + ) + self.emit_phase(self.session, "artifacts_write", artifacts_started, status="skipped") + except Exception: # noqa: BLE001 + pass + return + run_state = build_failure_run_state(self.runtime, self.session, final_error) self.runtime._write_artifacts(run_state, self.session.selected_contexts, errors=self.session.errors) - self.emit_phase(self.session, "artifacts_write", artifacts_started, status="error") + try: + self.emit_phase(self.session, "artifacts_write", artifacts_started, status="error") + except Exception: # noqa: BLE001 + pass def _print_final_status(self, final_error: str) -> None: - interactive_tty = failure_hyperlink_interactive_tty(self.runtime) - rendered_error = self.render_final_failure_status( - self.runtime, - self.session, - final_error, - interactive_tty=interactive_tty, - ) - self.print_fn( - render_paths_in_terminal_text( - rendered_error, - paths=local_paths_in_text(rendered_error), - env=self.runtime.env, - stream=self.output_stream, + try: + interactive_tty = failure_hyperlink_interactive_tty(self.runtime) + rendered_error = self.render_final_failure_status( + self.runtime, + self.session, + final_error, interactive_tty=interactive_tty, ) - ) + self.print_fn( + render_paths_in_terminal_text( + rendered_error, + paths=local_paths_in_text(rendered_error), + env=self.runtime.env, + stream=self.output_stream, + interactive_tty=interactive_tty, + ) + ) + except Exception: # noqa: BLE001 + pass def finalize_failed_startup( @@ -122,6 +186,21 @@ def finalize_failed_startup( ).finalize() +def _has_retained_managed_requirements(session: StartupSession) -> bool: + for requirements in session.requirements_by_project.values(): + for definition in dependency_definitions(): + component = requirements.component(definition.id) + if not bool(component.get("enabled", False)): + continue + if bool(component.get("external", False)) or bool(component.get("simulated", False)): + continue + if not bool(component.get("success", False)) and not str(component.get("container_name") or "").strip(): + continue + if component_port_values(component) or str(component.get("container_name") or "").strip(): + return True + return False + + def failure_hyperlink_interactive_tty(runtime: StartupRuntime) -> bool | None: link_mode = str(runtime.env.get("ENVCTL_UI_HYPERLINK_MODE", "")).strip().lower() return True if link_mode == "on" else None diff --git a/python/envctl_engine/startup/finalization_run_state.py b/python/envctl_engine/startup/finalization_run_state.py index 37a34cc9..8ca2f0c0 100644 --- a/python/envctl_engine/startup/finalization_run_state.py +++ b/python/envctl_engine/startup/finalization_run_state.py @@ -101,6 +101,8 @@ def _build_run_state(runtime: StartupRuntime, session: StartupSession, *, failed metadata["dashboard_shared_dependency_project"] = shared_dependency_project if session.warnings: metadata["warnings"] = list(session.warnings) + if session.unterminated_services: + metadata["termination_failed_services"] = sorted(session.unterminated_services) if session.plan_agent_launch_result is not None: launch_result = session.plan_agent_launch_result metadata["plan_agent_launch_status"] = str(getattr(launch_result, "status", "")).strip() @@ -197,9 +199,7 @@ def _build_run_state(runtime: StartupRuntime, session: StartupSession, *, failed return run_state -def _project_configured_services_metadata( - runtime: StartupRuntime, session: StartupSession -) -> dict[str, list[str]]: +def _project_configured_services_metadata(runtime: StartupRuntime, session: StartupSession) -> dict[str, list[str]]: configured: dict[str, list[str]] = {} for context in session.selected_contexts: service_types = [ diff --git a/python/envctl_engine/startup/protocols.py b/python/envctl_engine/startup/protocols.py index 29ff9962..61b3f05a 100644 --- a/python/envctl_engine/startup/protocols.py +++ b/python/envctl_engine/startup/protocols.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from pathlib import Path from typing import Protocol @@ -40,8 +40,14 @@ def _requirements_ready(self, result: RequirementsResult) -> bool: ... def _assert_project_services_post_start_truth( self, *, context: ProjectContextLike, services: dict[str, ServiceRecord] ) -> None: ... - def _terminate_started_services(self, services: dict[str, ServiceRecord]) -> None: ... - def _try_load_existing_state(self, *, mode: str, strict_mode_match: bool = False) -> RunState | None: ... + def _terminate_started_services(self, services: dict[str, ServiceRecord]) -> set[str]: ... + def _try_load_existing_state( + self, + *, + mode: str, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, + ) -> RunState | None: ... def _requirement_enabled(self, requirement_name: str, *, mode: str, route: Route | None = None) -> bool: ... def _service_enabled_for_mode(self, mode: str, service_name: str) -> bool: ... def _set_plan_port_from_component(self, plan: PortPlan, component: dict[str, object]) -> None: ... @@ -86,7 +92,7 @@ def _terminate_services_from_state( selected_services: set[str] | None = None, aggressive: bool = False, verify_ownership: bool = False, - ) -> None: ... + ) -> set[str]: ... def _enforce_runtime_readiness_contract(self, *, scope: str) -> bool: ... def _new_run_id(self) -> str: ... def _current_session_id(self) -> str | None: ... diff --git a/python/envctl_engine/startup/requirements_component_startup.py b/python/envctl_engine/startup/requirements_component_startup.py index a251b247..72e14291 100644 --- a/python/envctl_engine/startup/requirements_component_startup.py +++ b/python/envctl_engine/startup/requirements_component_startup.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Protocol +from envctl_engine.requirements.core import dependency_definitions from envctl_engine.requirements.orchestrator import FailureClass, RequirementOutcome from envctl_engine.requirements.supabase import ( evaluate_managed_supabase_reliability_contract, @@ -273,6 +274,8 @@ def _on_retry( failure_class: FailureClass, error: str | None, ) -> None: + if retry_port != failed_port: + self._release_failed_requirement_port(failed_port) reason_code = self.runtime.requirements.reason_code_for_failure(service_name, failure_class, error=error) self.runtime._emit( "requirements.retry", @@ -287,6 +290,23 @@ def _on_retry( error=(error or "").strip() or None, ) + def _release_failed_requirement_port(self, port: int) -> None: + release = getattr(self.runtime.port_planner, "release", None) + if not callable(release): + return + resource_keys = { + resource.legacy_port_key + for definition in dependency_definitions() + if definition.id == self.name + for resource in definition.resources + } + for owner_suffix in (*sorted(resource_keys), "requirements"): + try: + release(port, owner=f"{self.context.name}:{owner_suffix}") + except TypeError: + release(port) + return + def _finalize_outcome(self, outcome: RequirementOutcome) -> RequirementOutcome: if outcome.success and isinstance(self.native_effective_port, int) and self.native_effective_port > 0: outcome.final_port = self.native_effective_port diff --git a/python/envctl_engine/startup/requirements_execution.py b/python/envctl_engine/startup/requirements_execution.py index cf02b0f4..0845bcac 100644 --- a/python/envctl_engine/startup/requirements_execution.py +++ b/python/envctl_engine/startup/requirements_execution.py @@ -17,6 +17,7 @@ ) from envctl_engine.startup.startup_progress import suppress_timing_output from envctl_engine.startup.startup_selection_support import restart_include_requirements +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RequirementsResult _DOCKER_SOCKET_PATTERNS = ( @@ -61,7 +62,12 @@ def requirements_for_restart_context( if restart_include_requirements(route): return orchestrator.start_requirements_for_project(context, mode=mode, route=route) - previous = rt._try_load_existing_state(mode=mode, strict_mode_match=True) + previous = call_state_loader( + rt._try_load_existing_state, + mode=mode, + strict_mode_match=True, + project_names=[context.name], + ) if previous is not None: existing = previous.requirements.get(context.name) if isinstance(existing, RequirementsResult): diff --git a/python/envctl_engine/startup/requirements_project_startup.py b/python/envctl_engine/startup/requirements_project_startup.py index 9747253e..914224d0 100644 --- a/python/envctl_engine/startup/requirements_project_startup.py +++ b/python/envctl_engine/startup/requirements_project_startup.py @@ -15,7 +15,7 @@ external_dependency_resources, external_dependency_url, ) -from envctl_engine.requirements.orchestrator import RequirementOutcome +from envctl_engine.requirements.orchestrator import FailureClass, RequirementOutcome from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.runtime_context import resolve_port_allocator from envctl_engine.shared.parsing import parse_bool, parse_int @@ -27,6 +27,12 @@ REQUIREMENTS_PROGRESS_PROJECT_FLAG = "_requirements_progress_project" +def _has_reserved_ports(final_port: object, resources: dict[str, int]) -> bool: + return (isinstance(final_port, int) and not isinstance(final_port, bool) and final_port > 0) or any( + isinstance(port, int) and not isinstance(port, bool) and port > 0 for port in resources.values() + ) + + def requirements_parallel_platform_default() -> bool: return sys.platform != "darwin" @@ -248,31 +254,62 @@ def _run_components_parallel(self, enabled_definitions: list[Any], *, worker_cou future_map[future] = definition.id for future in concurrent.futures.as_completed(future_map): definition_id = future_map[future] - self.outcomes[definition_id] = future.result() + try: + self.outcomes[definition_id] = future.result() + except Exception as exc: # noqa: BLE001 + self.outcomes[definition_id] = self._failed_outcome( + definition_id, + self._plan_for_dependency(definition_id), + exc, + ) def _run_component(self, component: str, plan: object, *, strict: bool = False) -> RequirementOutcome: started = time.monotonic() with self.progress_state_lock: self.pending_requirements.discard(component) self.active_requirements.add(component) - self._emit_progress() + self._emit_progress_safely() try: - outcome = self.orchestrator.runtime._start_requirement_component( - self.context, - component, - plan, - self._reserve_next, - strict=strict, - route=self.route, - ) + try: + outcome = self.orchestrator.runtime._start_requirement_component( + self.context, + component, + plan, + self._reserve_next, + strict=strict, + route=self.route, + ) + except Exception as exc: # noqa: BLE001 + outcome = self._failed_outcome(component, plan, exc) duration_ms = round((time.monotonic() - started) * 1000.0, 2) self.component_timings_ms[component] = duration_ms - self._emit_component_timing(component, outcome=outcome, enabled=True, duration_ms=duration_ms) + try: + self._emit_component_timing(component, outcome=outcome, enabled=True, duration_ms=duration_ms) + except Exception: # noqa: BLE001 + pass return outcome finally: with self.progress_state_lock: self.active_requirements.discard(component) - self._emit_progress() + self._emit_progress_safely() + + @staticmethod + def _failed_outcome(component: str, plan: object, error: BaseException) -> RequirementOutcome: + return RequirementOutcome( + service_name=component, + success=False, + requested_port=int(getattr(plan, "requested", 0) or 0), + final_port=int(getattr(plan, "final", 0) or 0), + retries=0, + failure_class=FailureClass.HARD_START_FAILURE, + error=str(error), + ) + + def _emit_progress_safely(self) -> None: + try: + self._emit_progress() + except Exception: # noqa: BLE001 + return def _build_components( self, @@ -301,6 +338,18 @@ def _build_components( "error": outcome.error, "container_name": outcome.container_name, } + if ( + enabled_lookup[definition.id] + and not external_lookup[definition.id] + and _has_reserved_ports( + outcome.final_port, + resources, + ) + ): + raw_session_id = getattr(resolve_port_allocator(rt), "session_id", None) + session_id = raw_session_id.strip() if isinstance(raw_session_id, str) else "" + if session_id: + components[definition.id]["port_lock_session"] = session_id if external_lookup[definition.id]: components[definition.id].update( { @@ -318,9 +367,7 @@ def _emit_progress(self) -> None: return progress_project_flag = self.route.flags.get(REQUIREMENTS_PROGRESS_PROJECT_FLAG) progress_project = ( - str(progress_project_flag).strip() - if progress_project_flag is not None - else self.context.name + str(progress_project_flag).strip() if progress_project_flag is not None else self.context.name ) progress_message = format_requirements_progress_message( active=self.active_requirements, diff --git a/python/envctl_engine/startup/restart_prestop_support.py b/python/envctl_engine/startup/restart_prestop_support.py index aa0d1e98..639387cb 100644 --- a/python/envctl_engine/startup/restart_prestop_support.py +++ b/python/envctl_engine/startup/restart_prestop_support.py @@ -1,11 +1,13 @@ from __future__ import annotations -from dataclasses import dataclass -from pathlib import Path +from dataclasses import dataclass, field from typing import Any, Callable, Iterable from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.lifecycle_requirement_ports import release_port_reservation +from envctl_engine.state.lookup import call_state_loader from envctl_engine.runtime.runtime_context import resolve_port_allocator, resolve_process_runtime +from envctl_engine.shared.process_cwd import process_cwd from envctl_engine.shared.services import service_project_name, service_slug_from_record from envctl_engine.startup.startup_selection_support import ( _restart_selected_services, @@ -13,6 +15,8 @@ restart_target_projects, restart_target_projects_for_selected_services, ) +from envctl_engine.startup.run_reuse_dashboard_restore import metadata_without_dashboard_stopped_services +from envctl_engine.startup.session import metadata_with_state_sources @dataclass(frozen=True, slots=True) @@ -40,19 +44,16 @@ class RestartPrestopSelection: class RestartOrphanListenerScan: ports_by_type: dict[str, set[int]] selected_by_cwd: dict[str, set[str]] + owners_by_cwd_type: dict[tuple[str, str], tuple[str, ...]] = field(default_factory=dict) + port_lock_sessions_by_cwd_type: dict[tuple[str, str], tuple[str, ...]] = field(default_factory=dict) @dataclass(frozen=True, slots=True) class RestartOrphanListenerMatch: pid: int port: int - - -def process_cwd(pid: int) -> str | None: - try: - return str(Path(f"/proc/{pid}/cwd").resolve()) - except OSError: - return None + owner_candidates: tuple[str, ...] = () + port_lock_sessions: tuple[str, ...] = () def restart_fallback_start_route(route: Route, *, restart_lookup_mode: str) -> Route: @@ -68,7 +69,11 @@ def restart_fallback_start_route(route: Route, *, restart_lookup_mode: str) -> R def restart_prestop_state(*, route: Route, runtime: Any) -> RestartPrestopState: restart_lookup_mode = runtime._effective_start_mode(route) - resumed = runtime._try_load_existing_state(mode=restart_lookup_mode) + resumed = call_state_loader( + runtime._try_load_existing_state, + mode=restart_lookup_mode, + project_names=route.projects or None, + ) if resumed is not None and resumed.mode != restart_lookup_mode: runtime._emit( "restart.state_mode_mismatch", @@ -133,7 +138,7 @@ def handle_restart_prestop( runtime: Any, session: Any, suppress_progress_output: Callable[[Route], bool], - terminate_restart_orphan_listeners: Callable[..., None], + terminate_restart_orphan_listeners: Callable[..., set[int]], spinner_factory: Callable[..., Any], use_spinner_policy_fn: Callable[[Any], Any], resolve_spinner_policy_fn: Callable[[dict[str, str]], Any], @@ -181,17 +186,30 @@ def handle_restart_prestop( message="Restarting services...", ) try: - runtime._terminate_services_from_state( + termination_result = runtime._terminate_services_from_state( resumed, selected_services=selected_services, aggressive=False, verify_ownership=True, ) - terminate_restart_orphan_listeners( + failed_services = _failed_service_names(termination_result) + if failed_services: + session.preserve_existing_state_on_failure = True + raise RuntimeError( + "Restart aborted because existing services could not be stopped: " + + ", ".join(sorted(failed_services)) + ) + failed_orphan_pids = terminate_restart_orphan_listeners( state=resumed, selected_services=selected_services, aggressive=True, ) + if failed_orphan_pids: + session.preserve_existing_state_on_failure = True + raise RuntimeError( + "Restart aborted because orphan listeners could not be stopped: " + + ", ".join(str(pid) for pid in sorted(failed_orphan_pids)) + ) preservation = restart_prestop_preservation( resumed, selected_services=selected_services, @@ -202,6 +220,17 @@ def handle_restart_prestop( runtime._release_requirement_ports(requirements) session.preserved_requirements = dict(preservation.preserved_requirements) session.preserved_services = dict(preservation.preserved_services) + requested_service_names = { + str(name).strip() for name in route.flags.get("services", []) if str(name).strip() + } + inherited_metadata = metadata_without_dashboard_stopped_services( + {**dict(resumed.metadata), **dict(session.base_metadata)}, + restored_service_names=requested_service_names, + ) + session.base_metadata = metadata_with_state_sources( + inherited_metadata, + resumed, + ) if use_prestop_spinner: prestop_spinner.succeed("Restart pre-stop complete") runtime._emit( @@ -241,6 +270,12 @@ def handle_restart_prestop( return None +def _failed_service_names(result: object) -> set[str]: + if not isinstance(result, (set, frozenset)): + return set() + return {str(name).strip() for name in result if str(name).strip()} + + def restart_prestop_preservation( state: Any, *, @@ -342,6 +377,8 @@ def restart_orphan_listener_scan( "frontend": set(range(int(frontend_port_base), int(frontend_port_base) + span)), } selected_by_cwd: dict[str, set[str]] = {} + owners_by_cwd_type: dict[tuple[str, str], tuple[str, ...]] = {} + port_lock_sessions_by_cwd_type: dict[tuple[str, str], tuple[str, ...]] = {} for service_name, service in getattr(state, "services", {}).items(): if service_name not in selected_services: continue @@ -350,6 +387,20 @@ def restart_orphan_listener_scan( if service_type not in ports_by_type or not cwd: continue selected_by_cwd.setdefault(cwd, set()).add(service_type) + project = str(service_project_name(service) or service_name.rsplit(" ", 1)[0]).strip() + if project: + owners_by_cwd_type[(cwd, service_type)] = ( + f"{project}:{service_type}", + f"{project}:services:{service_type}-launch", + f"{project}:services", + ) + raw_port_lock_session = getattr(service, "port_lock_session", None) + port_lock_session = raw_port_lock_session.strip() if isinstance(raw_port_lock_session, str) else "" + if port_lock_session: + key = (cwd, service_type) + port_lock_sessions_by_cwd_type[key] = tuple( + dict.fromkeys((*port_lock_sessions_by_cwd_type.get(key, ()), port_lock_session)) + ) for attr_name in ("actual_port", "requested_port"): port = getattr(service, attr_name, None) if isinstance(port, int) and port > 0: @@ -357,6 +408,8 @@ def restart_orphan_listener_scan( return RestartOrphanListenerScan( ports_by_type=ports_by_type, selected_by_cwd=selected_by_cwd, + owners_by_cwd_type=owners_by_cwd_type, + port_lock_sessions_by_cwd_type=port_lock_sessions_by_cwd_type, ) @@ -377,7 +430,14 @@ def restart_matching_orphan_listeners( if process_cwd(pid) != cwd: continue seen_pids.add(pid) - matches.append(RestartOrphanListenerMatch(pid=pid, port=port)) + matches.append( + RestartOrphanListenerMatch( + pid=pid, + port=port, + owner_candidates=scan.owners_by_cwd_type.get((cwd, service_type), ()), + port_lock_sessions=scan.port_lock_sessions_by_cwd_type.get((cwd, service_type), ()), + ) + ) return matches @@ -392,8 +452,8 @@ def terminate_restart_orphan_listeners( listener_pids_for_port: Callable[[int], Iterable[int]] | None, process_cwd: Callable[[int], str | None], terminate_pid: Callable[..., bool] | None, - release_port: Callable[[int], None], -) -> None: + port_planner: Any, +) -> set[int]: scan = restart_orphan_listener_scan( state, selected_services=selected_services, @@ -402,17 +462,29 @@ def terminate_restart_orphan_listeners( port_spacing=port_spacing, ) if not scan.selected_by_cwd: - return + return set() if not callable(listener_pids_for_port) or not callable(terminate_pid): - return + return set() matches = restart_matching_orphan_listeners( scan, listener_pids_for_port=listener_pids_for_port, process_cwd=process_cwd, ) + failed_pids: set[int] = set() for match in matches: if terminate_pid(match.pid, term_timeout=0.5 if aggressive else 2.0, kill_timeout=1.0): - release_port(match.port) + expected_sessions = match.port_lock_sessions or (None,) + for expected_session in expected_sessions: + if release_port_reservation( + port_planner, + match.port, + owner_candidates=match.owner_candidates, + expected_session=expected_session, + ): + break + else: + failed_pids.add(match.pid) + return failed_pids def terminate_restart_orphan_listeners_with_runtime( @@ -421,10 +493,10 @@ def terminate_restart_orphan_listeners_with_runtime( state: Any, selected_services: set[str], aggressive: bool, -) -> None: +) -> set[int]: process_runtime = resolve_process_runtime(runtime) port_allocator = resolve_port_allocator(runtime) - terminate_restart_orphan_listeners( + return terminate_restart_orphan_listeners( state=state, selected_services=selected_services, aggressive=aggressive, @@ -432,7 +504,7 @@ def terminate_restart_orphan_listeners_with_runtime( frontend_port_base=int(runtime.config.frontend_port_base), port_spacing=int(getattr(runtime.config, "port_spacing", 20) or 20), listener_pids_for_port=getattr(runtime, "_listener_pids_for_port", None), - process_cwd=process_cwd, + process_cwd=getattr(process_runtime, "process_cwd", process_cwd), terminate_pid=getattr(process_runtime, "terminate", None), - release_port=port_allocator.release, + port_planner=port_allocator, ) diff --git a/python/envctl_engine/startup/resume_orchestrator.py b/python/envctl_engine/startup/resume_orchestrator.py index c9773809..45e512fa 100644 --- a/python/envctl_engine/startup/resume_orchestrator.py +++ b/python/envctl_engine/startup/resume_orchestrator.py @@ -6,7 +6,9 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_startup_support import mark_run_reused +from envctl_engine.runtime.lifecycle_operation_lease import release_lifecycle_operation from envctl_engine.state.models import RunState +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.ui.spinner import spinner, spinner_enabled, use_spinner_policy from envctl_engine.ui.spinner_service import emit_spinner_policy, resolve_spinner_policy @@ -47,9 +49,17 @@ def emit_phase(phase: str, started_at: float, **extra: object) -> None: ) state_load_started = time.monotonic() - state = rt._try_load_existing_state( # type: ignore[attr-defined] + raw_state_projects = route.flags.get("_state_project_names") + state_projects = ( + [str(project).strip() for project in raw_state_projects if str(project).strip()] + if isinstance(raw_state_projects, list) + else list(route.projects) + ) + state = call_state_loader( + rt._try_load_existing_state, # type: ignore[attr-defined] mode=route.mode, strict_mode_match=rt._state_lookup_strict_mode_match(route), # type: ignore[attr-defined] + project_names=state_projects or None, ) emit_phase("state_load", state_load_started, found=state is not None) if state is None: @@ -156,6 +166,17 @@ def emit_phase(phase: str, started_at: float, **extra: object) -> None: ) enter_interactive = bool(rt._should_enter_resume_interactive(route)) # type: ignore[attr-defined] + if not rt._state_has_resumable_services(state): # type: ignore[attr-defined] + rt._emit( # type: ignore[attr-defined] + "resume.empty_state", + run_id=state.run_id, + reason="no_resumable_services", + ) + print("No active services to resume.") + return 1 + + release_lifecycle_operation(rt) + if not enter_interactive: current_session_id = getattr(rt, "_current_session_id", None) session_id = current_session_id() if callable(current_session_id) else None @@ -173,14 +194,6 @@ def emit_phase(phase: str, started_at: float, **extra: object) -> None: print("Warning: stale services detected during resume: " + ", ".join(missing_services)) if restore_errors: print("Warning: resume restore encountered errors: " + "; ".join(restore_errors)) - if not rt._state_has_resumable_services(state): # type: ignore[attr-defined] - rt._emit( # type: ignore[attr-defined] - "resume.empty_state", - run_id=state.run_id, - reason="no_resumable_services", - ) - print("No active services to resume.") - return 1 if enter_interactive: return rt._run_interactive_dashboard_loop(state) # type: ignore[attr-defined] return 0 diff --git a/python/envctl_engine/startup/resume_restore_project.py b/python/envctl_engine/startup/resume_restore_project.py index efb52a0a..233814ee 100644 --- a/python/envctl_engine/startup/resume_restore_project.py +++ b/python/envctl_engine/startup/resume_restore_project.py @@ -5,6 +5,7 @@ import time from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.lifecycle_service_termination import release_service_ports from envctl_engine.state.models import RunState, ServiceRecord from envctl_engine.startup.resume_restore_policy import ( _configured_restore_service_types, @@ -349,9 +350,7 @@ def _stop_stale_services( ) if terminated: terminated_count += 1 - port = self.rt._service_port(service) # type: ignore[attr-defined] - if port is not None: - self.port_allocator.release(port) # type: ignore[attr-defined] + release_service_ports(self.rt, name=service_name, service=service) return terminated_count def _report_progress( diff --git a/python/envctl_engine/startup/run_reuse_dashboard_restore.py b/python/envctl_engine/startup/run_reuse_dashboard_restore.py index 05e23a34..759efee4 100644 --- a/python/envctl_engine/startup/run_reuse_dashboard_restore.py +++ b/python/envctl_engine/startup/run_reuse_dashboard_restore.py @@ -6,6 +6,7 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.startup.run_reuse_decision import mark_run_reused +from envctl_engine.startup.session import metadata_with_state_sources def dashboard_stopped_service_entries(state: object) -> list[dict[str, str]]: @@ -104,9 +105,12 @@ def _apply_restore(self, *, restore_entries: list[dict[str, str]], contexts_to_s stopped_service_names = sorted({entry["name"] for entry in restore_entries}) stopped_service_types = sorted({entry["type"] for entry in restore_entries}) target_project_names = sorted({entry["project"] for entry in restore_entries}, key=str.casefold) - self.session.base_metadata = metadata_without_dashboard_stopped_services( - mark_run_reused(self.candidate_state.metadata, reason="restore_stopped_services"), - restored_service_names=set(stopped_service_names), + self.session.base_metadata = metadata_with_state_sources( + metadata_without_dashboard_stopped_services( + mark_run_reused(self.candidate_state.metadata, reason="restore_stopped_services"), + restored_service_names=set(stopped_service_names), + ), + self.candidate_state, ) self.session.preserved_services = dict(self.candidate_state.services) self.session.preserved_requirements = dict(self.candidate_state.requirements) diff --git a/python/envctl_engine/startup/run_reuse_decision.py b/python/envctl_engine/startup/run_reuse_decision.py index c97bcfc6..b1470c2f 100644 --- a/python/envctl_engine/startup/run_reuse_decision.py +++ b/python/envctl_engine/startup/run_reuse_decision.py @@ -5,6 +5,7 @@ from typing import Any, Mapping from envctl_engine.runtime.command_router import Route +from envctl_engine.state.lookup import call_state_loader from envctl_engine.state.models import RunState from envctl_engine.startup.run_reuse_identity import ( _auto_resume_start_enabled, @@ -56,20 +57,37 @@ def evaluate(self) -> RunReuseDecision: selected_payload=selected_payload, startup_enabled=startup_enabled, ) - if not _auto_resume_start_enabled(self.route): + auto_resume_enabled = _auto_resume_start_enabled(self.route) + explicit_fresh_start = bool(self.route.flags.get("no_resume")) + if not auto_resume_enabled and not explicit_fresh_start: return self._fresh_without_state( reason="auto_resume_disabled", selected_payload=selected_payload, startup_enabled=startup_enabled, ) - candidate = self.runtime._try_load_existing_state(mode=self.runtime_mode, strict_mode_match=True) + candidate = call_state_loader( + self.runtime._try_load_existing_state, + mode=self.runtime_mode, + strict_mode_match=True, + project_names=[identity.name for identity in selected_identities], + ) if candidate is None: return self._fresh_without_state( reason="no_matching_state", selected_payload=selected_payload, startup_enabled=startup_enabled, ) + if explicit_fresh_start: + state_identities = project_identities_from_state(self.runtime, candidate) + return RunReuseDecision( + candidate_state=candidate, + decision_kind="fresh_run", + reason="explicit_no_resume", + selected_projects=selected_payload, + state_projects=identities_to_payload(state_identities), + startup_enabled=startup_enabled, + ) return self._evaluate_candidate( candidate=candidate, selected_identities=selected_identities, diff --git a/python/envctl_engine/startup/run_reuse_fresh_start.py b/python/envctl_engine/startup/run_reuse_fresh_start.py index 2cb75fc6..6f395c57 100644 --- a/python/envctl_engine/startup/run_reuse_fresh_start.py +++ b/python/envctl_engine/startup/run_reuse_fresh_start.py @@ -6,41 +6,60 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.shared.services import service_project_name, service_slug_from_record +from envctl_engine.startup.session import metadata_with_state_sources from envctl_engine.startup.startup_selection_support import _restart_service_types_for_project def fresh_start_replacement_services( *, - route: Route, selected_contexts: list[object], candidate_state: object, - configured_service_types: set[str], - additional_services: tuple[object, ...], project_name_from_service: Callable[[str], str], + route: Route | None = None, + additional_services: tuple[object, ...] = (), ) -> set[str]: - target_projects = {str(context.name).strip().lower() for context in selected_contexts} - target_projects.discard("") - if not target_projects: - return set() - selected_by_project = { - str(context.name).strip().lower(): _restart_service_types_for_project( - route=route, - project_name=str(context.name), - default_service_types=configured_service_types, - additional_services=additional_services, - ) + """Select old services that overlap the requested fresh-start scope.""" + + target_projects = { + str(context.name).strip().casefold(): str(context.name).strip() for context in selected_contexts if str(context.name).strip() } - selected: set[str] = set() + if not target_projects: + return set() + + services_by_project: dict[str, list[tuple[str, object, str]]] = { + project: [] for project in target_projects + } for service_name, service in getattr(candidate_state, "services", {}).items(): - project = service_project_name(service) or project_name_from_service(service_name) - project_key = str(project).strip().lower() - if project_key not in target_projects: + project = str(service_project_name(service) or project_name_from_service(service_name)).strip().casefold() + if project not in target_projects: + continue + services_by_project[project].append( + (service_name, service, _service_type(service_name, service)) + ) + + selected: set[str] = set() + for project, project_services in services_by_project.items(): + if route is None or not _route_has_app_service_scope(route): + selected.update(name for name, _service, _service_type_name in project_services) continue - service_type = service_slug_from_record(service) - if service_type and service_type in selected_by_project.get(project_key, set()): - selected.add(service_name) + configured_types = { + service_type_name + for _name, _service, service_type_name in project_services + if service_type_name + } + selected_types = _restart_service_types_for_project( + route=route, + project_name=target_projects[project], + default_service_types=configured_types, + additional_services=additional_services, + ) + selected.update( + name + for name, _service, service_type_name in project_services + if service_type_name in selected_types + ) return selected @@ -50,25 +69,57 @@ class FreshStartServiceReplacer: session: Any candidate_state: Any reason: str - fresh_start_replacement_services: Callable[..., set[str]] announce_session_identifiers: Callable[[Any], None] report_progress: Callable[[Route, str], None] - terminate_restart_orphan_listeners: Callable[..., None] + terminate_restart_orphan_listeners: Callable[..., set[int]] def replace(self) -> None: - if self.reason != "startup_fingerprint_mismatch": - return route = self.session.effective_route - if route.flags.get("runtime_scope") == "dependencies": - return - selected_services = self.fresh_start_replacement_services(self.session, candidate_state=self.candidate_state) - if not selected_services: - return - self.announce_session_identifiers(self.session) - self.report_progress( - route, - f"Startup selection changed; replacing {len(selected_services)} existing service(s)...", + dependencies_only = route.flags.get("runtime_scope") == "dependencies" + selected_services = ( + set() + if dependencies_only + else fresh_start_replacement_services( + selected_contexts=list(self.session.selected_contexts), + candidate_state=self.candidate_state, + project_name_from_service=self.runtime._project_name_from_service, + route=route, + additional_services=tuple( + getattr(getattr(self.runtime, "config", None), "additional_services", ()) or () + ), + ) + ) + target_projects = { + str(context.name).strip().casefold() + for context in self.session.selected_contexts + if str(context.name).strip() + } + preserved_services = ( + dict(self.candidate_state.services) + if dependencies_only + else { + name: service + for name, service in self.candidate_state.services.items() + if name not in selected_services + } ) + replace_requirements = _fresh_start_replaces_requirements(route) + if replace_requirements: + preserved_requirements, requirements_to_release = _split_requirements( + self.candidate_state, + target_projects=target_projects, + ) + else: + preserved_requirements = dict(self.candidate_state.requirements) + requirements_to_release = {} + if selected_services: + self.announce_session_identifiers(self.session) + progress_message = ( + f"Auto-resume disabled; replacing {len(selected_services)} existing service(s)..." + if self.reason == "explicit_no_resume" + else f"Startup selection changed; replacing {len(selected_services)} existing service(s)..." + ) + self.report_progress(route, progress_message) self.runtime._emit( "state.run_reuse.replace_existing_services", run_id=self.candidate_state.run_id, @@ -76,16 +127,41 @@ def replace(self) -> None: reason=self.reason, selected_services=sorted(selected_services), ) - self.runtime._terminate_services_from_state( + if selected_services: + termination_result = self.runtime._terminate_services_from_state( + self.candidate_state, + selected_services=selected_services, + aggressive=False, + verify_ownership=True, + ) + failed_services = _failed_service_names(termination_result) + if failed_services: + self.session.preserve_existing_state_on_failure = True + raise RuntimeError( + "Fresh start aborted because existing services could not be stopped: " + + ", ".join(sorted(failed_services)) + ) + failed_orphan_pids = self.terminate_restart_orphan_listeners( + state=self.candidate_state, + selected_services=selected_services, + aggressive=True, + ) + if failed_orphan_pids: + self.session.preserve_existing_state_on_failure = True + raise RuntimeError( + "Fresh start aborted because orphan listeners could not be stopped: " + + ", ".join(str(pid) for pid in sorted(failed_orphan_pids)) + ) + for requirements in requirements_to_release.values(): + self.runtime._release_requirement_ports(requirements) + self.session.preserved_services = preserved_services + self.session.preserved_requirements = preserved_requirements + self.session.base_metadata = metadata_with_state_sources( + { + **dict(self.candidate_state.metadata), + **dict(self.session.base_metadata), + }, self.candidate_state, - selected_services=selected_services, - aggressive=False, - verify_ownership=True, - ) - self.terminate_restart_orphan_listeners( - state=self.candidate_state, - selected_services=selected_services, - aggressive=True, ) @@ -95,17 +171,15 @@ def replace_existing_project_services_for_fresh_start( session: Any, candidate_state: Any, reason: str, - fresh_start_replacement_services: Callable[..., set[str]], announce_session_identifiers: Callable[[Any], None], report_progress: Callable[[Route, str], None], - terminate_restart_orphan_listeners: Callable[..., None], + terminate_restart_orphan_listeners: Callable[..., set[int]], ) -> None: FreshStartServiceReplacer( runtime=runtime, session=session, candidate_state=candidate_state, reason=reason, - fresh_start_replacement_services=fresh_start_replacement_services, announce_session_identifiers=announce_session_identifiers, report_progress=report_progress, terminate_restart_orphan_listeners=terminate_restart_orphan_listeners, @@ -118,26 +192,70 @@ def replace_existing_project_services_for_fresh_start_with_defaults( session: Any, candidate_state: Any, reason: str, - configured_service_types: set[str], - additional_services: tuple[object, ...], announce_session_identifiers: Callable[[Any], None], report_progress: Callable[[Route, str], None], - terminate_restart_orphan_listeners: Callable[..., None], + terminate_restart_orphan_listeners: Callable[..., set[int]], ) -> None: replace_existing_project_services_for_fresh_start( runtime=runtime, session=session, candidate_state=candidate_state, reason=reason, - fresh_start_replacement_services=lambda _, *, candidate_state: fresh_start_replacement_services( - route=session.effective_route, - selected_contexts=list(session.selected_contexts), - candidate_state=candidate_state, - configured_service_types=configured_service_types, - additional_services=additional_services, - project_name_from_service=runtime._project_name_from_service, - ), announce_session_identifiers=announce_session_identifiers, report_progress=report_progress, terminate_restart_orphan_listeners=terminate_restart_orphan_listeners, ) + + +def _split_requirements( + state: object, + *, + target_projects: set[str], +) -> tuple[dict[str, object], dict[str, object]]: + preserved: dict[str, object] = {} + replaced: dict[str, object] = {} + for name, requirements in getattr(state, "requirements", {}).items(): + project = str(getattr(requirements, "project", "") or name).strip().casefold() + destination = replaced if project in target_projects else preserved + destination[name] = requirements + return preserved, replaced + + +def _failed_service_names(result: object) -> set[str]: + if not isinstance(result, (set, frozenset)): + return set() + return {str(name).strip() for name in result if str(name).strip()} + + +def _route_has_app_service_scope(route: Route) -> bool: + runtime_scope = str(route.flags.get("runtime_scope") or "").strip().lower() + if runtime_scope in {"backend", "frontend", "fullstack", "dependencies"}: + return True + if isinstance(route.flags.get("services"), list) and bool(route.flags.get("services")): + return True + if isinstance(route.flags.get("restart_service_types"), list): + return True + return route.flags.get("launch_backend") is False or route.flags.get("launch_frontend") is False + + +def _fresh_start_replaces_requirements(route: Route) -> bool: + runtime_scope = str(route.flags.get("runtime_scope") or "").strip().lower() + if runtime_scope in {"dependencies", "entire-system"}: + return True + if _route_has_app_service_scope(route): + return False + if route.flags.get("launch_dependencies") is False: + return False + return True + + +def _service_type(service_name: str, service: object) -> str: + explicit = service_slug_from_record(service) + if explicit: + return explicit + lowered = str(service_name).strip().casefold() + for suffix in ("backend", "frontend"): + if lowered.endswith(f" {suffix}"): + return suffix + parts = lowered.rsplit(" ", 1) + return parts[-1] if len(parts) == 2 else "" diff --git a/python/envctl_engine/startup/run_reuse_identity.py b/python/envctl_engine/startup/run_reuse_identity.py index bd8e66f4..c26f5a8d 100644 --- a/python/envctl_engine/startup/run_reuse_identity.py +++ b/python/envctl_engine/startup/run_reuse_identity.py @@ -27,19 +27,31 @@ def build_startup_identity_metadata( route: Route | None = None, ) -> dict[str, object]: metadata = dict(base_metadata or {}) - project_roots = { - str(getattr(context, "name", "")).strip(): root - for context, root in ( - (context, normalize_project_root(getattr(context, "root", None))) for context in project_contexts - ) - if str(getattr(context, "name", "")).strip() and root - } + roots_by_key: dict[str, tuple[str, str]] = {} + existing_roots = metadata.get("project_roots") + if isinstance(existing_roots, Mapping): + for raw_name, raw_root in existing_roots.items(): + name = str(raw_name).strip() + root = normalize_project_root(raw_root) + if name and root: + roots_by_key[name.casefold()] = (name, root) + for context in project_contexts: + name = str(getattr(context, "name", "")).strip() + root = normalize_project_root(getattr(context, "root", None)) + if name and root: + roots_by_key[name.casefold()] = (name, root) + project_roots = {name: root for name, root in (roots_by_key[key] for key in sorted(roots_by_key))} identity_payload = _startup_identity_payload( runtime, runtime_mode=runtime_mode, project_contexts=project_contexts, route=route, ) + identity_payload["projects"] = [{"name": name, "root": root} for name, root in project_roots.items()] + identity_payload.pop("fingerprint", None) + identity_payload["fingerprint"] = hashlib.sha256( + json.dumps(identity_payload, sort_keys=True).encode("utf-8") + ).hexdigest() metadata["project_roots"] = project_roots metadata["startup_identity"] = identity_payload return metadata diff --git a/python/envctl_engine/startup/run_reuse_resolution.py b/python/envctl_engine/startup/run_reuse_resolution.py index cb9beb3c..7eb95778 100644 --- a/python/envctl_engine/startup/run_reuse_resolution.py +++ b/python/envctl_engine/startup/run_reuse_resolution.py @@ -21,7 +21,7 @@ ) from envctl_engine.startup.run_reuse_fresh_start import replace_existing_project_services_for_fresh_start_with_defaults from envctl_engine.startup.run_reuse_resume import StartupRunReuseResumeHandler -from envctl_engine.startup.session import StartupSession +from envctl_engine.startup.session import StartupSession, metadata_with_state_sources from envctl_engine.startup.session_lifecycle import announce_session_identifiers, emit_startup_phase from envctl_engine.startup.service_bootstrap_domain import configured_service_types_for_mode from envctl_engine.startup.startup_progress import report_progress @@ -137,7 +137,10 @@ def _apply_reuse_expand(self, *, decision: RunReuseDecision, candidate_state: An self.session.run_id = None self.session.preserved_services = dict(candidate_state.services) self.session.preserved_requirements = dict(candidate_state.requirements) - self.session.base_metadata = mark_run_reused(candidate_state.metadata, reason="reuse_expand") + self.session.base_metadata = metadata_with_state_sources( + mark_run_reused(candidate_state.metadata, reason="reuse_expand"), + candidate_state, + ) state_project_names = { str(project.get("name", "")).strip().casefold() for project in decision.state_projects @@ -321,7 +324,7 @@ def resolve_startup_run_reuse_with_runtime( runtime: Any, session: StartupSession, *, - terminate_restart_orphan_listeners: Callable[..., None], + terminate_restart_orphan_listeners: Callable[..., set[int]], validate_attach_target_fn: Callable[..., Any], attach_plan_agent_terminal: Callable[..., Any], progress_lock: Any, @@ -362,13 +365,11 @@ def report_progress_fn(route: Route, message: str) -> None: session=session, validate_plan_agent_handoff=validate_plan_agent_handoff, attach_plan_agent_terminal=attach_plan_agent_terminal, - print_headless_plan_session_summary=lambda session, *, attach_target: ( - print_headless_plan_session_summary( - session, - validate_plan_agent_handoff=validate_plan_agent_handoff, - print_fn=print_fn, - attach_target=attach_target, - ) + print_headless_plan_session_summary=lambda session, *, attach_target: print_headless_plan_session_summary( + session, + validate_plan_agent_handoff=validate_plan_agent_handoff, + print_fn=print_fn, + attach_target=attach_target, ), ), print_headless_plan_session_summary=lambda session: print_headless_plan_session_summary( @@ -391,8 +392,6 @@ def report_progress_fn(route: Route, message: str) -> None: session=session, candidate_state=candidate_state, reason=reason, - configured_service_types=set(configured_service_types_for_mode(runtime.config, session.runtime_mode)), - additional_services=tuple(getattr(runtime.config, "additional_services", ()) or ()), announce_session_identifiers=lambda session: announce_session_identifiers( runtime, session, diff --git a/python/envctl_engine/startup/run_reuse_resume.py b/python/envctl_engine/startup/run_reuse_resume.py index bed5d184..248ee463 100644 --- a/python/envctl_engine/startup/run_reuse_resume.py +++ b/python/envctl_engine/startup/run_reuse_resume.py @@ -6,6 +6,10 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_startup_support import mark_run_reused +from envctl_engine.runtime.lifecycle_operation_lease import ( + lifecycle_operation_active, + release_lifecycle_operation, +) from envctl_engine.runtime.runtime_context import resolve_state_repository from envctl_engine.state.runtime_map import build_runtime_map from envctl_engine.startup.finalization import build_planning_dashboard_state @@ -40,21 +44,25 @@ def resume_matching_run(self) -> int | None: self._emit_resume_match_phase() self._emit_resume_match_events() + lifecycle_lease_was_active = lifecycle_operation_active(self.runtime) resume_code = self.runtime._resume(self._resume_route()) if int(resume_code) == 0: return self._resume_success_result() + lifecycle_lease_was_released = lifecycle_lease_was_active and not lifecycle_operation_active(self.runtime) self._handle_resume_failure( resume_code=int(resume_code), previous_run_id=previous_run_id, previous_identifiers_announced=previous_identifiers_announced, + allow_fresh_fallback=not lifecycle_lease_was_released, ) - return None + return int(resume_code) if lifecycle_lease_was_released else None def resume_dashboard_run(self) -> int | None: self.session.run_id = self.candidate_state.run_id self.announce_session_identifiers(self.session) self._persist_dashboard_resume_metadata() self._emit_dashboard_resume_events() + release_lifecycle_operation(self.runtime) if self.headless_plan_output_only(self.session): self.print_headless_plan_session_summary(self.session) @@ -119,6 +127,7 @@ def _resume_route(self) -> Route: **self.route.flags, "_resume_source_command": self.route.command, "_run_reuse_reason": self.decision.decision_kind, + "_state_project_names": [str(name) for name in self._selected_project_names], } if self._attach_plan_agent_after_resume(): resume_flags["batch"] = True @@ -150,6 +159,7 @@ def _handle_resume_failure( resume_code: int, previous_run_id: str | None, previous_identifiers_announced: bool, + allow_fresh_fallback: bool = True, ) -> None: self.session.run_id = previous_run_id self.session.identifiers_announced = previous_identifiers_announced @@ -170,11 +180,12 @@ def _handle_resume_failure( mode=self.runtime_mode, command=self.route.command, ) - self.replace_existing_project_services_for_fresh_start( - self.session, - candidate_state=self.candidate_state, - reason=self.decision.reason, - ) + if allow_fresh_fallback: + self.replace_existing_project_services_for_fresh_start( + self.session, + candidate_state=self.candidate_state, + reason=self.decision.reason, + ) def _persist_dashboard_resume_metadata(self) -> None: self.candidate_state.metadata = build_planning_dashboard_state( diff --git a/python/envctl_engine/startup/selected_context_startup.py b/python/envctl_engine/startup/selected_context_startup.py index 7be9b313..b96fcf00 100644 --- a/python/envctl_engine/startup/selected_context_startup.py +++ b/python/envctl_engine/startup/selected_context_startup.py @@ -7,7 +7,11 @@ from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.engine_runtime_env import effective_dependency_scope from envctl_engine.startup.protocols import ProjectContextLike, StartupRuntime -from envctl_engine.startup.session import ProjectStartupResult, StartupSession +from envctl_engine.startup.session import ( + ProjectStartupResult, + StartupSession, + track_startup_failure, +) from envctl_engine.startup.startup_progress import suppress_progress_output as default_suppress_progress_output @@ -138,8 +142,11 @@ def start_selected_contexts( project_spinner_group=project_spinner_group, use_project_spinner_group=use_project_spinner_group, ) - except RuntimeError: - _fail_single_spinner(runtime=runtime, active_spinner=active_spinner, enabled=use_single_spinner) + except Exception: + try: + _fail_single_spinner(runtime=runtime, active_spinner=active_spinner, enabled=use_single_spinner) + except Exception: # noqa: BLE001 + pass raise _succeed_single_spinner( runtime=runtime, @@ -228,6 +235,7 @@ def _start_contexts_parallel( ) -> None: completed: dict[str, ProjectStartupResult] = {} failures: list[str] = [] + ui_failures: list[Exception] = [] with concurrent.futures.ThreadPoolExecutor(max_workers=parallel_workers) as executor: future_map = { executor.submit( @@ -243,7 +251,50 @@ def _start_contexts_parallel( context = future_map[future] try: result = cast(ProjectStartupResult, future.result()) - completed[context.name] = result + except RuntimeError as exc: + track_startup_failure(session, exc) + try: + degrade = should_degrade_to_plan_agent_handoff(session, str(exc)) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + degrade = False + if degrade: + try: + record_plan_agent_handoff_local_startup_failure( + session, + project_name=context.name, + error=str(exc), + ) + if use_project_spinner_group: + project_spinner_group.mark_success( + context.name, + "AI session running; local startup failed", + ) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + continue + failures.append(str(exc)) + try: + runtime._emit("startup.project.failed", project=context.name, error=str(exc)) + if use_project_spinner_group: + project_spinner_group.mark_failure(context.name, str(exc)) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + continue + except Exception as exc: # noqa: BLE001 + track_startup_failure(session, exc) + failures.append(str(exc)) + try: + runtime._emit("startup.project.failed", project=context.name, error=str(exc)) + if use_project_spinner_group: + project_spinner_group.mark_failure(context.name, str(exc)) + except Exception as callback_exc: # noqa: BLE001 + ui_failures.append(callback_exc) + continue + + completed[context.name] = result + record_project_startup(session, context, result) + try: _record_parallel_success( runtime=runtime, session=session, @@ -257,22 +308,12 @@ def _start_contexts_parallel( use_project_spinner_group=use_project_spinner_group, completed_count=len(completed), ) - except RuntimeError as exc: - if should_degrade_to_plan_agent_handoff(session, str(exc)): - record_plan_agent_handoff_local_startup_failure(session, project_name=context.name, error=str(exc)) - if use_project_spinner_group: - project_spinner_group.mark_success(context.name, "AI session running; local startup failed") - continue - failures.append(str(exc)) - runtime._emit("startup.project.failed", project=context.name, error=str(exc)) - if use_project_spinner_group: - project_spinner_group.mark_failure(context.name, str(exc)) - for context in session.contexts_to_start: - result = completed.get(context.name) - if result is not None: - record_project_startup(session, context, result) + except Exception as exc: # noqa: BLE001 + ui_failures.append(exc) if failures: raise RuntimeError("; ".join(failures)) + if ui_failures: + raise ui_failures[0] def _record_parallel_success( @@ -335,6 +376,7 @@ def _start_contexts_sequential( ), ) except RuntimeError as exc: + track_startup_failure(session, exc) if should_degrade_to_plan_agent_handoff(session, str(exc)): record_plan_agent_handoff_local_startup_failure(session, project_name=context.name, error=str(exc)) if use_project_spinner_group: diff --git a/python/envctl_engine/startup/service_attach_execution.py b/python/envctl_engine/startup/service_attach_execution.py index 1421c745..c0e51fc2 100644 --- a/python/envctl_engine/startup/service_attach_execution.py +++ b/python/envctl_engine/startup/service_attach_execution.py @@ -1,15 +1,17 @@ from __future__ import annotations -from collections.abc import Callable, Mapping -from dataclasses import dataclass -import os +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field from pathlib import Path import time from typing import Any, cast -from envctl_engine.runtime.service_manager import ServiceStartDescriptor +from envctl_engine.runtime.service_manager import ServiceCleanupError, ServiceStartDescriptor +from envctl_engine.runtime.service_truth_diagnostics import service_listener_failure_class +from envctl_engine.shared.reason_codes import ServiceFailureReason from envctl_engine.startup.service_execution_policy import ordered_service_layers from envctl_engine.startup.service_execution_records import PreparedServiceLaunch +from envctl_engine.startup.session import unconfirmed_service_names from envctl_engine.state.models import PortPlan, ServiceRecord @@ -34,6 +36,8 @@ class ServiceAttachRunner: additional_services: tuple[Any, ...] backend_listener_expected: bool rebound_delta: int + _active_service_ports: dict[str, int] = field(default_factory=dict, init=False, repr=False) + _rebound_launch_ports: dict[str, int] = field(default_factory=dict, init=False, repr=False) def start( self, @@ -42,24 +46,94 @@ def start( on_service_retry: Callable[[str, int, int, int, str | None], None], ) -> dict[str, ServiceRecord]: descriptors = self._service_descriptors() - generic_attach = getattr(self.runtime.services, "start_services_with_attach", None) - if callable(generic_attach): - return self._start_with_generic_attach( - descriptors=descriptors, - attach_parallel=attach_parallel, - on_service_retry=on_service_retry, - generic_attach=cast(Callable[..., dict[str, ServiceRecord]], generic_attach), - ) - if self.selected_service_types <= {"backend", "frontend"}: - return self._start_core_services_with_legacy_attach( - attach_parallel=attach_parallel, - on_service_retry=on_service_retry, - ) - raise RuntimeError("Service manager does not support additional app services") + self._active_service_ports = { + descriptor.service_type: descriptor.requested_port + for descriptor in descriptors + if descriptor.requested_port > 0 + } + + def release_failed_port_then_report( + service_type: str, + failed_port: int, + retry_port: int, + attempt: int, + error: str | None, + ) -> None: + self._release_failed_service_port(service_type, failed_port) + self._release_rebound_launch_port(service_type) + if retry_port > 0: + self._active_service_ports[service_type] = retry_port + on_service_retry(service_type, failed_port, retry_port, attempt, error) + + try: + if hasattr(self.runtime.services, "process_runner"): + self.runtime.services.process_runner = self.process_runtime + generic_attach = getattr(self.runtime.services, "start_services_with_attach", None) + if callable(generic_attach): + records = self._start_with_generic_attach( + descriptors=descriptors, + attach_parallel=attach_parallel, + on_service_retry=release_failed_port_then_report, + generic_attach=cast(Callable[..., dict[str, ServiceRecord]], generic_attach), + ) + elif self.selected_service_types <= {"backend", "frontend"}: + records = self._start_core_services_with_legacy_attach( + attach_parallel=attach_parallel, + on_service_retry=release_failed_port_then_report, + ) + else: + raise RuntimeError("Service manager does not support additional app services") + self._record_port_lock_session(records.values()) + return records + except Exception as exc: + self._record_port_lock_session(dict(getattr(exc, "unterminated_services", {}) or {}).values()) + self._release_failed_start_reservations(exc) + raise + + def _record_port_lock_session(self, services: Iterable[object]) -> None: + raw_session_id = getattr(self.port_allocator, "session_id", None) + session_id = raw_session_id.strip() if isinstance(raw_session_id, str) else "" + if not session_id: + return + for service in services: + if isinstance(service, ServiceRecord): + service.port_lock_session = session_id def reserve_next(self, port: int) -> int: return self.port_allocator.reserve_next(port, owner=f"{self.project_name}:services") + def _release_failed_service_port(self, service_type: str, port: int) -> None: + release = getattr(self.port_allocator, "release", None) + if not callable(release): + return + for owner in ( + f"{self.project_name}:{service_type}", + f"{self.project_name}:services", + f"{self.project_name}:services:{service_type}-launch", + ): + release(port, owner=owner) + + def _release_rebound_launch_port(self, service_type: str) -> None: + port = self._rebound_launch_ports.pop(service_type, None) + if port is None: + return + release = getattr(self.port_allocator, "release", None) + if callable(release): + release(port, owner=f"{self.project_name}:services:{service_type}-launch") + + def _release_failed_start_reservations(self, error: BaseException) -> None: + preserved_service_types = { + str(getattr(service, "type", "") or "").strip().lower() + for service in dict(getattr(error, "unterminated_services", {}) or {}).values() + if _service_has_recorded_pid(service) + } + for service_type, port in tuple(self._active_service_ports.items()): + if service_type in preserved_service_types: + continue + self._release_failed_service_port(service_type, port) + self._active_service_ports.pop(service_type, None) + self._release_rebound_launch_port(service_type) + def _start_with_generic_attach( self, *, @@ -70,18 +144,42 @@ def _start_with_generic_attach( ) -> dict[str, ServiceRecord]: descriptor_by_type = {descriptor.service_type: descriptor for descriptor in descriptors} records: dict[str, ServiceRecord] = {} - for layer in ordered_service_layers( - tuple(descriptor.service_type for descriptor in descriptors), - self.additional_services, - ): - layer_records = generic_attach( - project=self.project_name, - descriptors=tuple(descriptor_by_type[name] for name in layer), - reserve_next=self.reserve_next, - on_retry=on_service_retry, - parallel_start=attach_parallel and len(layer) > 1, - ) - records.update(layer_records) + try: + for layer in ordered_service_layers( + tuple(descriptor.service_type for descriptor in descriptors), + self.additional_services, + ): + layer_records = generic_attach( + project=self.project_name, + descriptors=tuple(descriptor_by_type[name] for name in layer), + reserve_next=self.reserve_next, + on_retry=on_service_retry, + parallel_start=attach_parallel and len(layer) > 1, + ) + records.update(layer_records) + except Exception as exc: + unterminated = dict(getattr(exc, "unterminated_services", {}) or {}) + terminator = getattr(self.runtime, "_terminate_started_services", None) + if records: + if callable(terminator): + try: + failed_names = unconfirmed_service_names( + terminator(records), + records, + ) + except Exception as cleanup_exc: # noqa: BLE001 + failed_names = set(records) + exc = RuntimeError(f"{exc}; cleanup error: {cleanup_exc}") + else: + failed_names = set(records) + unterminated.update((name, service) for name, service in records.items() if name in failed_names) + if unterminated: + names = ", ".join(sorted(unterminated)) + raise ServiceCleanupError( + f"{exc}; cleanup could not confirm exit for: {names}", + unterminated, + ) from exc + raise return records def _start_core_services_with_legacy_attach( @@ -140,9 +238,7 @@ def _service_descriptors(self) -> list[ServiceStartDescriptor]: service_type=service.name, cwd=str(launch.cwd), requested_port=launch.requested_port, - start=lambda port, service_name=service.name: self.start_additional_service( - service_name, port - ), + start=lambda port, service_name=service.name: self.start_additional_service(service_name, port), detect_actual=lambda pid, requested, service_name=service.name: self.detect_additional_actual( service_name, pid, requested ), @@ -197,24 +293,39 @@ def _launch_prepared_process( launch_port: int, event_port: int, env_extra: dict[str, str], - pid_fallback: int, ) -> tuple[bool, str | None, int | None]: launch_started = time.monotonic() - process = self._start_process( - command, - cwd=str(launch.cwd), - env=self.command_env_builder(port=launch_port, extra=env_extra), - log_path=launch.log_path, - ) - self._emit_attach_phase(service_name, "process_launch", launch_started) - self.runtime._emit( - "service.start", - project=self.project_name, - service=service_name, - port=event_port, - retry=False, - ) - return True, None, getattr(process, "pid", pid_fallback) + try: + process = self._start_process( + command, + cwd=str(launch.cwd), + env=self.command_env_builder(port=launch_port, extra=env_extra), + log_path=launch.log_path, + ) + except OSError as exc: + detail = f"process spawn failed for {command[0]}: {exc}" + self.runtime._emit( + "service.failure", + project=self.project_name, + service=service_name, + failure_class=ServiceFailureReason.PROCESS_SPAWN_FAILED.value, + requested_port=event_port, + detail=detail, + ) + return False, detail, None + pid = getattr(process, "pid", None) + try: + self._emit_attach_phase(service_name, "process_launch", launch_started) + self.runtime._emit( + "service.start", + project=self.project_name, + service=service_name, + port=event_port, + retry=False, + ) + except Exception: # noqa: BLE001 + pass + return True, None, pid if isinstance(pid, int) and pid > 0 else None def start_backend(self, port: int) -> tuple[bool, str | None, int | None]: conflict_result = self._conflicting_start_result("backend", port) @@ -234,7 +345,6 @@ def start_backend(self, port: int) -> tuple[bool, str | None, int | None]: launch_port=port, event_port=port, env_extra=self.backend_env_extra, - pid_fallback=os.getpid(), ) def start_frontend(self, port: int) -> tuple[bool, str | None, int | None]: @@ -247,22 +357,29 @@ def start_frontend(self, port: int) -> tuple[bool, str | None, int | None]: launch_port, owner=f"{self.project_name}:services:frontend-launch", ) - command_resolve_started = time.monotonic() - command, _resolved_source = self.runtime._service_start_command_resolved( - service_name="frontend", - project_root=self.project_root, - port=launch_port, - ) - self._emit_attach_phase("frontend", "command_resolution", command_resolve_started) - return self._launch_prepared_process( - service_name="frontend", - command=command, - launch=self.prepared_launches["frontend"], - launch_port=launch_port, - event_port=port, - env_extra=self.frontend_env_extra, - pid_fallback=os.getpid() + 1, - ) + self._rebound_launch_ports["frontend"] = launch_port + try: + command_resolve_started = time.monotonic() + command, _resolved_source = self.runtime._service_start_command_resolved( + service_name="frontend", + project_root=self.project_root, + port=launch_port, + ) + self._emit_attach_phase("frontend", "command_resolution", command_resolve_started) + result = self._launch_prepared_process( + service_name="frontend", + command=command, + launch=self.prepared_launches["frontend"], + launch_port=launch_port, + event_port=port, + env_extra=self.frontend_env_extra, + ) + except Exception: + self._release_rebound_launch_port("frontend") + raise + if not isinstance(result[2], int) or isinstance(result[2], bool) or result[2] <= 0: + self._release_rebound_launch_port("frontend") + return result def start_additional_service(self, service_name: str, port: int) -> tuple[bool, str | None, int | None]: conflict_result = self._conflicting_start_result(service_name, port) @@ -291,7 +408,6 @@ def start_additional_service(self, service_name: str, port: int) -> tuple[bool, launch_port=port, event_port=port, env_extra=launch.env, - pid_fallback=os.getpid(), ) def detect_backend_actual(self, pid: int | None, requested: int) -> int | None: @@ -431,17 +547,22 @@ def _resolve_detected_actual( if self.runtime._listener_truth_enforced(): detail = self.runtime._service_listener_failure_detail(log_path=log_path, pid=pid) error_suffix = f" ({detail})" if detail else "" + failure_class = service_listener_failure_class(detail) self.runtime._emit( "service.failure", project=self.project_name, service=service_name, - failure_class="listener_not_detected", + failure_class=failure_class, requested_port=requested, detail=detail, ) - raise RuntimeError( - f"{service_name} listener not detected for {self.project_name} on port {failure_port}{error_suffix}" - ) + if failure_class == ServiceFailureReason.DEPENDENCY_MISSING.value: + failure_summary = f"{service_name} is missing a required executable or module" + elif failure_class == ServiceFailureReason.PROCESS_EXITED.value: + failure_summary = f"{service_name} process exited before opening its listener" + else: + failure_summary = f"{service_name} listener not detected" + raise RuntimeError(f"{failure_summary} for {self.project_name} on port {failure_port}{error_suffix}") self.runtime._emit( "service.failure", project=self.project_name, @@ -466,3 +587,10 @@ def _emit_attach_phase(self, service_name: str, phase: str, started: float) -> N phase=phase, duration_ms=round((time.monotonic() - started) * 1000.0, 2), ) + + +def _service_has_recorded_pid(service: object) -> bool: + listener_pids = getattr(service, "listener_pids", None) + listeners = listener_pids if isinstance(listener_pids, (list, tuple, set, frozenset)) else () + candidates = [getattr(service, "pid", None), *listeners] + return any(isinstance(pid, int) and not isinstance(pid, bool) and pid > 0 for pid in candidates) diff --git a/python/envctl_engine/startup/session.py b/python/envctl_engine/startup/session.py index 9cd91a69..84d0367c 100644 --- a/python/envctl_engine/startup/session.py +++ b/python/envctl_engine/startup/session.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Collection, Mapping import time from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, cast @@ -54,11 +55,13 @@ class StartupSession: restart_state: RunState | None = None requirements_by_project: dict[str, RequirementsResult] = field(default_factory=dict) services_by_project: dict[str, dict[str, ServiceRecord]] = field(default_factory=dict) + unterminated_services: dict[str, ServiceRecord] = field(default_factory=dict) started_context_names: list[str] = field(default_factory=list) dashboard_after_start: bool = False disabled_startup_mode: bool = False used_project_spinner_group: bool = False strict_truth_failed: bool = False + preserve_existing_state_on_failure: bool = False failure_message: str | None = None errors: list[str] = field(default_factory=list) warnings: list[str] = field(default_factory=list) @@ -82,6 +85,7 @@ def merged_services(self) -> dict[str, ServiceRecord]: services = dict(self.preserved_services) for project_services in self.services_by_project.values(): services.update(project_services) + services.update(self.unterminated_services) return services @property @@ -111,3 +115,57 @@ def plan_agent_launch_degraded(self) -> bool: if launch_result is None: return False return str(getattr(launch_result, "status", "")).strip().lower() == "partial" + + +def unconfirmed_service_names( + termination_result: object, + services: Mapping[str, object], +) -> set[str]: + """Return services whose exit was not positively confirmed. + + The termination contract returns a collection of names that remain alive; + an explicit empty collection therefore confirms complete cleanup. Legacy + adapters and failed mocks may return ``None`` or another scalar. Treating + those values as success loses process and port ownership, so they retain + every service for a later verified cleanup pass. + """ + + if not isinstance(termination_result, Collection) or isinstance( + termination_result, + (str, bytes, Mapping), + ): + return set(services) + return {name for raw_name in termination_result if (name := str(raw_name).strip()) and name in services} + + +def metadata_with_state_sources(metadata: dict[str, Any], state: RunState) -> dict[str, Any]: + """Return metadata that records every state replaced by a new startup run.""" + + updated = dict(metadata) + source_run_ids = {state.run_id} + for source in ( + updated.get("state_source_run_ids"), + state.metadata.get("state_source_run_ids"), + ): + if not isinstance(source, list): + continue + source_run_ids.update(str(run_id).strip() for run_id in source if str(run_id).strip()) + updated["state_source_run_ids"] = sorted(source_run_ids) + return updated + + +def track_startup_failure(session: StartupSession, failure: BaseException) -> None: + project = str(getattr(failure, "project", "") or "").strip() + requirements = getattr(failure, "requirements", None) + if project and isinstance(requirements, RequirementsResult): + session.requirements_by_project[project] = requirements + services = getattr(failure, "unterminated_services", None) + if not isinstance(services, dict): + return + for name, service in services.items(): + if isinstance(name, str) and isinstance(service, ServiceRecord): + session.unterminated_services[name] = service + + +def track_unterminated_services(session: StartupSession, failure: BaseException) -> None: + track_startup_failure(session, failure) diff --git a/python/envctl_engine/startup/startup_execution_support.py b/python/envctl_engine/startup/startup_execution_support.py index 631fb4a7..51989c56 100644 --- a/python/envctl_engine/startup/startup_execution_support.py +++ b/python/envctl_engine/startup/startup_execution_support.py @@ -29,8 +29,8 @@ startup_breakdown_enabled, ) from envctl_engine.startup.service_execution import start_project_services as start_project_services_impl -from envctl_engine.startup.session import ProjectStartupResult -from envctl_engine.state.models import PortPlan, RequirementsResult +from envctl_engine.startup.session import ProjectStartupResult, unconfirmed_service_names +from envctl_engine.state.models import PortPlan, RequirementsResult, ServiceRecord @dataclass(slots=True) @@ -40,6 +40,23 @@ class _SharedProjectContext: ports: Mapping[str, PortPlan] +class ProjectStartupFailure(RuntimeError): + """A project failure that preserves already-started runtime resources.""" + + def __init__( + self, + message: str, + *, + project: str, + requirements: RequirementsResult, + unterminated_services: dict[str, ServiceRecord] | None = None, + ) -> None: + super().__init__(message) + self.project = project + self.requirements = requirements + self.unterminated_services = dict(unterminated_services or {}) + + def start_project_context( orchestrator: StartupOrchestratorLike, *, @@ -55,50 +72,74 @@ def start_project_context( report_progress_fn(route, f"Starting project {context.name}...", project=context.name) _reserve_project_ports(rt, context, route=route) requirements = _requirements_for_project_context(orchestrator, context=context, mode=mode, route=route) - if not rt._requirements_ready(requirements): - raise RuntimeError(_requirements_failure_message(context.name, requirements)) - _sync_supabase_auth_users_before_services( - orchestrator, - context=context, - mode=mode, - route=route, - requirements=requirements, - ) - requirements_progress_message = _requirements_ready_progress_message( - orchestrator, - context=context, - mode=mode, - route=route, - requirements=requirements, - ) - if requirements_progress_message is not None: - report_progress_fn(route, requirements_progress_message, project=context.name) - project_services = orchestrator.start_project_services( - context, - requirements=requirements, - run_id=run_id, - route=route, - ) + project_services: dict[str, ServiceRecord] = {} try: + if not rt._requirements_ready(requirements): + raise RuntimeError(_requirements_failure_message(context.name, requirements)) + _sync_supabase_auth_users_before_services( + orchestrator, + context=context, + mode=mode, + route=route, + requirements=requirements, + ) + requirements_progress_message = _requirements_ready_progress_message( + orchestrator, + context=context, + mode=mode, + route=route, + requirements=requirements, + ) + if requirements_progress_message is not None: + report_progress_fn(route, requirements_progress_message, project=context.name) + project_services = orchestrator.start_project_services( + context, + requirements=requirements, + run_id=run_id, + route=route, + ) rt._assert_project_services_post_start_truth(context=context, services=project_services) - except RuntimeError: - rt._terminate_started_services(project_services) - raise - report_progress_fn( - route, - f"Services ready for {context.name}: " - + " ".join( - f"{service_type}={_service_ready_label(project_services.get(f'{context.name} {service_type.title()}'))}" - for service_type in ("backend", "frontend") - if f"{context.name} {service_type.title()}" in project_services - ), - project=context.name, - ) - return ProjectStartupResult( - requirements=requirements, - services=project_services, - warnings=list(rt._consume_project_startup_warnings(context.name)), - ) + report_progress_fn( + route, + f"Services ready for {context.name}: " + + " ".join( + f"{service_type}={_service_ready_label(project_services.get(f'{context.name} {service_type.title()}'))}" + for service_type in ("backend", "frontend") + if f"{context.name} {service_type.title()}" in project_services + ), + project=context.name, + ) + return ProjectStartupResult( + requirements=requirements, + services=project_services, + warnings=list(rt._consume_project_startup_warnings(context.name)), + ) + except Exception as exc: + unterminated = dict(getattr(exc, "unterminated_services", {}) or {}) + cleanup_error: Exception | None = None + if project_services: + try: + failed_services = unconfirmed_service_names( + rt._terminate_started_services(project_services), + project_services, + ) + except Exception as cleanup_exc: # noqa: BLE001 + cleanup_error = cleanup_exc + failed_services = set(project_services) + unterminated.update( + (name, service) for name, service in project_services.items() if name in failed_services + ) + message = str(exc) + if unterminated and "cleanup could not confirm exit for:" not in message: + message = f"{message}; cleanup could not confirm exit for: {', '.join(sorted(unterminated))}" + if cleanup_error is not None: + message = f"{message}; cleanup error: {cleanup_error}" + raise ProjectStartupFailure( + message, + project=context.name, + requirements=requirements, + unterminated_services=unterminated, + ) from exc def _reserve_project_ports(rt: Any, context: ProjectContextLike, *, route: Route | None) -> None: @@ -335,10 +376,13 @@ def _annotate_shared_main_requirements( ) supabase = cloned.component("supabase") if bool(supabase.get("enabled", False)): - supabase["container_name"] = build_supabase_project_name( - project_root=project_root, - project_name="Main", - ) + "-supabase-db-1" + supabase["container_name"] = ( + build_supabase_project_name( + project_root=project_root, + project_name="Main", + ) + + "-supabase-db-1" + ) return _reconcile_shared_main_requirements(orchestrator, cloned) diff --git a/python/envctl_engine/startup/startup_selection_support.py b/python/envctl_engine/startup/startup_selection_support.py index 2235e19a..d08ace79 100644 --- a/python/envctl_engine/startup/startup_selection_support.py +++ b/python/envctl_engine/startup/startup_selection_support.py @@ -97,13 +97,19 @@ def trees_start_selection_required(*, route: Route, runtime_mode: str) -> bool: def tree_preselected_projects_from_state( *, runtime: StartupRuntime, project_contexts: list[ProjectContextLike] ) -> list[str]: - state = runtime._try_load_existing_state(mode="trees", strict_mode_match=True) + repository = getattr(runtime, "state_repository", None) + load_all = getattr(repository, "load_all", None) + states = [state for state in load_all(mode="trees") if isinstance(state, RunState)] if callable(load_all) else [] + if not states: + state = runtime._try_load_existing_state(mode="trees", strict_mode_match=True) + states = [state] if state is not None else [] available = {str(context.name).strip().lower(): str(context.name).strip() for context in project_contexts} - if state is None: + if not states: return _tree_preselected_projects_from_plans(runtime=runtime, project_contexts=project_contexts) preselected: list[str] = [] seen: set[str] = set() - for name in sorted(state_project_names(runtime=runtime, state=state)): + active_names = {name for state in states for name in state_project_names(runtime=runtime, state=state)} + for name in sorted(active_names): normalized = str(name).strip().lower() resolved = available.get(normalized) if not resolved: @@ -138,9 +144,7 @@ def _tree_preselected_projects_from_plans( ] existing_counts = planning_existing_counts(projects=projects, planning_files=planning_files) plan_counts = OrderedDict( - (plan_file, count) - for plan_file in planning_files - if (count := int(existing_counts.get(plan_file, 0))) > 0 + (plan_file, count) for plan_file in planning_files if (count := int(existing_counts.get(plan_file, 0))) > 0 ) if not plan_counts: return [] @@ -354,7 +358,6 @@ def _project_name_from_service_name(name: str) -> str | None: return None - def _service_filter_types_for_project( *, route: Route, project_name: str, configured_service_types: set[str] ) -> set[str]: @@ -470,7 +473,10 @@ def _restart_service_types_for_project( ) return _apply_startup_service_launch_flags(configured_service_types, route=route) + def _apply_startup_service_launch_flags(service_types: set[str], *, route: Route) -> set[str]: + if route.flags.get("launch_backend") is False and route.flags.get("launch_frontend") is False: + return set() selected = set(service_types) if route.flags.get("launch_backend") is False: selected.discard("backend") diff --git a/python/envctl_engine/state/__init__.py b/python/envctl_engine/state/__init__.py index 37b34ba2..5a70281c 100644 --- a/python/envctl_engine/state/__init__.py +++ b/python/envctl_engine/state/__init__.py @@ -30,6 +30,11 @@ def load_state(path: str, *, allowed_root: str | None = None) -> RunState: pid=svc.get("pid"), requested_port=svc.get("requested_port"), actual_port=svc.get("actual_port"), + port_lock_session=( + svc["port_lock_session"].strip() + if isinstance(svc.get("port_lock_session"), str) and svc["port_lock_session"].strip() + else None + ), log_path=svc.get("log_path"), status=svc.get("status", "unknown"), synthetic=bool(svc.get("synthetic", False)), @@ -169,6 +174,7 @@ def state_to_dict(state: RunState) -> dict[str, object]: "pid": svc.pid, "requested_port": svc.requested_port, "actual_port": svc.actual_port, + "port_lock_session": svc.port_lock_session, "log_path": svc.log_path, "status": svc.status, "synthetic": svc.synthetic, diff --git a/python/envctl_engine/state/fingerprints.py b/python/envctl_engine/state/fingerprints.py new file mode 100644 index 00000000..30087f57 --- /dev/null +++ b/python/envctl_engine/state/fingerprints.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +from envctl_engine.state import state_to_dict +from envctl_engine.state.models import RunState + + +def text_fingerprint(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def file_fingerprint(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def state_fingerprint(state: RunState) -> str: + return text_fingerprint(json.dumps(state_to_dict(state), sort_keys=True)) diff --git a/python/envctl_engine/state/lookup.py b/python/envctl_engine/state/lookup.py new file mode 100644 index 00000000..ce5580bb --- /dev/null +++ b/python/envctl_engine/state/lookup.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import inspect +from collections.abc import Callable, Sequence +from typing import Any + + +def call_state_loader( + loader: Callable[..., Any], + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, +) -> Any: + """Call current and legacy state-loader adapters without injecting unsupported keywords.""" + + accepts_any_keyword, accepted_keywords = _accepted_keywords(loader) + options: dict[str, object] = {} + for keyword, value in ( + ("mode", mode), + ("strict_mode_match", strict_mode_match), + ): + if accepts_any_keyword or keyword in accepted_keywords: + options[keyword] = value + if project_names is not None: + if not accepts_any_keyword and "project_names" not in accepted_keywords: + raise TypeError("Targeted state lookup requires a loader that accepts project_names") + options["project_names"] = project_names + return loader(**options) + + +def _accepted_keywords(loader: Callable[..., Any]) -> tuple[bool, frozenset[str]]: + try: + parameters = tuple(inspect.signature(loader).parameters.values()) + except (TypeError, ValueError): + return True, frozenset() + return ( + any(parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters), + frozenset( + parameter.name + for parameter in parameters + if parameter.kind in {inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY} + ), + ) + + +__all__ = ["call_state_loader"] diff --git a/python/envctl_engine/state/models.py b/python/envctl_engine/state/models.py index cfcab440..68bb3c4f 100644 --- a/python/envctl_engine/state/models.py +++ b/python/envctl_engine/state/models.py @@ -24,6 +24,7 @@ class ServiceRecord: pid: int | None = None requested_port: int | None = None actual_port: int | None = None + port_lock_session: str | None = None log_path: str | None = None status: str = "unknown" synthetic: bool = False diff --git a/python/envctl_engine/state/persistence.py b/python/envctl_engine/state/persistence.py new file mode 100644 index 00000000..6d8e29b6 --- /dev/null +++ b/python/envctl_engine/state/persistence.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import fcntl +import errno +import os +import re +import stat +import tempfile +import time +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + + +_ATOMIC_TEMP_TOKEN = re.compile(r"[a-z0-9_]{8}") + + +def atomic_write_text(path: Path, text: str) -> None: + """Replace a UTF-8 text file without exposing a partially written value.""" + + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + prefix=f".{path.name}.", + suffix=".tmp", + dir=path.parent, + delete=False, + ) as temporary_file: + temporary_path = Path(temporary_file.name) + temporary_file.write(text) + temporary_file.flush() + os.fsync(temporary_file.fileno()) + os.replace(temporary_path, path) + fsync_directory(path.parent) + except BaseException: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + raise + + +def scavenge_atomic_write_temps(path: Path) -> int: + """Remove crash leftovers for one exact atomic-write target. + + The caller must already hold the exclusive lock that serializes writes to + ``path``. Scanning is deliberately target-local: retired runtime history + can be unbounded, and unrelated ``*.tmp`` files must never be touched. + Directory and entry symlinks are not followed. + """ + + parent = path.parent + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + directory_fd = os.open(parent, flags) + removed = 0 + expected_prefix = f".{path.name}." + try: + if not stat.S_ISDIR(os.fstat(directory_fd).st_mode): + raise OSError(f"Atomic-write parent is not a real directory: {parent}") + with os.scandir(directory_fd) as entries: + for entry in entries: + name = entry.name + if not name.startswith(expected_prefix) or not name.endswith(".tmp"): + continue + token = name[len(expected_prefix) : -len(".tmp")] + if _ATOMIC_TEMP_TOKEN.fullmatch(token) is None: + continue + try: + metadata = entry.stat(follow_symlinks=False) + except FileNotFoundError: + continue + if not stat.S_ISREG(metadata.st_mode): + continue + try: + os.unlink(name, dir_fd=directory_fd) + except FileNotFoundError: + continue + removed += 1 + if removed: + fsync_directory(parent) + return removed + finally: + os.close(directory_fd) + + +def durable_mkdir(path: Path) -> None: + """Create a directory chain and durably publish every new directory entry.""" + + missing: list[Path] = [] + cursor = path + while not cursor.exists(): + missing.append(cursor) + if cursor.parent == cursor: + break + cursor = cursor.parent + if cursor.is_symlink() or not cursor.is_dir(): + raise OSError(f"Directory ancestor is not a real directory: {cursor}") + for directory in reversed(missing): + try: + directory.mkdir() + except FileExistsError: + if directory.is_symlink() or not directory.is_dir(): + raise + if directory.is_symlink() or not directory.is_dir(): + raise OSError(f"Created path is not a real directory: {directory}") + fsync_directory(directory) + fsync_directory(directory.parent) + + +def require_path_component(value: object, *, label: str) -> str: + """Return a single portable path component or reject ambiguous traversal input.""" + + raw = str(value) + if ( + not raw + or raw != raw.strip() + or raw in {".", ".."} + or "/" in raw + or "\\" in raw + or "\0" in raw + or Path(raw).is_absolute() + ): + raise ValueError(f"{label} must be a non-empty path component") + return raw + + +def fsync_directory(path: Path) -> None: + """Durably publish directory entry changes without following a replaced symlink.""" + + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + directory_fd = os.open(path, flags) + try: + if not stat.S_ISDIR(os.fstat(directory_fd).st_mode): + raise OSError(f"Path is not a real directory: {path}") + try: + os.fsync(directory_fd) + except OSError as exc: + if exc.errno not in {errno.EBADF, errno.EINVAL, errno.ENOTSUP, errno.EROFS}: + raise + finally: + os.close(directory_fd) + + +@contextmanager +def advisory_file_lock( + path: Path, + *, + exclusive: bool, + timeout_seconds: float | None = None, +) -> Iterator[None]: + """Hold a process-safe advisory lock backed by a stable lock-file inode.""" + + path.parent.mkdir(parents=True, exist_ok=True) + flags = os.O_CREAT | os.O_RDWR + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags, 0o600) + try: + lock_file = os.fdopen(descriptor, "a+b") + except BaseException: + os.close(descriptor) + raise + with lock_file: + if not stat.S_ISREG(os.fstat(lock_file.fileno()).st_mode): + raise OSError(f"Lock path is not a regular file: {path}") + operation = fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH + if timeout_seconds is None: + fcntl.flock(lock_file.fileno(), operation) + else: + deadline = time.monotonic() + max(0.0, timeout_seconds) + while True: + try: + fcntl.flock(lock_file.fileno(), operation | fcntl.LOCK_NB) + break + except BlockingIOError as exc: + if time.monotonic() >= deadline: + raise TimeoutError(f"Timed out acquiring lock: {path}") from exc + time.sleep(0.01) + try: + yield + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + +__all__ = [ + "advisory_file_lock", + "atomic_write_text", + "durable_mkdir", + "fsync_directory", + "require_path_component", + "scavenge_atomic_write_temps", +] diff --git a/python/envctl_engine/state/project_runtime.py b/python/envctl_engine/state/project_runtime.py index c3a69c4f..aa68f87d 100644 --- a/python/envctl_engine/state/project_runtime.py +++ b/python/envctl_engine/state/project_runtime.py @@ -1,13 +1,34 @@ from __future__ import annotations +import json from dataclasses import dataclass from pathlib import Path from typing import Any, Mapping +from envctl_engine.dashboard_metadata import ( + DASHBOARD_CONFIGURED_SERVICE_TYPES_KEY, + DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY, + DASHBOARD_STOPPED_SERVICES_KEY, + normalize_dashboard_service_types, +) from envctl_engine.shared.services import service_project_name +from envctl_engine.state.fingerprints import text_fingerprint from envctl_engine.state.models import RequirementsResult, RunState +PROJECT_SCOPED_METADATA_MAP_KEYS = ( + "project_roots", + "project_pr_links", + "project_test_summaries", + "project_action_reports", + DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY, +) +_PROJECT_TEST_METADATA_COMPANION_KEYS = ( + "project_test_results_root", + "project_test_results_updated_at", +) + + @dataclass(slots=True) class ProjectRuntimeResolution: ok: bool @@ -140,6 +161,7 @@ def resolve_requested_project_state( state=filter_state_to_projects(state, selected, runtime=runtime), ) + def _service_project_name(service_name: str, service: object, *, runtime: Any | None = None) -> str: explicit_project = str(getattr(service, "project", "") or "").strip() if explicit_project: @@ -171,10 +193,8 @@ def filter_state_to_projects( if _service_project_name(name, service, runtime=runtime) in selected } requirements = _filtered_requirements(state.requirements, selected) - metadata = dict(state.metadata) - roots = metadata.get("project_roots") - if isinstance(roots, Mapping): - metadata["project_roots"] = {str(name): value for name, value in roots.items() if str(name) in selected} + metadata = filter_project_scoped_metadata(state.metadata, selected) + metadata["project_names"] = sorted(selected, key=lambda value: (value.casefold(), value)) return RunState( run_id=state.run_id, mode=state.mode, @@ -187,6 +207,104 @@ def filter_state_to_projects( ) +def filter_project_scoped_metadata( + metadata: Mapping[str, object], + projects: list[str] | tuple[str, ...] | set[str] | frozenset[str], + *, + case_sensitive: bool = True, +) -> dict[str, object]: + """Return metadata containing only records owned by the selected projects. + + Runtime state has several independently evolved project-keyed metadata + surfaces. Keeping their filtering in one place prevents a selected state + from retaining dashboard, PR, test, or action data for another run. + """ + + selected = {str(project).strip() for project in projects if str(project).strip()} + normalized_selected = selected if case_sensitive else {project.casefold() for project in selected} + + def selected_project(project: object) -> bool: + value = str(project).strip() + if not value: + return False + return value in normalized_selected if case_sensitive else value.casefold() in normalized_selected + + filtered = dict(metadata) + for key in PROJECT_SCOPED_METADATA_MAP_KEYS: + raw = metadata.get(key) + if not isinstance(raw, Mapping): + if key in metadata: + filtered.pop(key, None) + continue + retained = {str(project): value for project, value in raw.items() if selected_project(project)} + if retained: + filtered[key] = retained + else: + filtered.pop(key, None) + + raw_stopped = metadata.get(DASHBOARD_STOPPED_SERVICES_KEY) + if isinstance(raw_stopped, list): + stopped: list[dict[str, object]] = [] + for item in raw_stopped: + if not isinstance(item, Mapping) or not selected_project(item.get("project", "")): + continue + if not str(item.get("name", "")).strip() or not str(item.get("type", "")).strip(): + continue + stopped.append(dict(item)) + if stopped: + filtered[DASHBOARD_STOPPED_SERVICES_KEY] = stopped + else: + filtered.pop(DASHBOARD_STOPPED_SERVICES_KEY, None) + elif DASHBOARD_STOPPED_SERVICES_KEY in metadata: + filtered.pop(DASHBOARD_STOPPED_SERVICES_KEY, None) + + raw_summaries = metadata.get("project_test_summaries") + retained_summaries = filtered.get("project_test_summaries") + summaries_were_narrowed = ( + isinstance(raw_summaries, Mapping) + and isinstance(retained_summaries, Mapping) + and len(retained_summaries) != len(raw_summaries) + ) + if not isinstance(retained_summaries, Mapping) or not retained_summaries or summaries_were_narrowed: + for key in _PROJECT_TEST_METADATA_COMPANION_KEYS: + filtered.pop(key, None) + + if DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY in metadata or DASHBOARD_STOPPED_SERVICES_KEY in metadata: + configured_types: set[str] = set() + configured = filtered.get(DASHBOARD_PROJECT_CONFIGURED_SERVICES_KEY) + if isinstance(configured, Mapping): + for service_types in configured.values(): + configured_types.update(normalize_dashboard_service_types(service_types)) + stopped = filtered.get(DASHBOARD_STOPPED_SERVICES_KEY) + if isinstance(stopped, list): + configured_types.update( + normalize_dashboard_service_types( + [item.get("type", "") for item in stopped if isinstance(item, Mapping)] + ) + ) + if configured_types: + filtered[DASHBOARD_CONFIGURED_SERVICE_TYPES_KEY] = sorted(configured_types) + else: + filtered.pop(DASHBOARD_CONFIGURED_SERVICE_TYPES_KEY, None) + + raw_identity = metadata.get("startup_identity") + if isinstance(raw_identity, Mapping): + identity = dict(raw_identity) + raw_projects = identity.get("projects") + identity_projects = raw_projects if isinstance(raw_projects, list) else [] + identity["projects"] = [ + dict(project) + for project in identity_projects + if isinstance(project, Mapping) and selected_project(project.get("name", "")) + ] + identity.pop("fingerprint", None) + serialized = json.dumps(identity, sort_keys=True) + identity["fingerprint"] = text_fingerprint(serialized) + filtered["startup_identity"] = identity + + return filtered + + def _filtered_requirements( requirements: Mapping[str, RequirementsResult], selected: set[str], @@ -281,7 +399,6 @@ def cwd_runtime_warnings( ] - def project_resolution_event_payload( resolution: ProjectRuntimeResolution, state: RunState, diff --git a/python/envctl_engine/state/repository.py b/python/envctl_engine/state/repository.py index b750117b..408e863a 100644 --- a/python/envctl_engine/state/repository.py +++ b/python/envctl_engine/state/repository.py @@ -1,17 +1,39 @@ from __future__ import annotations import json -import hashlib import shutil -from collections.abc import Mapping +import stat +import uuid +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Callable, cast +from envctl_engine.dashboard_metadata import DASHBOARD_STOPPED_SERVICES_KEY from envctl_engine.shared.artifact_names import safe_artifact_stem -from envctl_engine.state.models import RunState -from envctl_engine.state import dump_state, load_legacy_shell_state, load_state, load_state_from_pointer, state_to_dict +from envctl_engine.state import load_legacy_shell_state, load_state, load_state_from_pointer, state_to_dict +from envctl_engine.state.fingerprints import file_fingerprint, state_fingerprint, text_fingerprint +from envctl_engine.state.models import RunState, ServiceRecord +from envctl_engine.state.persistence import ( + advisory_file_lock, + atomic_write_text, + durable_mkdir, + fsync_directory, + require_path_component, + scavenge_atomic_write_temps, +) +from envctl_engine.state.project_runtime import filter_project_scoped_metadata +from envctl_engine.state.run_index import RunIndex, RunIndexCandidate, StateSelector +from envctl_engine.state.runtime_map import build_runtime_map +from envctl_engine.state.state_aggregation import ( + IndexedState as _IndexedState, + filter_state_to_owned_projects, + normalized_project_names, + select_indexed_owners, + state_from_indexed_owners, +) @dataclass(slots=True) @@ -24,6 +46,25 @@ class RuntimeStateRepository: COMPAT_READ_WRITE: str = "compat_read_write" COMPAT_READ_ONLY: str = "compat_read_only" SCOPED_ONLY: str = "scoped_only" + _RUNTIME_ARTIFACT_NAMES = frozenset( + { + "events.jsonl", + "runtime_readiness_report.json", + } + ) + _REVISION_ANCILLARY_ARTIFACT_NAMES = ( + "ports_manifest.json", + "error_report.json", + "events.jsonl", + "runtime_readiness_report.json", + ) + _RUN_ALIAS_ARTIFACT_NAMES = ( + "run_state.json", + "runtime_map.json", + *_REVISION_ANCILLARY_ARTIFACT_NAMES, + ) + _RUN_ALIAS_MANIFEST_NAME = ".revision_alias_manifest.json" + _REVISION_CLEANUP_DIR_NAME = ".revision-cleanup" def __init__( self, @@ -34,9 +75,18 @@ def __init__( runtime_scope_id: str, compat_mode: str, ) -> None: - self.runtime_root: Path = runtime_root - self.runtime_legacy_root: Path = runtime_legacy_root - self.runtime_dir: Path = runtime_dir + input_runtime_root = runtime_root.expanduser().absolute() + input_legacy_root = runtime_legacy_root.expanduser().absolute() + if input_runtime_root.is_symlink() or input_legacy_root.is_symlink(): + raise ValueError("runtime roots must not be symlinks") + self.runtime_dir: Path = runtime_dir.resolve() + self.runtime_root: Path = runtime_root.resolve() + self.runtime_legacy_root: Path = runtime_legacy_root.resolve() + try: + self.runtime_root.relative_to(self.runtime_dir) + self.runtime_legacy_root.relative_to(self.runtime_dir) + except ValueError as exc: + raise ValueError("runtime roots must be contained by runtime_dir") from exc self.runtime_scope_id: str = runtime_scope_id self.compat_mode: str = ( compat_mode @@ -48,6 +98,16 @@ def __init__( } else self.COMPAT_READ_WRITE ) + self.run_index = RunIndex( + runtime_root=self.runtime_root, + runtime_dir=self.runtime_dir, + runtime_scope_id=self.runtime_scope_id, + ) + self._lock_path = self.runtime_root / ".state_repository.lock" + self._registry_marker_path = self.runtime_root / "run_registry.json" + self._alias_manifest_path = self.runtime_root / "current_alias.json" + self._runtime_root_identity = self._existing_directory_identity(self.runtime_root) + self._legacy_root_identity = self._existing_directory_identity(self.runtime_legacy_root) def run_state_path(self) -> Path: return self.runtime_root / "run_state.json" @@ -61,25 +121,36 @@ def ports_manifest_path(self) -> Path: def error_report_path(self) -> Path: return self.runtime_root / "error_report.json" + def lifecycle_operation_lock_path(self) -> Path: + self._validate_runtime_roots() + return self.runtime_root / ".lifecycle_operation.lock" + + def ensure_runtime_roots(self) -> None: + self._validate_runtime_roots() + def run_dir_path(self, run_id: str | None) -> Path: + runs_root = self._contained_child(self.runtime_root, "runs", label="runs directory") if run_id is None: - return self.runtime_root / "runs" - return self.runtime_root / "runs" / run_id + return runs_root + normalized_run_id = require_path_component(run_id, label="run_id") + return self._contained_child(runs_root, normalized_run_id, label="run_id") def test_results_dir_path(self, run_id: str, stamp: str | None = None) -> Path: - root = self.run_dir_path(run_id) / "test-results" + root = self._contained_child(self.run_dir_path(run_id), "test-results", label="test results directory") if not stamp: return root - return root / stamp + normalized_stamp = require_path_component(stamp, label="test result stamp") + return self._contained_child(root, normalized_stamp, label="test result stamp") def tree_diffs_dir_path(self, run_id: str | None = None, name: str | None = None) -> Path: if run_id: - root = self.run_dir_path(run_id) / "tree-diffs" + root = self._contained_child(self.run_dir_path(run_id), "tree-diffs", label="tree diffs directory") else: - root = self.runtime_root / "tree-diffs" + root = self._contained_child(self.runtime_root, "tree-diffs", label="tree diffs directory") if not name: return root - return root / name + normalized_name = require_path_component(name, label="tree diff name") + return self._contained_child(root, normalized_name, label="tree diff name") def save_run( self, @@ -92,69 +163,56 @@ def save_run( runtime_map_builder: Callable[[RunState], dict[str, object]], write_runtime_readiness_report: Callable[[Path], None] | None = None, ) -> StateRepositoryPaths: - run_dir = self.run_dir_path(state.run_id) - run_dir.mkdir(parents=True, exist_ok=True) - - run_state_path = run_dir / "run_state.json" - run_state_text = self._write_run_state(state=state, run_state_path=run_state_path, emit=emit) - - runtime_map_text = self._write_runtime_map_text( - state=state, - runtime_map=runtime_map_builder(state), - runtime_map_path=run_dir / "runtime_map.json", - emit_path=run_dir / "runtime_map.json", - emit=emit, - ) - - ports_manifest = { - "run_id": state.run_id, - "mode": state.mode, - "projects": [self._context_ports_payload(context) for context in contexts], - } - ports_manifest_text = self._json_text(ports_manifest) - self._write_run_and_current_artifact( - run_path=run_dir / "ports_manifest.json", - current_path=self.ports_manifest_path(), - text=ports_manifest_text, + project_names = sorted( + { + *self._context_names(contexts), + *self._runtime_project_names_from_state(state), + }, + key=str.casefold, ) - - error_report = { - "run_id": state.run_id, - "errors": errors, - "generated_at": datetime.now(tz=UTC).isoformat(), + self._prepare_state_for_save(state, project_names=project_names, contexts=contexts) + runtime_map = runtime_map_builder(state) + artifacts = { + "run_state.json": self._json_text(state_to_dict(state)), + "runtime_map.json": self._json_text(runtime_map), + "ports_manifest.json": self._json_text( + { + "run_id": state.run_id, + "mode": state.mode, + "projects": [self._context_ports_payload(context) for context in contexts], + } + ), + "error_report.json": self._json_text( + { + "run_id": state.run_id, + "errors": errors, + "generated_at": datetime.now(tz=UTC).isoformat(), + } + ), + "events.jsonl": self._events_text(events), } - error_report_text = self._json_text(error_report) - self._write_run_and_current_artifact( - run_path=run_dir / "error_report.json", - current_path=self.error_report_path(), - text=error_report_text, - ) - - events_text = self._events_text(events) - self._write_run_and_current_artifact( - run_path=run_dir / "events.jsonl", - current_path=self.runtime_root / "events.jsonl", - text=events_text, - ) - - if write_runtime_readiness_report is not None: - write_runtime_readiness_report(run_dir) - - self._write_scoped_pointers(run_state_path=run_state_path, state=state, contexts=contexts) - - if self.compat_mode == self.COMPAT_READ_WRITE: - self._write_legacy_compat( - run_state_path=run_state_path, - run_state_text=run_state_text, - runtime_map_text=runtime_map_text, - ports_manifest_text=ports_manifest_text, - error_report_text=error_report_text, - events_text=events_text, + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + self._carry_forward_runtime_artifacts_unlocked(state, artifacts) + revision_state_path = self._commit_revision_unlocked( state=state, - contexts=contexts, + project_names=project_names, + artifacts=artifacts, + supersede_run_ids=self._source_run_ids(state), ) - - return StateRepositoryPaths(run_dir=run_dir, run_state=run_state_path) + run_dir = self.run_dir_path(state.run_id) + if write_runtime_readiness_report is not None: + try: + write_runtime_readiness_report(run_dir) + except Exception as exc: + self._safe_emit(emit, "state.readiness_report.error", run_id=state.run_id, error=str(exc)) + self._emit_saved_state( + emit, + state=state, + state_path=revision_state_path, + runtime_map_text=artifacts["runtime_map.json"], + ) + return StateRepositoryPaths(run_dir=run_dir, run_state=run_dir / "run_state.json") def save_resume_state( self, @@ -175,65 +233,202 @@ def save_selected_stop_state( state: RunState, emit: Callable[..., None], runtime_map_builder: Callable[[RunState], dict[str, object]], + authoritative_project_names: Sequence[str] | None = None, ) -> dict[str, object]: return self._save_runtime_update( state=state, emit=emit, runtime_map_builder=runtime_map_builder, + authoritative_project_names=authoritative_project_names, ) - def load_latest(self, *, mode: str | None = None, strict_mode_match: bool = False) -> RunState | None: - allowed_root = str(self.runtime_dir) - - def load_for_mode(expected_mode: str | None) -> RunState | None: - def candidate_matches(candidate: RunState) -> bool: - if expected_mode is not None and candidate.mode != expected_mode: - return False - return self._state_matches_scope(candidate) + def load_latest( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, + ) -> RunState | None: + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + self._reconcile_current_aliases_unlocked() + return self._load_latest_unlocked( + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ) - candidate = self._load_state_file_candidate(self.run_state_path(), allowed_root=allowed_root) - if candidate is not None and candidate_matches(candidate): - return candidate + def load_all(self, *, mode: str | None = None) -> list[RunState]: + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + self._reconcile_current_aliases_unlocked() + candidates = self.run_index.candidates(StateSelector(mode=mode, project_names=())) + indexed_states = self._load_valid_index_states(candidates, allowed_root=str(self.runtime_dir)) + if len(indexed_states) != len(candidates): + invalid_run_ids = sorted( + {candidate.run_id for candidate in candidates} + - {indexed.candidate.run_id for indexed in indexed_states} + ) + raise RuntimeError( + "Active runtime state failed integrity validation for run(s): " + ", ".join(invalid_run_ids) + ) + return [ + filter_state_to_owned_projects( + indexed_state.state, + frozenset(indexed_state.candidate.project_names), + service_project_name=self._service_project_name, + ) + for indexed_state in indexed_states + ] + + def has_active_runs(self) -> bool: + """Return registry activity without trusting or loading mutable run payloads.""" + + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + return bool(self.run_index.candidates(StateSelector(mode=None, project_names=()))) + + def deactivate_run(self, run_id: str) -> bool: + return self.deactivate_runs([run_id]) + + def deactivate_runs(self, run_ids: Sequence[str]) -> bool: + normalized_run_ids = {require_path_component(run_id, label="run_id") for run_id in run_ids} + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + candidates = self.run_index.candidates(StateSelector(mode=None, project_names=())) + removed_candidates = [candidate for candidate in candidates if candidate.run_id in normalized_run_ids] + if not removed_candidates: + return False + removed_run_ids = sorted({candidate.run_id for candidate in removed_candidates}) + self.run_index.remove_many(removed_run_ids) + for candidate in removed_candidates: + self._remove_pointers_to_state(self.runtime_root, candidate.state_path) + if self.compat_mode == self.COMPAT_READ_WRITE: + self._remove_pointers_to_state(self.runtime_legacy_root, candidate.state_path) + try: + self._promote_latest_active_state() + finally: + for candidate in removed_candidates: + self._prune_or_enqueue_revision_cleanup_unlocked(candidate) + return True - candidate = self._load_legacy_shell_candidate( - self.runtime_root / "run_state.state", allowed_root=allowed_root + def write_runtime_artifact( + self, + *, + run_id: str | None, + artifact_name: str, + text: str, + ) -> bool: + """Persist a mutable runtime artifact without bypassing lifecycle fencing. + + Run-bound writers are accepted only while that exact run remains in the + active registry. An unbound writer is a deliberate scope-level diagnostic + write (for example, ``envctl doctor``) and is never guessed onto a run. + """ + safe_artifact_name = require_path_component(artifact_name, label="runtime artifact name") + if safe_artifact_name not in self._RUNTIME_ARTIFACT_NAMES: + raise ValueError(f"unsupported mutable runtime artifact: {safe_artifact_name}") + normalized_run_id = require_path_component(run_id, label="run_id") if run_id is not None else None + + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + if normalized_run_id is None: + candidates = self.run_index.candidates(StateSelector(mode=None, project_names=())) + if candidates: + self._write_scope_diagnostic_artifact(safe_artifact_name, text) + else: + self._remove_scope_diagnostic_artifact(safe_artifact_name) + self._write_scope_artifact_aliases(safe_artifact_name, text) + return True + + candidates = self.run_index.candidates(StateSelector(mode=None, project_names=())) + indexed_states = self._load_valid_index_states( + candidates, + allowed_root=str(self.runtime_dir), ) - if candidate is not None and candidate_matches(candidate): - return candidate - - candidate = self._load_pointer_candidates( - self._ordered_scoped_pointers(mode=expected_mode), - allowed_root=allowed_root, - predicate=candidate_matches, + indexed_state = next( + (candidate for candidate in indexed_states if candidate.candidate.run_id == normalized_run_id), + None, ) - if candidate is not None: - return candidate + if indexed_state is None: + return False + + candidate = indexed_state.candidate + self._write_text(candidate.state_path.parent / safe_artifact_name, text) + self._write_text(self.run_dir_path(candidate.run_id) / safe_artifact_name, text) + self._run_alias_manifest_path(candidate.run_id).unlink(missing_ok=True) + self._promote_latest_active_state() + return True + + def _write_scope_diagnostic_artifact(self, artifact_name: str, text: str) -> None: + diagnostics_dir = self._contained_child( + self.runtime_root, + "diagnostics", + label="diagnostics directory", + ) + durable_mkdir(diagnostics_dir) + self._write_text(diagnostics_dir / artifact_name, text) - if self.compat_mode == self.SCOPED_ONLY: - return None + def _remove_scope_diagnostic_artifact(self, artifact_name: str) -> None: + diagnostics_dir = self.runtime_root / "diagnostics" + if diagnostics_dir.is_symlink() or not diagnostics_dir.is_dir(): + return + (diagnostics_dir / artifact_name).unlink(missing_ok=True) + try: + diagnostics_dir.rmdir() + except OSError: + pass - candidate = self._load_state_file_candidate( - self.runtime_legacy_root / "run_state.json", - allowed_root=allowed_root, - ) - if candidate is not None and candidate_matches(candidate): - return candidate + def _write_scope_artifact_aliases(self, artifact_name: str, text: str) -> None: + self._write_text(self.runtime_root / artifact_name, text) + if self.compat_mode == self.COMPAT_READ_WRITE: + self._write_text(self.runtime_legacy_root / artifact_name, text) - candidate = self._load_legacy_shell_candidate( - self.runtime_legacy_root / "run_state.state", - allowed_root=allowed_root, - ) - if candidate is not None and candidate_matches(candidate): - return candidate + def _load_latest_unlocked( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: Sequence[str] | None = None, + ) -> RunState | None: + allowed_root = str(self.runtime_dir) + selected_projects = tuple(project_names or ()) - candidate = self._load_pointer_candidates( - self._ordered_legacy_pointers(mode=expected_mode), + def load_for_mode(expected_mode: str | None) -> RunState | None: + candidates = self.run_index.candidates(StateSelector(mode=expected_mode, project_names=())) + if expected_mode is None and candidates: + valid_candidates = self._load_valid_index_states( + candidates, + allowed_root=allowed_root, + ) + selected_project_keys = normalized_project_names(selected_projects) + if selected_project_keys: + valid_candidates = [ + indexed + for indexed in valid_candidates + if selected_project_keys.intersection( + normalized_project_names( + self._project_names_from_state(indexed.state) + ) + ) + ] + if not valid_candidates: + return None + newest_mode = max( + valid_candidates, + key=lambda indexed: indexed.candidate.activation_sequence, + ).candidate.mode + candidates = [ + indexed.candidate + for indexed in valid_candidates + if indexed.candidate.mode == newest_mode + ] + indexed_candidate = self._load_index_candidates( + candidates, allowed_root=allowed_root, - predicate=candidate_matches, + selected_projects=selected_projects, ) - if candidate is not None: - return candidate - return None + return indexed_candidate matched = load_for_mode(mode) if matched is not None: @@ -248,42 +443,608 @@ def _save_runtime_update( state: RunState, emit: Callable[..., None], runtime_map_builder: Callable[[RunState], dict[str, object]], + authoritative_project_names: Sequence[str] | None = None, ) -> dict[str, object]: - run_state_path = self._runtime_update_run_state_path(state) - run_state_text = self._write_run_state(state=state, run_state_path=run_state_path, emit=emit) - + project_names = ( + self._runtime_project_names_from_state(state) + if authoritative_project_names is None + else sorted( + { + str(project_name).strip() + for project_name in authoritative_project_names + if str(project_name).strip() + }, + key=str.casefold, + ) + ) + self._prepare_state_for_save(state, project_names=project_names, authoritative_projects=True) runtime_map = runtime_map_builder(state) - runtime_map_text = self._write_runtime_map_text( + if not project_names: + run_ids = [state.run_id, *self._source_run_ids(state)] + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + self.run_index.remove_many(run_ids) + self._promote_latest_active_state() + self._safe_emit(emit, "state.deactivate", run_ids=sorted(set(run_ids))) + return runtime_map + artifacts = { + "run_state.json": self._json_text(state_to_dict(state)), + "runtime_map.json": self._json_text(runtime_map), + } + with self._repository_lock(exclusive=True): + self._ensure_active_index_unlocked() + self._carry_forward_runtime_artifacts_unlocked(state, artifacts) + revision_state_path = self._commit_revision_unlocked( + state=state, + project_names=project_names, + artifacts=artifacts, + supersede_run_ids=self._source_run_ids(state), + ) + self._emit_saved_state( + emit, state=state, - runtime_map=runtime_map, - runtime_map_path=run_state_path.parent / "runtime_map.json", - emit_path=self.runtime_map_path(), - emit=emit, + state_path=revision_state_path, + runtime_map_text=artifacts["runtime_map.json"], ) + return runtime_map + def _carry_forward_runtime_artifacts_unlocked( + self, + state: RunState, + artifacts: dict[str, str], + ) -> None: + source_run_ids = {state.run_id, *self._source_run_ids(state)} + source_candidates = [ + candidate + for candidate in self.run_index.candidates(StateSelector(mode=state.mode, project_names=())) + if candidate.run_id in source_run_ids + ] + current_ports_manifest = artifacts.get("ports_manifest.json") + artifacts["ports_manifest.json"] = self._merged_ports_manifest_text( + state, + source_candidates, + current_manifest_text=current_ports_manifest, + ) + artifacts.setdefault( + "error_report.json", + self._merged_error_report_text(state, source_candidates), + ) + if not source_candidates: + return + source_candidate = max( + source_candidates, + key=lambda candidate: candidate.activation_sequence, + ) + source_directories = ( + source_candidate.state_path.parent, + self.run_dir_path(source_candidate.run_id), + ) + for artifact_name in self._RUNTIME_ARTIFACT_NAMES: + if artifact_name in artifacts: + continue + for source_directory in source_directories: + source = source_directory / artifact_name + if source.is_file() and not source.is_symlink(): + try: + artifacts[artifact_name] = source.read_text(encoding="utf-8") + except (OSError, UnicodeError): + continue + break + + def _merged_ports_manifest_text( + self, + state: RunState, + source_candidates: Sequence[RunIndexCandidate], + *, + current_manifest_text: str | None = None, + ) -> str: project_names = self._project_names_from_state(state) - self._write_mode_pointers( - root=self.runtime_root, - run_state_path=run_state_path, - mode=state.mode, - project_names=project_names, + selected = normalized_project_names(project_names) + projects_by_key: dict[str, dict[str, object]] = {} + for candidate in sorted( + source_candidates, + key=lambda item: item.activation_sequence, + ): + payload = self._read_revision_json_artifact(candidate, "ports_manifest.json") + raw_projects = payload.get("projects") if isinstance(payload, Mapping) else None + if not isinstance(raw_projects, list): + continue + for raw_project in raw_projects: + if not isinstance(raw_project, Mapping): + continue + project_name = str(raw_project.get("project", "")).strip() + project_key = project_name.casefold() + if project_key and project_key in selected: + projects_by_key[project_key] = dict(raw_project) + + if current_manifest_text is not None: + try: + current_payload = json.loads(current_manifest_text) + except (TypeError, ValueError, json.JSONDecodeError): + current_payload = {} + raw_current_projects = current_payload.get("projects") if isinstance(current_payload, Mapping) else None + if isinstance(raw_current_projects, list): + for raw_project in raw_current_projects: + if not isinstance(raw_project, Mapping): + continue + project_name = str(raw_project.get("project", "")).strip() + project_key = project_name.casefold() + if project_key and project_key in selected: + projects_by_key[project_key] = dict(raw_project) + + roots = state.metadata.get("project_roots") + project_roots = roots if isinstance(roots, Mapping) else {} + for project_name in project_names: + project_key = project_name.casefold() + project_payload = projects_by_key.get(project_key) + if project_payload is None: + project_payload = { + "project": project_name, + "root": next( + ( + str(root) + for name, root in project_roots.items() + if str(name).strip().casefold() == project_key + ), + "", + ), + "ports": {}, + } + projects_by_key[project_key] = project_payload + raw_ports = project_payload.get("ports") + ports = dict(raw_ports) if isinstance(raw_ports, Mapping) else {} + for service_name, service in state.services.items(): + if self._service_project_name(service_name, service).casefold() != project_key: + continue + service_key = str(service.type or service_name).strip().casefold() or "service" + ports.setdefault( + service_key, + { + "requested": service.requested_port, + "assigned": service.actual_port or service.requested_port, + "final": service.actual_port, + "source": "runtime_state", + "retries": None, + }, + ) + project_payload["ports"] = ports + return self._json_text( + { + "run_id": state.run_id, + "mode": state.mode, + "projects": [projects_by_key[key] for key in sorted(projects_by_key)], + } ) - if self.compat_mode == self.COMPAT_READ_WRITE: - self.runtime_legacy_root.mkdir(parents=True, exist_ok=True) - self._write_text(self.runtime_legacy_root / "run_state.json", run_state_text) - self._write_text(self.runtime_legacy_root / "runtime_map.json", runtime_map_text) - self._write_mode_pointers( - root=self.runtime_legacy_root, - run_state_path=run_state_path, + def _merged_error_report_text( + self, + state: RunState, + source_candidates: Sequence[RunIndexCandidate], + ) -> str: + errors: list[str] = [] + for candidate in sorted(source_candidates, key=lambda item: item.activation_sequence): + payload = self._read_revision_json_artifact(candidate, "error_report.json") + raw_errors = payload.get("errors") if isinstance(payload, Mapping) else None + if not isinstance(raw_errors, list): + continue + for raw_error in raw_errors: + error = str(raw_error).strip() + if error and error not in errors: + errors.append(error) + return self._json_text( + { + "run_id": state.run_id, + "errors": errors, + "source_run_ids": sorted({candidate.run_id for candidate in source_candidates}), + "generated_at": datetime.now(tz=UTC).isoformat(), + } + ) + + @staticmethod + def _read_revision_json_artifact( + candidate: RunIndexCandidate, + artifact_name: str, + ) -> Mapping[str, object]: + path = candidate.state_path.parent / artifact_name + if path.is_symlink() or not path.is_file(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return {} + return payload if isinstance(payload, Mapping) else {} + + def _commit_revision_unlocked( + self, + *, + state: RunState, + project_names: Sequence[str], + artifacts: Mapping[str, str], + supersede_run_ids: Sequence[str] = (), + ) -> Path: + superseded_run_ids = { + require_path_component(run_id, label="superseded run_id") + for run_id in supersede_run_ids + if str(run_id).strip() and str(run_id).strip() != state.run_id + } + superseded_candidates = [ + candidate + for candidate in self.run_index.candidates(StateSelector(mode=None, project_names=())) + if candidate.run_id in superseded_run_ids + ] + run_dir = self.run_dir_path(state.run_id) + revisions_dir = self._contained_child(run_dir, "revisions", label="revisions directory") + revision_id = f"{datetime.now(tz=UTC).strftime('%Y%m%dT%H%M%S%fZ')}-{uuid.uuid4().hex[:12]}" + revision_dir = self._contained_child(revisions_dir, revision_id, label="revision id") + revision_state_path = revision_dir / "run_state.json" + committed = False + try: + durable_mkdir(revision_dir) + for artifact_name, text in artifacts.items(): + safe_name = require_path_component(artifact_name, label="artifact name") + self._write_text(revision_dir / safe_name, text) + self.run_index.record( + state_path=revision_state_path, + run_id=state.run_id, mode=state.mode, project_names=project_names, + supersede_run_ids=supersede_run_ids, ) + committed = True + finally: + if not committed: + shutil.rmtree(revision_dir, ignore_errors=True) - return runtime_map + try: + self._publish_revision_aliases( + state=state, + artifacts=artifacts, + ) + self._promote_latest_active_state() + for candidate in superseded_candidates: + self._prune_or_enqueue_revision_cleanup_unlocked(candidate) + except OSError: + # The index is the commit point. Aliases are repaired on the next repository read. + for candidate in superseded_candidates: + self._enqueue_revision_cleanup_if_needed_unlocked(candidate) + except BaseException: + for candidate in superseded_candidates: + self._enqueue_revision_cleanup_if_needed_unlocked(candidate) + raise + return revision_state_path + + def _publish_revision_aliases( + self, + *, + state: RunState, + artifacts: Mapping[str, str], + ) -> None: + run_dir = self.run_dir_path(state.run_id) + for artifact_name in self._RUN_ALIAS_ARTIFACT_NAMES: + alias = run_dir / artifact_name + text = artifacts.get(artifact_name) + if text is None: + alias.unlink(missing_ok=True) + else: + self._write_text(alias, text) + candidates = self.run_index.candidates(StateSelector(mode=state.mode, project_names=())) + committed = next( + (candidate for candidate in candidates if candidate.run_id == state.run_id), + None, + ) + if committed is not None: + self._write_run_alias_manifest(committed) + + def _run_alias_manifest_path(self, run_id: str) -> Path: + return self.run_dir_path(run_id) / self._RUN_ALIAS_MANIFEST_NAME + + def _write_run_alias_manifest(self, candidate: RunIndexCandidate) -> None: + source_dir = candidate.state_path.parent + run_dir = self.run_dir_path(candidate.run_id) + artifacts = { + artifact_name: { + "source": self._path_lstat_signature(source_dir / artifact_name), + "alias": self._path_lstat_signature(run_dir / artifact_name), + } + for artifact_name in self._RUN_ALIAS_ARTIFACT_NAMES + } + self._write_text( + self._run_alias_manifest_path(candidate.run_id), + self._json_text( + { + "version": 1, + "source_dir": str(source_dir), + "artifacts": artifacts, + } + ), + ) + + def _run_aliases_match_manifest(self, candidate: RunIndexCandidate) -> bool: + manifest_path = self._run_alias_manifest_path(candidate.run_id) + if manifest_path.is_symlink() or not manifest_path.is_file(): + return False + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return False + if not isinstance(manifest, Mapping): + return False + if manifest.get("version") != 1 or manifest.get("source_dir") != str(candidate.state_path.parent): + return False + expected_artifacts = manifest.get("artifacts") + if not isinstance(expected_artifacts, Mapping): + return False + source_dir = candidate.state_path.parent + run_dir = self.run_dir_path(candidate.run_id) + actual_artifacts = { + artifact_name: { + "source": self._path_lstat_signature(source_dir / artifact_name), + "alias": self._path_lstat_signature(run_dir / artifact_name), + } + for artifact_name in self._RUN_ALIAS_ARTIFACT_NAMES + } + return expected_artifacts == actual_artifacts + + @staticmethod + def _path_lstat_signature(path: Path) -> dict[str, object]: + try: + metadata = path.lstat() + except FileNotFoundError: + return {"kind": "missing"} + if not stat.S_ISREG(metadata.st_mode): + return { + "kind": "unsafe", + "mode": metadata.st_mode, + "inode": metadata.st_ino, + "size": metadata.st_size, + "mtime_ns": metadata.st_mtime_ns, + "ctime_ns": metadata.st_ctime_ns, + } + return { + "kind": "file", + "inode": metadata.st_ino, + "size": metadata.st_size, + "mtime_ns": metadata.st_mtime_ns, + "ctime_ns": metadata.st_ctime_ns, + } + + def _prune_revisions_for_candidates_unlocked( + self, + indexed_states: Sequence[_IndexedState], + ) -> None: + seen_run_ids: set[str] = set() + for indexed in indexed_states: + candidate = indexed.candidate + if candidate.run_id in seen_run_ids: + continue + seen_run_ids.add(candidate.run_id) + try: + self._prune_run_revisions_unlocked(candidate) + except OSError: + # Active candidates are retried by the next reconciliation. + continue + self._retry_pending_revision_cleanups_unlocked() + + def _prune_run_revisions_unlocked(self, candidate: RunIndexCandidate) -> None: + run_dir = self.run_dir_path(candidate.run_id) + revisions_dir = self._contained_child( + run_dir, + "revisions", + label="revisions directory", + ) + current_revision = candidate.state_path.parent + if ( + revisions_dir.is_symlink() + or not revisions_dir.is_dir() + or current_revision.is_symlink() + or not current_revision.is_dir() + or current_revision.resolve().parent != revisions_dir.resolve() + ): + return + siblings: list[tuple[int, str, Path]] = [] + for path in revisions_dir.iterdir(): + if path == current_revision or path.is_symlink() or not path.is_dir(): + continue + try: + if path.resolve().parent != revisions_dir.resolve(): + continue + metadata = path.stat() + except OSError: + continue + siblings.append((metadata.st_mtime_ns, path.name, path)) + siblings.sort(reverse=True) + removed = False + try: + for _, _, stale_revision in siblings[1:]: + shutil.rmtree(stale_revision) + removed = True + finally: + if removed: + fsync_directory(revisions_dir) + + def _enqueue_revision_cleanup_unlocked(self, candidate: RunIndexCandidate) -> None: + marker_path = self._revision_cleanup_marker_path(candidate.run_id) + run_dir = self.run_dir_path(candidate.run_id) + revisions_dir = self._contained_child( + run_dir, + "revisions", + label="revisions directory", + ) + state_path = candidate.state_path.resolve() + if state_path.name != "run_state.json" or state_path.parent.parent != revisions_dir.resolve(): + raise ValueError("revision cleanup state path is outside the run revisions directory") + cleanup_dir = marker_path.parent + durable_mkdir(cleanup_dir) + self._write_text( + marker_path, + self._json_text( + { + "version": 1, + "runtime_scope_id": self.runtime_scope_id, + "run_id": candidate.run_id, + "mode": candidate.mode, + "state_path": str(state_path), + } + ), + ) + + def _prune_or_enqueue_revision_cleanup_unlocked( + self, + candidate: RunIndexCandidate, + ) -> None: + try: + self._prune_run_revisions_unlocked(candidate) + except OSError: + self._enqueue_revision_cleanup_if_needed_unlocked(candidate) + + def _enqueue_revision_cleanup_if_needed_unlocked( + self, + candidate: RunIndexCandidate, + ) -> None: + if self._run_revision_count(candidate.run_id) <= 2: + return + try: + self._enqueue_revision_cleanup_unlocked(candidate) + except (OSError, RuntimeError, ValueError): + return + + def _retry_pending_revision_cleanups_unlocked(self) -> None: + cleanup_dir = self.runtime_root / self._REVISION_CLEANUP_DIR_NAME + if cleanup_dir.is_symlink() or not cleanup_dir.is_dir(): + return + try: + marker_paths = list(cleanup_dir.iterdir()) + except OSError: + return + for marker_path in marker_paths: + if marker_path.is_symlink() or not marker_path.is_file(): + continue + candidate = self._revision_cleanup_candidate(marker_path) + if candidate is None: + self._remove_revision_cleanup_marker(marker_path) + continue + try: + self._prune_run_revisions_unlocked(candidate) + except OSError: + continue + if self._run_revision_count(candidate.run_id) <= 2: + self._remove_revision_cleanup_marker(marker_path) + try: + cleanup_dir.rmdir() + except OSError: + return + try: + fsync_directory(self.runtime_root) + except OSError: + pass + + def _revision_cleanup_candidate( + self, + marker_path: Path, + ) -> RunIndexCandidate | None: + try: + payload = json.loads(marker_path.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + return None + if payload.get("version") != 1 or payload.get("runtime_scope_id") != self.runtime_scope_id: + return None + raw_run_id = payload.get("run_id") + raw_mode = payload.get("mode") + raw_state_path = payload.get("state_path") + if not all(isinstance(value, str) and value for value in (raw_run_id, raw_mode, raw_state_path)): + return None + run_id = require_path_component(raw_run_id, label="run_id") + if marker_path != self._revision_cleanup_marker_path(run_id): + return None + run_dir = self.run_dir_path(run_id) + revisions_dir = self._contained_child( + run_dir, + "revisions", + label="revisions directory", + ) + state_path = Path(raw_state_path).expanduser().resolve() + if state_path.name != "run_state.json" or state_path.parent.parent != revisions_dir.resolve(): + return None + state = self._load_state_file_candidate( + state_path, + allowed_root=str(self.runtime_dir), + ) + if state is None or state.run_id != run_id: + return None + if state.metadata.get("repo_scope_id") != self.runtime_scope_id: + return None + if state.mode.casefold() != raw_mode.casefold(): + return None + return RunIndexCandidate( + state_path=state_path, + run_id=run_id, + mode=state.mode, + project_names=(), + sequence=0, + ) + except (OSError, RuntimeError, TypeError, ValueError, json.JSONDecodeError): + return None + + def _revision_cleanup_marker_path(self, run_id: str) -> Path: + normalized_run_id = require_path_component(run_id, label="run_id") + cleanup_dir = self._contained_child( + self.runtime_root, + self._REVISION_CLEANUP_DIR_NAME, + label="revision cleanup directory", + ) + digest = text_fingerprint(normalized_run_id) + marker_name = f"{digest}.json" + return self._contained_child( + cleanup_dir, + marker_name, + label="revision cleanup marker", + ) + + def _remove_revision_cleanup_marker(self, marker_path: Path) -> None: + try: + marker_path.unlink(missing_ok=True) + fsync_directory(marker_path.parent) + except OSError: + return + + def _run_revision_count(self, run_id: str) -> int: + revisions_dir = self.run_dir_path(run_id) / "revisions" + if revisions_dir.is_symlink() or not revisions_dir.is_dir(): + return 0 + try: + return sum(1 for path in revisions_dir.iterdir() if not path.is_symlink() and path.is_dir()) + except OSError: + return 3 + + def _emit_saved_state( + self, + emit: Callable[..., None], + *, + state: RunState, + state_path: Path, + runtime_map_text: str, + ) -> None: + self._safe_emit(emit, "state.save", run_id=state.run_id, path=str(state_path)) + self._safe_emit( + emit, + "state.fingerprint.after_save", + run_id=state.run_id, + state_fingerprint=state_fingerprint(state), + ) + self._safe_emit(emit, "runtime_map.write", path=str(state_path.parent / "runtime_map.json")) + self._safe_emit( + emit, + "runtime_map.fingerprint", + run_id=state.run_id, + runtime_map_fingerprint=text_fingerprint(runtime_map_text), + ) - def load_by_pointer(self, pointer_path: str) -> RunState: - return load_state_from_pointer(pointer_path, allowed_root=str(self.runtime_dir)) + @staticmethod + def _safe_emit(emit: Callable[..., None], event: str, **payload: object) -> None: + try: + emit(event, **payload) + except Exception: + return def _load_state_file_candidate(self, path: Path, *, allowed_root: str) -> RunState | None: if not path.is_file(): @@ -293,68 +1054,499 @@ def _load_state_file_candidate(self, path: Path, *, allowed_root: str) -> RunSta except Exception: return None - def _load_legacy_shell_candidate(self, path: Path, *, allowed_root: str) -> RunState | None: - if not path.is_file(): + def _load_index_candidates( + self, + candidates: Sequence[RunIndexCandidate], + *, + allowed_root: str, + selected_projects: Sequence[str], + ) -> RunState | None: + indexed_states = self._load_valid_index_states(candidates, allowed_root=allowed_root) + if not indexed_states: return None + chosen_states = select_indexed_owners( + indexed_states, + selected_projects=selected_projects, + service_project_name=self._service_project_name, + ) + return state_from_indexed_owners( + chosen_states, + project_names_from_state=self._project_names_from_state, + source_run_ids=self._source_run_ids, + ) + + def _ensure_active_index_unlocked(self) -> None: + if not self.run_index.needs_rebuild(): + self.run_index.repair_copies() + if not self._registry_marker_path.is_file(): + self._write_text( + self._registry_marker_path, + self._json_text({"version": 1, "runtime_scope_id": self.runtime_scope_id}), + ) + return + if self._registry_marker_path.is_file(): + raise RuntimeError( + "Active run registry is unavailable because both the primary and backup indexes are invalid." + ) + + recovered: list[tuple[int, Path, RunState, list[str]]] = [] + runs_root = self.run_dir_path(None) + if runs_root.is_dir(): + for run_dir in runs_root.iterdir(): + if run_dir.is_symlink() or not run_dir.is_dir(): + continue + state_path = run_dir / "run_state.json" + if state_path.is_symlink() or not state_path.is_file(): + continue + try: + run_id = require_path_component(run_dir.name, label="run_id") + if run_dir.resolve().parent != runs_root.resolve(): + continue + state = self._load_state_file_candidate(state_path, allowed_root=str(self.runtime_dir)) + if state is None or state.run_id != run_id: + continue + scope = state.metadata.get("repo_scope_id") + if isinstance(scope, str) and scope and scope != self.runtime_scope_id: + continue + project_names = self._project_names_from_state(state) + if not project_names: + project_names = self._project_names_from_ports_manifest( + state_path.parent / "ports_manifest.json" + ) + if not project_names: + project_names = self._project_names_from_pointers( + self.runtime_root, + state_path, + ) + modified_at = state_path.stat().st_mtime_ns + state.metadata["repo_scope_id"] = self.runtime_scope_id + state.metadata["project_names"] = sorted(set(project_names), key=str.casefold) + recovered.append((modified_at, state_path, state, project_names)) + except (OSError, RuntimeError, ValueError): + continue + recovered_run_ids = {state.run_id for _, _, state, _ in recovered} + for root in self._legacy_migration_roots(): + legacy_candidates = [ + (root / "run_state.json", self._load_state_file_candidate), + (root / "run_state.state", self._load_legacy_shell_candidate), + ] + legacy_candidates.extend( + (pointer, self._load_pointer_candidate) for pointer in self._legacy_pointer_paths(root) + ) + for candidate_path, legacy_loader in legacy_candidates: + if candidate_path.is_symlink() or not candidate_path.is_file(): + continue + try: + state = legacy_loader(candidate_path, allowed_root=str(self.runtime_dir)) + if state is None or state.run_id in recovered_run_ids: + continue + existing_scope = state.metadata.get("repo_scope_id") + if isinstance(existing_scope, str) and existing_scope and existing_scope != self.runtime_scope_id: + continue + project_names = self._project_names_from_state(state) + if not project_names: + project_names = self._project_names_from_ports_manifest(root / "ports_manifest.json") + if not project_names: + project_names = self._project_names_from_pointers(root, candidate_path) + if not project_names and candidate_path.name.startswith(".last_state.trees."): + project_name = candidate_path.name.removeprefix(".last_state.trees.").strip() + if project_name: + project_names = [project_name] + state.metadata["repo_scope_id"] = self.runtime_scope_id + state.metadata["project_names"] = sorted(set(project_names), key=str.casefold) + recovered.append((candidate_path.stat().st_mtime_ns, candidate_path, state, project_names)) + recovered_run_ids.add(state.run_id) + except (OSError, RuntimeError, ValueError): + continue + + indexed_entries: list[RunIndexCandidate] = [] + registry_committed = False try: - return load_legacy_shell_state(str(path), allowed_root=allowed_root) + for sequence, (_, source_path, state, project_names) in enumerate( + sorted(recovered, key=lambda item: (item[0], str(item[1]))), + start=1, + ): + revision_state_path = self._stage_legacy_revision( + state=state, + source_path=source_path, + ) + indexed_entries.append( + RunIndexCandidate( + state_path=revision_state_path, + run_id=state.run_id, + mode=state.mode, + project_names=tuple(sorted(normalized_project_names(project_names))), + sequence=sequence, + ) + ) + self.run_index.replace_all(indexed_entries) + registry_committed = True + finally: + if not registry_committed: + for entry in indexed_entries: + shutil.rmtree(entry.state_path.parent, ignore_errors=True) + self._write_text( + self._registry_marker_path, + self._json_text({"version": 1, "runtime_scope_id": self.runtime_scope_id}), + ) + + def _legacy_migration_roots(self) -> list[Path]: + roots = [self.runtime_root] + if self.compat_mode != self.SCOPED_ONLY and self.runtime_legacy_root != self.runtime_root: + roots.append(self.runtime_legacy_root) + return roots + + @staticmethod + def _legacy_pointer_paths(root: Path) -> list[Path]: + candidates = [ + root / ".last_state.main", + *sorted(root.glob(".last_state.trees.*")), + root / ".last_state", + ] + return [pointer for pointer in candidates if not pointer.is_symlink() and pointer.is_file()] + + @staticmethod + def _load_pointer_candidate(path: Path, *, allowed_root: str) -> RunState | None: + try: + return load_state_from_pointer(str(path), allowed_root=allowed_root) except Exception: return None - def _load_pointer_candidates( + def _stage_legacy_revision(self, *, state: RunState, source_path: Path) -> Path: + run_dir = self.run_dir_path(state.run_id) + revisions_dir = self._contained_child(run_dir, "revisions", label="revisions directory") + revision_id = f"legacy-{uuid.uuid4().hex}" + revision_dir = self._contained_child(revisions_dir, revision_id, label="revision id") + revision_state_path = revision_dir / "run_state.json" + staged = False + try: + durable_mkdir(revision_dir) + self._write_text(revision_state_path, self._json_text(state_to_dict(state))) + for artifact_name in ( + "runtime_map.json", + *self._REVISION_ANCILLARY_ARTIFACT_NAMES, + ): + source_artifact = source_path.parent / artifact_name + if source_artifact.is_symlink() or not source_artifact.is_file(): + continue + try: + self._write_text( + revision_dir / artifact_name, + source_artifact.read_text(encoding="utf-8"), + ) + except (OSError, UnicodeError): + continue + staged = True + finally: + if not staged: + shutil.rmtree(revision_dir, ignore_errors=True) + return revision_state_path + + @staticmethod + def _project_names_from_ports_manifest(path: Path) -> list[str]: + if not path.is_file(): + return [] + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return [] + raw_projects = payload.get("projects") if isinstance(payload, Mapping) else None + if not isinstance(raw_projects, list): + return [] + return sorted( + { + str(project.get("project", "")).strip() + for project in raw_projects + if isinstance(project, Mapping) and str(project.get("project", "")).strip() + }, + key=str.casefold, + ) + + @staticmethod + def _project_names_from_pointers(root: Path, state_path: Path) -> list[str]: + expected = state_path.resolve() + project_names: set[str] = set() + for pointer in root.glob(".last_state.trees.*"): + if pointer.is_symlink() or not pointer.is_file(): + continue + try: + raw_target = pointer.read_text(encoding="utf-8").splitlines()[0].strip() + target = Path(raw_target).expanduser().resolve() + except (IndexError, OSError, RuntimeError, ValueError): + continue + if target == expected: + project_names.add(pointer.name.removeprefix(".last_state.trees.")) + return sorted(project_names, key=str.casefold) + + def _load_valid_index_states( self, - pointers: list[Path], + candidates: Sequence[RunIndexCandidate], *, allowed_root: str, - predicate: Callable[[RunState], bool], - ) -> RunState | None: - for pointer in pointers: + ) -> list[_IndexedState]: + loaded: list[_IndexedState] = [] + for indexed in candidates: try: - candidate = load_state_from_pointer(str(pointer), allowed_root=allowed_root) - except Exception: + if file_fingerprint(indexed.state_path) != indexed.state_fingerprint: + continue + if indexed.runtime_map_fingerprint is not None: + runtime_map_path = indexed.state_path.parent / "runtime_map.json" + if ( + runtime_map_path.is_symlink() + or not runtime_map_path.is_file() + or file_fingerprint(runtime_map_path) != indexed.runtime_map_fingerprint + ): + continue + except OSError: continue - if predicate(candidate): - return candidate - return None + state = self._load_state_file_candidate(indexed.state_path, allowed_root=allowed_root) + if state is None: + continue + if state.run_id != indexed.run_id or state.mode.casefold() != indexed.mode: + continue + if state.metadata.get("repo_scope_id") != self.runtime_scope_id: + continue + state_projects = normalized_project_names(self._project_names_from_state(state)) + indexed_projects = frozenset(indexed.project_names) + if indexed_projects and not indexed_projects.issubset(state_projects): + continue + loaded.append(_IndexedState(candidate=indexed, state=state)) + return loaded + + def _promote_latest_active_state(self) -> None: + candidates = self.run_index.candidates(StateSelector(mode=None, project_names=())) + if not candidates: + self._remove_current_aliases() + return + valid_candidates = self._load_valid_index_states( + candidates, + allowed_root=str(self.runtime_dir), + ) + if not valid_candidates: + self._remove_current_aliases() + return + latest_candidate = max( + valid_candidates, + key=lambda indexed: indexed.candidate.activation_sequence, + ).candidate + mode_candidates = [indexed for indexed in valid_candidates if indexed.candidate.mode == latest_candidate.mode] + chosen_states = select_indexed_owners( + mode_candidates, + selected_projects=(), + service_project_name=self._service_project_name, + ) + latest_state = state_from_indexed_owners( + chosen_states, + project_names_from_state=self._project_names_from_state, + source_run_ids=self._source_run_ids, + ) + if latest_state is None: + self._remove_current_aliases() + return + selected_candidates = [indexed.candidate for indexed in chosen_states] + latest_selected = max( + selected_candidates, + key=lambda candidate: ( + candidate.activation_sequence, + candidate.sequence, + str(candidate.state_path), + ), + ) + source_dir = latest_selected.state_path.parent + current_state_text = self._json_text(state_to_dict(latest_state)) + current_runtime_map_text = self._json_text(build_runtime_map(latest_state)) + aggregate_artifacts = { + "ports_manifest.json": self._merged_ports_manifest_text( + latest_state, + selected_candidates, + ), + "error_report.json": self._merged_error_report_text( + latest_state, + selected_candidates, + ), + } + self._write_text(self.run_state_path(), current_state_text) + self._write_text(self.runtime_map_path(), current_runtime_map_text) + for artifact_name in self._REVISION_ANCILLARY_ARTIFACT_NAMES: + current = self.runtime_root / artifact_name + aggregate_text = aggregate_artifacts.get(artifact_name) + if aggregate_text is not None: + self._write_text(current, aggregate_text) + else: + source = source_dir / artifact_name + if source.is_file() and not source.is_symlink(): + self._write_text(current, source.read_text(encoding="utf-8")) + else: + current.unlink(missing_ok=True) + self._rebuild_mode_pointers(self.runtime_root, valid_candidates) + if self.compat_mode == self.COMPAT_READ_WRITE: + self._write_text(self.runtime_legacy_root / "run_state.json", current_state_text) + self._write_text(self.runtime_legacy_root / "runtime_map.json", current_runtime_map_text) + for artifact_name in self._REVISION_ANCILLARY_ARTIFACT_NAMES: + current = self.runtime_legacy_root / artifact_name + aggregate_text = aggregate_artifacts.get(artifact_name) + if aggregate_text is not None: + self._write_text(current, aggregate_text) + else: + source = source_dir / artifact_name + if source.is_file() and not source.is_symlink(): + self._write_text(current, source.read_text(encoding="utf-8")) + else: + current.unlink(missing_ok=True) + self._rebuild_mode_pointers(self.runtime_legacy_root, valid_candidates) + self._write_text( + self._alias_manifest_path, + self._json_text( + { + "version": 2, + "sources": { + indexed.candidate.run_id: str(indexed.candidate.state_path) for indexed in valid_candidates + }, + "source_fingerprints": { + indexed.candidate.run_id: state_fingerprint(indexed.state) for indexed in valid_candidates + }, + "state_fingerprint": text_fingerprint(current_state_text), + "runtime_map_fingerprint": text_fingerprint(current_runtime_map_text), + "source_artifact_signatures": self._revision_artifact_signatures(valid_candidates), + "scoped_alias_signatures": self._artifact_signatures(self.runtime_root), + "legacy_alias_signatures": ( + self._artifact_signatures(self.runtime_legacy_root) + if self.compat_mode == self.COMPAT_READ_WRITE + else None + ), + } + ), + ) + self._prune_revisions_for_candidates_unlocked(valid_candidates) + + def _reconcile_current_aliases_unlocked(self) -> None: + candidates = self.run_index.candidates(StateSelector(mode=None, project_names=())) + if not candidates: + if self.run_state_path().exists() or (self.runtime_root / ".last_state").exists(): + self._remove_current_aliases() + return + valid_candidates = self._load_valid_index_states( + candidates, + allowed_root=str(self.runtime_dir), + ) + expected_sources = {indexed.candidate.run_id: str(indexed.candidate.state_path) for indexed in valid_candidates} + expected_source_fingerprints = { + indexed.candidate.run_id: state_fingerprint(indexed.state) for indexed in valid_candidates + } + expected_source_artifact_signatures = self._revision_artifact_signatures(valid_candidates) + try: + self._reconcile_run_aliases_unlocked(valid_candidates) + except OSError: + pass + try: + manifest = json.loads(self._alias_manifest_path.read_text(encoding="utf-8")) + state_text = self.run_state_path().read_text(encoding="utf-8") + runtime_map_text = self.runtime_map_path().read_text(encoding="utf-8") + if ( + isinstance(manifest, Mapping) + and manifest.get("version") == 2 + and manifest.get("sources") == expected_sources + and manifest.get("source_fingerprints") == expected_source_fingerprints + and manifest.get("source_artifact_signatures") == expected_source_artifact_signatures + and manifest.get("state_fingerprint") == text_fingerprint(state_text) + and manifest.get("runtime_map_fingerprint") == text_fingerprint(runtime_map_text) + and manifest.get("scoped_alias_signatures") == self._artifact_signatures(self.runtime_root) + and manifest.get("legacy_alias_signatures") + == ( + self._artifact_signatures(self.runtime_legacy_root) + if self.compat_mode == self.COMPAT_READ_WRITE + else None + ) + and self._mode_pointers_match(self.runtime_root, valid_candidates) + and ( + self.compat_mode != self.COMPAT_READ_WRITE + or self._mode_pointers_match(self.runtime_legacy_root, valid_candidates) + ) + ): + self._prune_revisions_for_candidates_unlocked(valid_candidates) + return + except (OSError, TypeError, ValueError, json.JSONDecodeError): + pass + try: + self._promote_latest_active_state() + except OSError: + return - def _write_run_state( + def _revision_artifact_signatures( self, - *, - state: RunState, - run_state_path: Path, - emit: Callable[..., None], - ) -> str: - run_state_path.parent.mkdir(parents=True, exist_ok=True) - dump_state(state, str(run_state_path)) - run_state_text = run_state_path.read_text(encoding="utf-8") - self.run_state_path().parent.mkdir(parents=True, exist_ok=True) - self._write_text(self.run_state_path(), run_state_text) - emit("state.save", run_id=state.run_id, path=str(run_state_path)) - emit("state.fingerprint.after_save", run_id=state.run_id, state_fingerprint=self._state_fingerprint(state)) - return run_state_text - - def _write_runtime_map_text( + indexed_states: Sequence[_IndexedState], + ) -> dict[str, dict[str, dict[str, object]]]: + return { + indexed.candidate.run_id: self._artifact_signatures(indexed.candidate.state_path.parent) + for indexed in indexed_states + } + + def _artifact_signatures(self, root: Path) -> dict[str, dict[str, object]]: + return { + artifact_name: self._path_lstat_signature(root / artifact_name) + for artifact_name in self._RUN_ALIAS_ARTIFACT_NAMES + } + + def _reconcile_run_aliases_unlocked( self, - *, - state: RunState, - runtime_map: dict[str, object], - runtime_map_path: Path, - emit_path: Path, - emit: Callable[..., None], - ) -> str: - runtime_map_text = self._json_text(runtime_map) - self._write_run_and_current_artifact( - run_path=runtime_map_path, - current_path=self.runtime_map_path(), - text=runtime_map_text, - ) - emit("runtime_map.write", path=str(emit_path)) - emit( - "runtime_map.fingerprint", - run_id=state.run_id, - runtime_map_fingerprint=self._text_fingerprint(runtime_map_text), - ) - return runtime_map_text + indexed_states: Sequence[_IndexedState], + ) -> None: + for indexed in indexed_states: + candidate = indexed.candidate + if self._run_aliases_match_manifest(candidate): + continue + source_dir = candidate.state_path.parent + run_dir = self.run_dir_path(candidate.run_id) + for artifact_name in self._RUN_ALIAS_ARTIFACT_NAMES: + source = source_dir / artifact_name + alias = run_dir / artifact_name + if not source.is_file() or source.is_symlink(): + alias.unlink(missing_ok=True) + continue + self._write_text(alias, source.read_text(encoding="utf-8")) + self._write_run_alias_manifest(candidate) + + def _remove_current_aliases(self) -> None: + roots = [self.runtime_root] + if self.compat_mode == self.COMPAT_READ_WRITE and self.runtime_legacy_root != self.runtime_root: + roots.append(self.runtime_legacy_root) + for root in roots: + for name in ( + "run_state.json", + "runtime_map.json", + "ports_manifest.json", + "error_report.json", + "events.jsonl", + "runtime_readiness_report.json", + "current_alias.json", + ): + (root / name).unlink(missing_ok=True) + self._remove_owned_mode_pointers(root) + + @staticmethod + def _remove_pointers_to_state(root: Path, state_path: Path) -> None: + expected = str(state_path.resolve()) + for pointer in root.glob(".last_state*"): + if not pointer.is_file(): + continue + try: + target = pointer.read_text(encoding="utf-8").splitlines()[0].strip() + except (IndexError, OSError, UnicodeError): + continue + try: + resolved_target = str(Path(target).expanduser().resolve()) + except (OSError, RuntimeError, ValueError): + continue + if resolved_target == expected: + pointer.unlink(missing_ok=True) + + def _load_legacy_shell_candidate(self, path: Path, *, allowed_root: str) -> RunState | None: + if not path.is_file(): + return None + try: + return load_legacy_shell_state(str(path), allowed_root=allowed_root) + except Exception: + return None @staticmethod def _json_text(payload: Mapping[str, object]) -> str: @@ -365,17 +1557,75 @@ def _events_text(events: list[dict[str, object]]) -> str: return "".join(json.dumps(event, sort_keys=True) + "\n" for event in events) @staticmethod - def _write_run_and_current_artifact(*, run_path: Path, current_path: Path, text: str) -> None: - run_path.parent.mkdir(parents=True, exist_ok=True) - current_path.parent.mkdir(parents=True, exist_ok=True) - RuntimeStateRepository._write_text(run_path, text) - RuntimeStateRepository._write_text(current_path, text) + def _write_text(path: Path, text: str) -> None: + scavenge_atomic_write_temps(path) + atomic_write_text(path, text) + + @contextmanager + def _repository_lock(self, *, exclusive: bool) -> Iterator[None]: + self._validate_runtime_roots() + with advisory_file_lock(self._lock_path, exclusive=exclusive): + self._validate_runtime_roots() + yield + + def _validate_runtime_roots(self) -> None: + self._runtime_root_identity = self._validate_directory_identity( + self.runtime_root, + expected=self._runtime_root_identity, + create=True, + label="runtime_root", + ) + if self.compat_mode == self.SCOPED_ONLY: + return + self._legacy_root_identity = self._validate_directory_identity( + self.runtime_legacy_root, + expected=self._legacy_root_identity, + create=self.compat_mode == self.COMPAT_READ_WRITE, + label="runtime_legacy_root", + ) + + @staticmethod + def _existing_directory_identity(path: Path) -> tuple[int, int] | None: + if not path.exists(): + return None + if path.is_symlink() or not path.is_dir(): + raise ValueError(f"runtime root is not a real directory: {path}") + metadata = path.stat() + return metadata.st_dev, metadata.st_ino @staticmethod - def _write_text(path: Path, text: str) -> None: - _ = path.write_text(text, encoding="utf-8") + def _validate_directory_identity( + path: Path, + *, + expected: tuple[int, int] | None, + create: bool, + label: str, + ) -> tuple[int, int] | None: + if path.is_symlink() or path.resolve(strict=False) != path: + raise RuntimeError(f"{label} changed identity or became a symlink: {path}") + if create: + durable_mkdir(path) + if not path.exists(): + return expected + if not path.is_dir(): + raise RuntimeError(f"{label} is not a directory: {path}") + metadata = path.stat() + current = metadata.st_dev, metadata.st_ino + if expected is not None and current != expected: + raise RuntimeError(f"{label} changed identity: {path}") + return current def purge(self, *, aggressive: bool = False) -> None: + with self._repository_lock(exclusive=True): + self._purge_unlocked(aggressive=aggressive) + + def _purge_unlocked(self, *, aggressive: bool) -> None: + self._retry_pending_revision_cleanups_unlocked() + self.run_index.purge() + self._write_text( + self._registry_marker_path, + self._json_text({"version": 1, "runtime_scope_id": self.runtime_scope_id}), + ) scoped_paths = ( self.run_state_path(), self.runtime_map_path(), @@ -383,15 +1633,18 @@ def purge(self, *, aggressive: bool = False) -> None: self.error_report_path(), self.runtime_root / "events.jsonl", self.runtime_root / "runtime_readiness_report.json", - self.runtime_root / ".last_state", - self.runtime_root / ".last_state.main", + self._alias_manifest_path, ) for path in scoped_paths: path.unlink(missing_ok=True) - for pointer in self.runtime_root.glob(".last_state.trees.*"): - pointer.unlink(missing_ok=True) + diagnostics_dir = self.runtime_root / "diagnostics" + if diagnostics_dir.is_symlink(): + diagnostics_dir.unlink(missing_ok=True) + elif diagnostics_dir.is_dir(): + shutil.rmtree(diagnostics_dir) + self._remove_owned_mode_pointers(self.runtime_root) - if self.compat_mode != self.SCOPED_ONLY: + if self.compat_mode == self.COMPAT_READ_WRITE: legacy_paths = ( self.runtime_legacy_root / "run_state.json", self.runtime_legacy_root / "runtime_map.json", @@ -399,44 +1652,35 @@ def purge(self, *, aggressive: bool = False) -> None: self.runtime_legacy_root / "error_report.json", self.runtime_legacy_root / "events.jsonl", self.runtime_legacy_root / "runtime_readiness_report.json", - self.runtime_legacy_root / ".last_state", - self.runtime_legacy_root / ".last_state.main", ) for path in legacy_paths: path.unlink(missing_ok=True) - for pointer in self.runtime_legacy_root.glob(".last_state.trees.*"): - pointer.unlink(missing_ok=True) + self._remove_owned_mode_pointers(self.runtime_legacy_root) if aggressive: - for runs_dir in (self.runtime_root / "runs", self.runtime_legacy_root / "runs"): + runs_dirs = [self.runtime_root / "runs"] + if self.compat_mode == self.COMPAT_READ_WRITE and self.runtime_legacy_root != self.runtime_root: + runs_dirs.append(self.runtime_legacy_root / "runs") + for runs_dir in runs_dirs: if runs_dir.is_dir(): - shutil.rmtree(runs_dir, ignore_errors=True) - - def _ordered_scoped_pointers(self, *, mode: str | None) -> list[Path]: - main_pointer = self.runtime_root / ".last_state.main" - tree_pointers = sorted(self.runtime_root.glob(".last_state.trees.*")) - generic_pointer = self.runtime_root / ".last_state" - if mode == "trees": - return [*tree_pointers, generic_pointer, main_pointer] - if mode == "main": - return [main_pointer, generic_pointer, *tree_pointers] - return [main_pointer, *tree_pointers, generic_pointer] - - def _ordered_legacy_pointers(self, *, mode: str | None) -> list[Path]: - main_pointer = self.runtime_legacy_root / ".last_state.main" - tree_pointers = sorted(self.runtime_legacy_root.glob(".last_state.trees.*")) - generic_pointer = self.runtime_legacy_root / ".last_state" - if mode == "trees": - return [*tree_pointers, generic_pointer, main_pointer] - if mode == "main": - return [main_pointer, generic_pointer, *tree_pointers] - return [main_pointer, *tree_pointers, generic_pointer] - - def _state_matches_scope(self, state: RunState) -> bool: - scope = state.metadata.get("repo_scope_id") - if isinstance(scope, str) and scope: - return scope == self.runtime_scope_id - return True + shutil.rmtree(runs_dir) + cleanup_dir = self.runtime_root / self._REVISION_CLEANUP_DIR_NAME + if cleanup_dir.is_symlink(): + cleanup_dir.unlink(missing_ok=True) + elif cleanup_dir.is_dir(): + shutil.rmtree(cleanup_dir) + self._scavenge_atomic_temps() + + def _scavenge_atomic_temps(self) -> None: + roots = {self.runtime_root} + if self.compat_mode == self.COMPAT_READ_WRITE: + roots.add(self.runtime_legacy_root) + for root in roots: + if not root.is_dir(): + continue + for temporary_path in root.rglob(".*.tmp"): + if temporary_path.is_file() and not temporary_path.is_symlink(): + temporary_path.unlink(missing_ok=True) def _context_ports_payload(self, context: object) -> dict[str, object]: ports = getattr(context, "ports", {}) @@ -457,39 +1701,6 @@ def _context_ports_payload(self, context: object) -> dict[str, object]: "ports": payload_ports, } - def _write_scoped_pointers(self, *, run_state_path: Path, state: RunState, contexts: list[object]) -> None: - self._write_mode_pointers( - root=self.runtime_root, - run_state_path=run_state_path, - mode=state.mode, - project_names=self._context_names(contexts), - ) - - def _write_legacy_compat( - self, - *, - run_state_path: Path, - run_state_text: str, - runtime_map_text: str, - ports_manifest_text: str, - error_report_text: str, - events_text: str, - state: RunState, - contexts: list[object], - ) -> None: - self.runtime_legacy_root.mkdir(parents=True, exist_ok=True) - self._write_text(self.runtime_legacy_root / "run_state.json", run_state_text) - self._write_text(self.runtime_legacy_root / "runtime_map.json", runtime_map_text) - self._write_text(self.runtime_legacy_root / "ports_manifest.json", ports_manifest_text) - self._write_text(self.runtime_legacy_root / "error_report.json", error_report_text) - self._write_text(self.runtime_legacy_root / "events.jsonl", events_text) - self._write_mode_pointers( - root=self.runtime_legacy_root, - run_state_path=run_state_path, - mode=state.mode, - project_names=self._context_names(contexts), - ) - @staticmethod def _context_names(contexts: list[object]) -> list[str]: names: set[str] = set() @@ -499,28 +1710,115 @@ def _context_names(contexts: list[object]) -> list[str]: names.add(name) return sorted(names, key=str.lower) - def _project_names_from_state(self, state: RunState) -> list[str]: + def _prepare_state_for_save( + self, + state: RunState, + *, + project_names: Sequence[str], + contexts: Sequence[object] = (), + authoritative_projects: bool = False, + ) -> None: + _ = self.run_dir_path(state.run_id) + if state.mode not in {"main", "trees"}: + raise ValueError("state mode must be 'main' or 'trees'") + existing_scope = state.metadata.get("repo_scope_id") + if isinstance(existing_scope, str) and existing_scope and existing_scope != self.runtime_scope_id: + raise ValueError("state repo_scope_id does not match this repository") + state.metadata["repo_scope_id"] = self.runtime_scope_id + + canonical_names = sorted( + {str(name).strip() for name in project_names if str(name).strip()}, + key=str.casefold, + ) + if canonical_names or authoritative_projects: + state.metadata["project_names"] = canonical_names + + existing_roots = state.metadata.get("project_roots") + project_roots = dict(existing_roots) if isinstance(existing_roots, Mapping) else {} + for context in contexts: + name = str(getattr(context, "name", "")).strip() + root = getattr(context, "root", None) + if name and root is not None and str(root).strip(): + project_roots[name] = str(Path(str(root)).expanduser().resolve(strict=False)) + if authoritative_projects: + normalized_names = normalized_project_names(canonical_names) + project_roots = { + str(name): root + for name, root in project_roots.items() + if str(name).strip().casefold() in normalized_names + } + if project_roots: + state.metadata["project_roots"] = project_roots + elif authoritative_projects: + state.metadata.pop("project_roots", None) + if canonical_names or authoritative_projects: + state.metadata = filter_project_scoped_metadata( + state.metadata, + canonical_names, + case_sensitive=False, + ) + state.metadata["project_names"] = canonical_names + + def _runtime_project_names_from_state(self, state: RunState) -> list[str]: names: set[str] = set() for project in state.requirements: normalized = str(project).strip() if normalized: names.add(normalized) - for service_name in state.services: - project_name = self._project_name_from_service_name(service_name) + for service_name, service in state.services.items(): + project_name = self._service_project_name(service_name, service) if project_name: names.add(project_name) + non_main_names = {name for name in names if name.casefold() != "main"} + if state.mode == "trees" and non_main_names: + names = non_main_names if state.mode == "main": names.add("Main") - return sorted(names, key=str.lower) + if names: + return sorted(names, key=str.casefold) + if bool(state.metadata.get("dashboard_runs_disabled")) or bool( + state.metadata.get(DASHBOARD_STOPPED_SERVICES_KEY) + ): + return self._project_names_from_state(state) + return [] @staticmethod - def _text_fingerprint(text: str) -> str: - return hashlib.sha256(text.encode("utf-8")).hexdigest() + def _source_run_ids(state: RunState) -> list[str]: + raw = state.metadata.get("state_source_run_ids") + values = raw if isinstance(raw, list) else [] + return [str(run_id).strip() for run_id in values if str(run_id).strip()] - @staticmethod - def _state_fingerprint(state: RunState) -> str: - serialized = json.dumps(state_to_dict(state), sort_keys=True) - return RuntimeStateRepository._text_fingerprint(serialized) + def _project_names_from_state(self, state: RunState) -> list[str]: + names: set[str] = set() + metadata_names = state.metadata.get("project_names") + if isinstance(metadata_names, Sequence) and not isinstance(metadata_names, (str, bytes)): + for project in metadata_names: + normalized = str(project).strip() + if normalized: + names.add(normalized) + metadata_roots = state.metadata.get("project_roots") + if isinstance(metadata_roots, Mapping): + for project in metadata_roots: + normalized = str(project).strip() + if normalized: + names.add(normalized) + if names: + return sorted(names, key=str.casefold) + for project in state.requirements: + normalized = str(project).strip() + if normalized: + names.add(normalized) + for service_name, service in state.services.items(): + project_name = str(getattr(service, "project", "") or "").strip() + if not project_name: + project_name = self._project_name_from_service_name(service_name) or "" + if project_name: + names.add(project_name) + if state.mode == "trees" and len(names) > 1: + names.discard("Main") + if state.mode == "main": + names.add("Main") + return sorted(names, key=str.lower) @staticmethod def _project_name_from_service_name(service_name: str) -> str | None: @@ -536,39 +1834,149 @@ def _project_name_from_service_name(service_name: str) -> str | None: return project_name or None return None - def _runtime_update_run_state_path(self, state: RunState) -> Path: - run_id = str(getattr(state, "run_id", "")).strip() - if not run_id: - return self.run_state_path() - return self.run_dir_path(run_id) / "run_state.json" - - def _write_mode_pointers( + def _service_project_name(self, service_name: str, service: ServiceRecord) -> str: + explicit_project = str(getattr(service, "project", "") or "").strip() + if explicit_project: + return explicit_project + return self._project_name_from_service_name(service_name) or "" + + def _contained_child(self, root: Path, component: str, *, label: str) -> Path: + runtime_root = self.runtime_root.resolve() + if root.is_symlink(): + raise ValueError(f"{label} root must not be a symlink") + resolved_root = root.resolve() + try: + resolved_root.relative_to(runtime_root) + except ValueError as exc: + raise ValueError(f"{label} root escapes runtime_root") from exc + candidate = root / component + if candidate.is_symlink(): + raise ValueError(f"{label} must not be a symlink") + if candidate.resolve().parent != resolved_root: + raise ValueError(f"{label} escapes its expected directory") + return candidate + + def _rebuild_mode_pointers( self, - *, root: Path, - run_state_path: Path, - mode: str, - project_names: list[str], + indexed_states: Sequence[_IndexedState], ) -> None: - root.mkdir(parents=True, exist_ok=True) - pointer_text = str(run_state_path) + "\n" - self._write_text(root / ".last_state", pointer_text) - existing_tree_pointers = sorted(root.glob(".last_state.trees.*")) - if mode == "main": - self._write_text(root / ".last_state.main", pointer_text) - return + desired = self._desired_mode_pointers(indexed_states) + stale_names = self._owned_pointer_names(root).difference(desired) + for name, target in desired.items(): + self._write_text(root / name, target + "\n") + for name in stale_names: + (root / name).unlink(missing_ok=True) + self._write_text( + self._pointer_manifest_path(root), + self._json_text( + { + "version": 1, + "runtime_scope_id": self.runtime_scope_id, + "pointers": desired, + } + ), + ) - expected_tree_pointer_names: set[str] = set() - for project_name in project_names: - pointer_name = self._tree_pointer_name(project_name) - if pointer_name is None: + def _desired_mode_pointers( + self, + indexed_states: Sequence[_IndexedState], + ) -> dict[str, str]: + if not indexed_states: + return {} + latest_active = max( + indexed_states, + key=lambda indexed: indexed.candidate.activation_sequence, + ) + desired = {".last_state": str(latest_active.candidate.state_path)} + owned_tree_projects: set[str] = set() + latest_main = max( + (indexed for indexed in indexed_states if indexed.candidate.mode == "main"), + key=lambda indexed: indexed.candidate.activation_sequence, + default=None, + ) + if latest_main is not None: + desired[".last_state.main"] = str(latest_main.candidate.state_path) + for indexed in indexed_states: + candidate = indexed.candidate + target = str(candidate.state_path) + if candidate.mode != "trees": continue - expected_tree_pointer_names.add(pointer_name) - self._write_text(root / pointer_name, pointer_text) + display_names = {name.casefold(): name for name in self._project_names_from_state(indexed.state)} + for project_key in candidate.project_names: + if project_key in owned_tree_projects: + continue + owned_tree_projects.add(project_key) + display_name = display_names.get(project_key, project_key) + pointer_name = self._tree_pointer_name(display_name) + if pointer_name is None: + continue + if pointer_name in desired and desired[pointer_name] != target: + digest = text_fingerprint(display_name)[:10] + pointer_name = f"{pointer_name}-{digest}" + desired[pointer_name] = target + return desired + + def _mode_pointers_match( + self, + root: Path, + indexed_states: Sequence[_IndexedState], + ) -> bool: + desired = self._desired_mode_pointers(indexed_states) + try: + payload = json.loads(self._pointer_manifest_path(root).read_text(encoding="utf-8")) + if not isinstance(payload, Mapping) or payload.get("pointers") != desired: + return False + if self._owned_pointer_names(root) != set(desired): + return False + return all((root / name).read_text(encoding="utf-8") == target + "\n" for name, target in desired.items()) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return False + + def _remove_owned_mode_pointers(self, root: Path) -> None: + for name in self._owned_pointer_names(root): + (root / name).unlink(missing_ok=True) + self._pointer_manifest_path(root).unlink(missing_ok=True) + + def _owned_pointer_names(self, root: Path) -> set[str]: + names: set[str] = set() + try: + payload = json.loads(self._pointer_manifest_path(root).read_text(encoding="utf-8")) + raw_pointers = payload.get("pointers") if isinstance(payload, Mapping) else None + if isinstance(raw_pointers, Mapping): + for raw_name in raw_pointers: + name = str(raw_name) + if name not in {".last_state", ".last_state.main"} and not name.startswith(".last_state.trees."): + continue + if root == self.runtime_root or self._pointer_targets_runtime_root(root / name): + names.add(name) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + pass + for pointer in root.glob(".last_state*"): + if pointer.name not in {".last_state", ".last_state.main"} and not pointer.name.startswith( + ".last_state.trees." + ): + continue + if root == self.runtime_root or self._pointer_targets_runtime_root(pointer): + names.add(pointer.name) + return names - for pointer in existing_tree_pointers: - if pointer.name not in expected_tree_pointer_names: - pointer.unlink(missing_ok=True) + def _pointer_targets_runtime_root(self, pointer: Path) -> bool: + if pointer.is_symlink() or not pointer.is_file(): + return False + try: + target = Path(pointer.read_text(encoding="utf-8").splitlines()[0].strip()).resolve() + target.relative_to(self.runtime_root) + except (IndexError, OSError, RuntimeError, ValueError): + return False + return True + + def _pointer_manifest_path(self, root: Path) -> Path: + if root == self.runtime_root: + return root / "pointer_manifest.json" + scope_stem = safe_artifact_stem(self.runtime_scope_id, fallback="scope") + scope_hash = text_fingerprint(self.runtime_scope_id)[:10] + return root / f".pointer_manifest.{scope_stem}-{scope_hash}.json" @staticmethod def _tree_pointer_name(project_name: str) -> str | None: @@ -576,4 +1984,7 @@ def _tree_pointer_name(project_name: str) -> str | None: if not normalized_project or normalized_project.lower() == "main": return None pointer_suffix = safe_artifact_stem(normalized_project, fallback="project") + if pointer_suffix != normalized_project: + digest = text_fingerprint(normalized_project)[:10] + pointer_suffix = f"{pointer_suffix}-{digest}" return f".last_state.trees.{pointer_suffix}" diff --git a/python/envctl_engine/state/run_index.py b/python/envctl_engine/state/run_index.py new file mode 100644 index 00000000..b5ae0e3f --- /dev/null +++ b/python/envctl_engine/state/run_index.py @@ -0,0 +1,582 @@ +from __future__ import annotations + +import json +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Any + +from envctl_engine.state.fingerprints import file_fingerprint +from envctl_engine.state.persistence import ( + advisory_file_lock, + atomic_write_text, + durable_mkdir, + require_path_component, + scavenge_atomic_write_temps, +) + + +_INDEX_VERSION = 4 + + +def _normalize_mode(mode: str | None) -> str | None: + normalized = str(mode or "").strip().casefold() + return normalized or None + + +def _normalize_project_names(project_names: Sequence[str]) -> tuple[str, ...]: + return tuple(sorted({name.strip().casefold() for name in project_names if name.strip()})) + + +@dataclass(frozen=True, slots=True) +class StateSelector: + mode: str | None + project_names: tuple[str, ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "mode", _normalize_mode(self.mode)) + object.__setattr__(self, "project_names", _normalize_project_names(self.project_names)) + + +@dataclass(frozen=True, slots=True) +class RunIndexCandidate: + state_path: Path + run_id: str + mode: str + project_names: tuple[str, ...] + sequence: int + state_fingerprint: str = "" + runtime_map_fingerprint: str | None = None + activation_sequence: int = 0 + + +@dataclass(frozen=True, slots=True) +class _RegistrySnapshot: + generation: int + entries: tuple[RunIndexCandidate, ...] + retired_run_ids: frozenset[str] + + +class RunIndex: + """Selection-aware registry of active project ownership by run artifact.""" + + def __init__(self, *, runtime_root: Path, runtime_dir: Path, runtime_scope_id: str) -> None: + self._input_runtime_root = runtime_root.absolute() + if self._input_runtime_root.is_symlink(): + raise ValueError("runtime_root must not be a symlink") + self.runtime_dir = runtime_dir.resolve() + self.runtime_root = runtime_root.resolve() + self.runtime_scope_id = runtime_scope_id.strip() + if not self.runtime_scope_id: + raise ValueError("runtime_scope_id must not be empty") + if not self._is_within_runtime_dir(self.runtime_root): + raise ValueError("runtime_root must be contained by runtime_dir") + self.index_path = self.runtime_root / "run_index.json" + self.backup_path = self.runtime_root / "run_index.backup.json" + self._lock_path = self.runtime_root / ".run_index.lock" + self._runtime_root_identity = self._existing_directory_identity() + + def record( + self, + *, + state_path: Path, + run_id: str, + mode: str, + project_names: Sequence[str], + supersede_run_ids: Sequence[str] = (), + ) -> None: + normalized_run_id = require_path_component(run_id, label="run_id") + resolved_state_path = self._validated_revision_state_path(state_path, run_id=normalized_run_id) + normalized_mode = _normalize_mode(mode) + if normalized_mode is None: + raise ValueError("mode must not be empty") + normalized_projects = _normalize_project_names(project_names) + state_fingerprint = file_fingerprint(resolved_state_path) + runtime_map_path = resolved_state_path.parent / "runtime_map.json" + runtime_map_fingerprint = ( + file_fingerprint(runtime_map_path) + if runtime_map_path.is_file() and not runtime_map_path.is_symlink() + else None + ) + superseded = { + require_path_component(candidate_run_id, label="superseded run_id") + for candidate_run_id in supersede_run_ids + } + superseded.discard(normalized_run_id) + + with self._lock(exclusive=True): + snapshot = self._read_registry() + entries = list(snapshot.entries) if snapshot is not None else [] + retired_run_ids = set(snapshot.retired_run_ids) if snapshot is not None else set() + if normalized_run_id in retired_run_ids: + raise RuntimeError(f"run_id is retired and cannot publish new state: {normalized_run_id}") + latest_sequence = max((entry.sequence for entry in entries), default=0) + latest_activation_sequence = max( + (entry.activation_sequence for entry in entries), + default=0, + ) + existing_entry = next((entry for entry in entries if entry.run_id == normalized_run_id), None) + if not normalized_projects and existing_entry is not None: + normalized_projects = existing_entry.project_names + entries = [ + entry for entry in entries if entry.run_id != normalized_run_id and entry.run_id not in superseded + ] + retired_run_ids.update(superseded) + entries = self._subtract_replaced_project_owners( + entries, + mode=normalized_mode, + project_names=normalized_projects, + retired_run_ids=retired_run_ids, + ) + sequence = existing_entry.sequence if existing_entry is not None else latest_sequence + 1 + activates_current_mode = existing_entry is None or self._entry_owns_any_project( + existing_entry, + entries, + normalized_projects, + ) + activation_sequence = ( + latest_activation_sequence + 1 if activates_current_mode else existing_entry.activation_sequence + ) + entries.append( + RunIndexCandidate( + state_path=resolved_state_path, + run_id=normalized_run_id, + mode=normalized_mode, + project_names=normalized_projects, + sequence=sequence, + state_fingerprint=state_fingerprint, + runtime_map_fingerprint=runtime_map_fingerprint, + activation_sequence=activation_sequence, + ) + ) + self._write_entries(entries, retired_run_ids=retired_run_ids) + + @staticmethod + def _subtract_replaced_project_owners( + entries: Sequence[RunIndexCandidate], + *, + mode: str, + project_names: Sequence[str], + retired_run_ids: set[str], + ) -> list[RunIndexCandidate]: + replaced_projects = frozenset(project_names) + if not replaced_projects: + return list(entries) + + retained: list[RunIndexCandidate] = [] + for entry in entries: + if entry.mode != mode or replaced_projects.isdisjoint(entry.project_names): + retained.append(entry) + continue + remaining_projects = tuple(project for project in entry.project_names if project not in replaced_projects) + if remaining_projects: + retained.append(replace(entry, project_names=remaining_projects)) + else: + retired_run_ids.add(entry.run_id) + return retained + + def remove(self, run_id: str) -> bool: + return self.remove_many([run_id]) + + def remove_many(self, run_ids: Sequence[str]) -> bool: + normalized_run_ids = {require_path_component(run_id, label="run_id") for run_id in run_ids} + with self._lock(exclusive=True): + snapshot = self._read_registry() + entries = list(snapshot.entries) if snapshot is not None else [] + retired_run_ids = set(snapshot.retired_run_ids) if snapshot is not None else set() + retained = [entry for entry in entries if entry.run_id not in normalized_run_ids] + removed_active_entry = len(retained) != len(entries) + added_tombstone = not normalized_run_ids.issubset(retired_run_ids) + if not removed_active_entry and not added_tombstone: + return False + retired_run_ids.update(normalized_run_ids) + self._write_entries(retained, retired_run_ids=retired_run_ids) + return removed_active_entry + + def candidates(self, selector: StateSelector) -> list[RunIndexCandidate]: + with self._lock(exclusive=False): + entries = self._read_entries() + if entries is None: + return [] + + selected_projects = frozenset(selector.project_names) + ranked: list[tuple[int, int, str, RunIndexCandidate]] = [] + for entry in entries: + if selector.mode is not None and entry.mode != selector.mode: + continue + relation_rank = self._selection_rank( + indexed=frozenset(entry.project_names), + selected=selected_projects, + ) + if relation_rank is None: + continue + ranked.append((-entry.sequence, relation_rank, str(entry.state_path), entry)) + ranked.sort(key=lambda item: (item[0], item[1], item[2])) + return [entry for _, _, _, entry in ranked] + + def candidate_paths(self, selector: StateSelector) -> list[Path]: + return [candidate.state_path for candidate in self.candidates(selector)] + + def purge(self) -> None: + with self._lock(exclusive=True): + snapshot = self._read_registry() + retired_run_ids = set(snapshot.retired_run_ids) if snapshot is not None else set() + if snapshot is not None: + retired_run_ids.update(entry.run_id for entry in snapshot.entries) + self._write_entries([], retired_run_ids=retired_run_ids) + + def needs_rebuild(self) -> bool: + with self._lock(exclusive=False): + return self._read_snapshot(self.index_path) is None and self._read_snapshot(self.backup_path) is None + + def repair_copies(self) -> None: + with self._lock(exclusive=True): + primary = self._read_snapshot(self.index_path) + backup = self._read_snapshot(self.backup_path) + if primary is not None and backup is not None and primary == backup: + return + snapshot = self._read_registry() + if snapshot is not None: + self._write_entries( + list(snapshot.entries), + retired_run_ids=set(snapshot.retired_run_ids), + ) + + def initialize_empty(self) -> None: + with self._lock(exclusive=True): + self._write_entries([], retired_run_ids=set()) + + def replace_all(self, entries: Sequence[RunIndexCandidate]) -> None: + with self._lock(exclusive=True): + self._write_entries( + [self._candidate_with_fingerprints(entry) for entry in entries], + retired_run_ids=set(), + ) + + @staticmethod + def _selection_rank(*, indexed: frozenset[str], selected: frozenset[str]) -> int | None: + if not selected: + return 0 + if not indexed: + return None + if indexed == selected: + return 0 + if indexed.issuperset(selected): + return 1 + if indexed.issubset(selected): + return 2 + if indexed.intersection(selected): + return 3 + return None + + @staticmethod + def _entry_owns_any_project( + existing_entry: RunIndexCandidate, + other_entries: Sequence[RunIndexCandidate], + updated_projects: Sequence[str], + ) -> bool: + if not updated_projects: + return True + for project in updated_projects: + if not any( + other.mode == existing_entry.mode + and other.sequence > existing_entry.sequence + and project in other.project_names + for other in other_entries + ): + return True + return False + + @contextmanager + def _lock(self, *, exclusive: bool) -> Iterator[None]: + self._validate_runtime_root() + with advisory_file_lock(self._lock_path, exclusive=exclusive): + self._validate_runtime_root() + yield + + def _existing_directory_identity(self) -> tuple[int, int] | None: + if not self.runtime_root.exists(): + return None + if self.runtime_root.is_symlink() or not self.runtime_root.is_dir(): + raise ValueError(f"runtime_root is not a real directory: {self.runtime_root}") + metadata = self.runtime_root.stat() + return metadata.st_dev, metadata.st_ino + + def _validate_runtime_root(self) -> None: + if self.runtime_root.is_symlink() or self.runtime_root.resolve(strict=False) != self.runtime_root: + raise RuntimeError(f"runtime_root changed identity or became a symlink: {self.runtime_root}") + durable_mkdir(self.runtime_root) + metadata = self.runtime_root.stat() + current = metadata.st_dev, metadata.st_ino + if self._runtime_root_identity is not None and current != self._runtime_root_identity: + raise RuntimeError(f"runtime_root changed identity: {self.runtime_root}") + self._runtime_root_identity = current + + def _read_entries(self) -> list[RunIndexCandidate] | None: + snapshot = self._read_registry() + return list(snapshot.entries) if snapshot is not None else None + + def _read_registry(self) -> _RegistrySnapshot | None: + primary = self._read_snapshot(self.index_path) + backup = self._read_snapshot(self.backup_path) + self._reject_equal_generation_divergence(primary, backup) + snapshots = [snapshot for snapshot in (primary, backup) if snapshot is not None] + if not snapshots: + return None + return max(snapshots, key=lambda snapshot: snapshot.generation) + + @staticmethod + def _reject_equal_generation_divergence( + primary: _RegistrySnapshot | None, + backup: _RegistrySnapshot | None, + ) -> None: + if primary is not None and backup is not None and primary.generation == backup.generation and primary != backup: + raise RuntimeError( + "Run registry primary and backup diverged at generation " + f"{primary.generation}; refusing to choose one copy" + ) + + def _read_snapshot(self, path: Path) -> _RegistrySnapshot | None: + if not path.is_file(): + return None + try: + payload = json.loads(path.read_text(encoding="utf-8")) + generation = payload.get("generation") if isinstance(payload, dict) else None + if not isinstance(generation, int) or isinstance(generation, bool) or generation < 1: + return None + entries = self._parse_payload(payload) + raw_retired = payload.get("retired_run_ids") + if not isinstance(raw_retired, list) or not all(isinstance(run_id, str) for run_id in raw_retired): + return None + retired = frozenset(require_path_component(run_id, label="retired run_id") for run_id in raw_retired) + if retired.intersection(entry.run_id for entry in entries): + return None + return _RegistrySnapshot(generation, tuple(entries), retired) + except (OSError, RuntimeError, TypeError, ValueError, json.JSONDecodeError): + return None + + def _parse_payload(self, payload: Any) -> list[RunIndexCandidate]: + if not isinstance(payload, dict): + raise ValueError("run index must be a JSON object") + version = payload.get("version") + if version not in {2, 3, _INDEX_VERSION}: + raise ValueError("unsupported run index version") + if payload.get("runtime_scope_id") != self.runtime_scope_id: + raise ValueError("run index scope does not match this repository") + raw_entries = payload.get("entries") + if not isinstance(raw_entries, list): + raise ValueError("run index entries must be a list") + + entries: list[RunIndexCandidate] = [] + seen_run_ids: set[str] = set() + for raw_entry in raw_entries: + if not isinstance(raw_entry, dict) or raw_entry.get("runtime_scope_id") != self.runtime_scope_id: + raise ValueError("invalid run index entry scope") + run_id = raw_entry.get("run_id") + mode = _normalize_mode(raw_entry.get("mode") if isinstance(raw_entry.get("mode"), str) else None) + raw_projects = raw_entry.get("project_names") + sequence = raw_entry.get("sequence") + raw_state_path = raw_entry.get("state_path") + raw_state_fingerprint = raw_entry.get("state_fingerprint") + raw_runtime_map_fingerprint = raw_entry.get("runtime_map_fingerprint") + raw_activation_sequence = raw_entry.get("activation_sequence") + if ( + not isinstance(run_id, str) + or not run_id.strip() + or mode is None + or not isinstance(raw_projects, list) + or not all(isinstance(project, str) for project in raw_projects) + or not isinstance(sequence, int) + or isinstance(sequence, bool) + or sequence < 1 + or not isinstance(raw_state_path, str) + or not raw_state_path.strip() + ): + raise ValueError("invalid run index entry") + normalized_run_id = require_path_component(run_id, label="indexed run_id") + if normalized_run_id in seen_run_ids: + raise ValueError("duplicate run id in run index") + state_path = self._validated_revision_state_path(Path(raw_state_path), run_id=normalized_run_id) + if version == 2: + state_fingerprint = file_fingerprint(state_path) + runtime_map_path = state_path.parent / "runtime_map.json" + runtime_map_fingerprint = ( + file_fingerprint(runtime_map_path) + if runtime_map_path.is_file() and not runtime_map_path.is_symlink() + else None + ) + else: + if not self._valid_fingerprint(raw_state_fingerprint) or ( + raw_runtime_map_fingerprint is not None and not self._valid_fingerprint(raw_runtime_map_fingerprint) + ): + raise ValueError("invalid run index artifact fingerprint") + state_fingerprint = str(raw_state_fingerprint) + runtime_map_fingerprint = ( + str(raw_runtime_map_fingerprint) if raw_runtime_map_fingerprint is not None else None + ) + if version in {2, 3}: + activation_sequence = sequence + elif ( + not isinstance(raw_activation_sequence, int) + or isinstance(raw_activation_sequence, bool) + or raw_activation_sequence < 1 + ): + raise ValueError("invalid run index activation sequence") + else: + activation_sequence = raw_activation_sequence + seen_run_ids.add(normalized_run_id) + entries.append( + RunIndexCandidate( + state_path=state_path, + run_id=normalized_run_id, + mode=mode, + project_names=_normalize_project_names(raw_projects), + sequence=sequence, + state_fingerprint=state_fingerprint, + runtime_map_fingerprint=runtime_map_fingerprint, + activation_sequence=activation_sequence, + ) + ) + return entries + + def _write_entries( + self, + entries: list[RunIndexCandidate], + *, + retired_run_ids: set[str], + ) -> None: + primary = self._read_snapshot(self.index_path) + backup = self._read_snapshot(self.backup_path) + self._reject_equal_generation_divergence(primary, backup) + existing_generations = [snapshot.generation for snapshot in (primary, backup) if snapshot is not None] + payload = { + "version": _INDEX_VERSION, + "generation": max(existing_generations, default=0) + 1, + "runtime_scope_id": self.runtime_scope_id, + "retired_run_ids": sorted(retired_run_ids), + "entries": [ + { + "state_path": str(entry.state_path), + "run_id": entry.run_id, + "mode": entry.mode, + "project_names": list(entry.project_names), + "sequence": entry.sequence, + "state_fingerprint": entry.state_fingerprint, + "runtime_map_fingerprint": entry.runtime_map_fingerprint, + "activation_sequence": entry.activation_sequence, + "runtime_scope_id": self.runtime_scope_id, + } + for entry in entries + ], + } + text = json.dumps(payload, indent=2, sort_keys=True) + "\n" + errors: list[OSError] = [] + successful_writes = 0 + for path in (self.backup_path, self.index_path): + try: + scavenge_atomic_write_temps(path) + atomic_write_text(path, text) + successful_writes += 1 + except OSError as exc: + errors.append(exc) + expected = _RegistrySnapshot( + generation=payload["generation"], + entries=tuple(entries), + retired_run_ids=frozenset(retired_run_ids), + ) + if any(self._read_snapshot(path) == expected for path in (self.index_path, self.backup_path)): + return + if errors: + raise errors[0] + if successful_writes: + raise RuntimeError("run registry write returned without publishing the expected generation") + + def _validated_revision_state_path(self, state_path: Path, *, run_id: str) -> Path: + if not state_path.is_absolute(): + raise ValueError("indexed state_path must be absolute") + resolved = state_path.resolve() + try: + relative = resolved.relative_to(self.runtime_root) + except ValueError as exc: + raise ValueError("indexed state_path escapes runtime_root") from exc + parts = relative.parts + if len(parts) != 5 or parts[0] != "runs" or parts[1] != run_id or parts[2] != "revisions": + raise ValueError("indexed state_path must identify a run revision") + _ = require_path_component(parts[3], label="revision id") + if parts[4] != "run_state.json": + raise ValueError("indexed state_path must identify run_state.json") + if ".." in state_path.parts or state_path.is_symlink(): + raise ValueError("indexed state_path must be canonical and must not traverse symlinks") + lexical_relative: Path | None = None + lexical_root: Path | None = None + for candidate_root in (self._input_runtime_root, self.runtime_root): + try: + lexical_relative = state_path.relative_to(candidate_root) + lexical_root = candidate_root + break + except ValueError: + continue + if lexical_relative is None or lexical_root is None: + raise ValueError("indexed state_path must be canonical and must not traverse symlinks") + lexical_parent = lexical_root + for component in lexical_relative.parts[:-1]: + lexical_parent /= component + if lexical_parent.is_symlink(): + raise ValueError("indexed state_path must be canonical and must not traverse symlinks") + for parent in ( + self.runtime_root / "runs", + self.runtime_root / "runs" / run_id, + self.runtime_root / "runs" / run_id / "revisions", + resolved.parent, + ): + if parent.is_symlink(): + raise ValueError("indexed state_path must not traverse symlinks") + if not self._is_within_runtime_root(resolved): + raise ValueError("indexed state_path must be canonical inside runtime_root") + return resolved + + def _candidate_with_fingerprints(self, entry: RunIndexCandidate) -> RunIndexCandidate: + state_path = self._validated_revision_state_path(entry.state_path, run_id=entry.run_id) + runtime_map_path = state_path.parent / "runtime_map.json" + return RunIndexCandidate( + state_path=state_path, + run_id=entry.run_id, + mode=entry.mode, + project_names=entry.project_names, + sequence=entry.sequence, + state_fingerprint=entry.state_fingerprint or file_fingerprint(state_path), + runtime_map_fingerprint=( + entry.runtime_map_fingerprint + if entry.runtime_map_fingerprint is not None + else ( + file_fingerprint(runtime_map_path) + if runtime_map_path.is_file() and not runtime_map_path.is_symlink() + else None + ) + ), + activation_sequence=entry.activation_sequence or entry.sequence, + ) + + @staticmethod + def _valid_fingerprint(value: object) -> bool: + return ( + isinstance(value, str) and len(value) == 64 and all(character in "0123456789abcdef" for character in value) + ) + + def _is_within_runtime_dir(self, path: Path) -> bool: + try: + path.relative_to(self.runtime_dir) + except ValueError: + return False + return True + + def _is_within_runtime_root(self, path: Path) -> bool: + try: + path.relative_to(self.runtime_root) + except ValueError: + return False + return True + + +__all__ = ["RunIndex", "RunIndexCandidate", "StateSelector"] diff --git a/python/envctl_engine/state/state_aggregation.py b/python/envctl_engine/state/state_aggregation.py new file mode 100644 index 00000000..bf63e990 --- /dev/null +++ b/python/envctl_engine/state/state_aggregation.py @@ -0,0 +1,322 @@ +from __future__ import annotations + +import json +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass + +from envctl_engine.dashboard_metadata import DASHBOARD_STOPPED_SERVICES_KEY +from envctl_engine.state.fingerprints import text_fingerprint +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord +from envctl_engine.state.project_runtime import ( + PROJECT_SCOPED_METADATA_MAP_KEYS, + filter_project_scoped_metadata, +) +from envctl_engine.state.run_index import RunIndexCandidate + + +ServiceProjectResolver = Callable[[str, ServiceRecord], str] +ProjectNamesResolver = Callable[[RunState], Sequence[str]] +SourceRunIdsResolver = Callable[[RunState], Sequence[str]] + + +@dataclass(frozen=True, slots=True) +class IndexedState: + candidate: RunIndexCandidate + state: RunState + + +def normalized_project_names(project_names: Sequence[str]) -> frozenset[str]: + return frozenset(str(name).strip().casefold() for name in project_names if str(name).strip()) + + +def select_indexed_owners( + indexed_states: Sequence[IndexedState], + *, + selected_projects: Sequence[str], + service_project_name: ServiceProjectResolver, +) -> list[IndexedState]: + if not indexed_states: + return [] + owner_by_project: dict[str, str] = {} + for indexed_state in indexed_states: + for project_name in indexed_state.candidate.project_names: + owner_by_project.setdefault(project_name, indexed_state.candidate.run_id) + if not owner_by_project and not normalized_project_names(selected_projects): + return [indexed_states[0]] + + requested = normalized_project_names(selected_projects) or frozenset(owner_by_project) + chosen_run_ids = {owner_by_project[project_name] for project_name in requested if project_name in owner_by_project} + if not chosen_run_ids: + return [] + + chosen_states: list[IndexedState] = [] + for indexed_state in indexed_states: + run_id = indexed_state.candidate.run_id + if run_id not in chosen_run_ids: + continue + owned_projects = frozenset( + project_name for project_name, owner_run_id in owner_by_project.items() if owner_run_id == run_id + ) + filtered = filter_state_to_owned_projects( + indexed_state.state, + owned_projects, + service_project_name=service_project_name, + ) + chosen_states.append(IndexedState(candidate=indexed_state.candidate, state=filtered)) + return chosen_states + + +def state_from_indexed_owners( + chosen_states: Sequence[IndexedState], + *, + project_names_from_state: ProjectNamesResolver, + source_run_ids: SourceRunIdsResolver, +) -> RunState | None: + if not chosen_states: + return None + if len(chosen_states) == 1: + chosen = chosen_states[0] + if normalized_project_names(chosen.candidate.project_names) == normalized_project_names( + project_names_from_state(chosen.state) + ): + return chosen.state + return merge_indexed_states( + chosen_states, + project_names_from_state=project_names_from_state, + source_run_ids=source_run_ids, + ) + + +def filter_state_to_owned_projects( + state: RunState, + indexed_projects: frozenset[str], + *, + service_project_name: ServiceProjectResolver, +) -> RunState: + if not indexed_projects: + return state + + services = { + name: service + for name, service in state.services.items() + if service_project_name(name, service).casefold() in indexed_projects + } + requirements = { + name: requirement + for name, requirement in state.requirements.items() + if str(name).strip().casefold() in indexed_projects + or ( + state.mode == "trees" + and str(name).strip().casefold() == "main" + and bool(state.metadata.get("shared_dependencies")) + ) + } + metadata = filter_project_scoped_metadata( + state.metadata, + indexed_projects, + case_sensitive=False, + ) + metadata["project_names"] = _display_project_names( + state, + indexed_projects=indexed_projects, + service_project_name=service_project_name, + ) + filter_startup_identity_projects(metadata, indexed_projects) + return RunState( + run_id=state.run_id, + mode=state.mode, + schema_version=state.schema_version, + backend_mode=state.backend_mode, + services=services, + requirements=requirements, + pointers=dict(state.pointers), + metadata=metadata, + ) + + +def _display_project_names( + state: RunState, + *, + indexed_projects: frozenset[str], + service_project_name: ServiceProjectResolver, +) -> list[str]: + display_by_key: dict[str, str] = {} + + def add(raw_name: object) -> None: + name = str(raw_name or "").strip() + key = name.casefold() + if name and key in indexed_projects: + display_by_key.setdefault(key, name) + + metadata_names = state.metadata.get("project_names") + if isinstance(metadata_names, Sequence) and not isinstance(metadata_names, (str, bytes)): + for project in metadata_names: + add(project) + metadata_roots = state.metadata.get("project_roots") + if isinstance(metadata_roots, Mapping): + for project in metadata_roots: + add(project) + for project, requirements in state.requirements.items(): + add(getattr(requirements, "project", "") or project) + for service_name, service in state.services.items(): + add(service_project_name(service_name, service)) + return [display_by_key.get(key, key) for key in sorted(indexed_projects)] + + +def merge_indexed_states( + indexed_states: Sequence[IndexedState], + *, + project_names_from_state: ProjectNamesResolver, + source_run_ids: SourceRunIdsResolver, +) -> RunState: + ordered = sorted(indexed_states, key=lambda item: item.candidate.sequence) + latest = max( + indexed_states, + key=lambda item: ( + item.candidate.activation_sequence, + item.candidate.sequence, + str(item.candidate.state_path), + ), + ).state + services: dict[str, ServiceRecord] = {} + requirements: dict[str, RequirementsResult] = {} + pointers: dict[str, str] = {} + project_roots: dict[str, object] = {} + project_names: dict[str, str] = {} + for indexed_state in ordered: + state = indexed_state.state + services.update(state.services) + requirements.update(state.requirements) + pointers.update(state.pointers) + for name in project_names_from_state(state): + project_names.setdefault(name.casefold(), name) + roots = state.metadata.get("project_roots") + if isinstance(roots, Mapping): + project_roots.update({str(name): root for name, root in roots.items()}) + + selected_project_names = [project_names[key] for key in sorted(project_names)] + metadata = merge_project_scoped_metadata( + latest.metadata, + ordered, + project_names=selected_project_names, + ) + metadata["project_names"] = selected_project_names + all_source_run_ids = { + source_run_id + for indexed_state in ordered + for source_run_id in ( + indexed_state.candidate.run_id, + *source_run_ids(indexed_state.state), + ) + } + metadata["state_source_run_ids"] = sorted(all_source_run_ids) + if project_roots: + metadata["project_roots"] = project_roots + merge_startup_identities(metadata, ordered, project_roots=project_roots) + return RunState( + run_id=latest.run_id, + mode=latest.mode, + schema_version=latest.schema_version, + backend_mode=latest.backend_mode, + services=services, + requirements=requirements, + pointers=pointers, + metadata=metadata, + ) + + +def merge_project_scoped_metadata( + latest_metadata: Mapping[str, object], + indexed_states: Sequence[IndexedState], + *, + project_names: Sequence[str], +) -> dict[str, object]: + metadata = dict(latest_metadata) + for key in PROJECT_SCOPED_METADATA_MAP_KEYS: + merged: dict[str, object] = {} + for indexed_state in indexed_states: + raw = indexed_state.state.metadata.get(key) + if isinstance(raw, Mapping): + merged.update({str(project): value for project, value in raw.items()}) + if merged: + metadata[key] = merged + else: + metadata.pop(key, None) + + stopped_by_identity: dict[tuple[str, str, str], dict[str, object]] = {} + for indexed_state in indexed_states: + raw_stopped = indexed_state.state.metadata.get(DASHBOARD_STOPPED_SERVICES_KEY) + if not isinstance(raw_stopped, list): + continue + for item in raw_stopped: + if not isinstance(item, Mapping): + continue + project = str(item.get("project", "")).strip() + name = str(item.get("name", "")).strip() + service_type = str(item.get("type", "")).strip().lower() + if not project or not name or not service_type: + continue + stopped_by_identity[(project.casefold(), name.casefold(), service_type)] = dict(item) + if stopped_by_identity: + metadata[DASHBOARD_STOPPED_SERVICES_KEY] = [stopped_by_identity[key] for key in sorted(stopped_by_identity)] + else: + metadata.pop(DASHBOARD_STOPPED_SERVICES_KEY, None) + + if len(indexed_states) > 1: + metadata.pop("project_test_results_root", None) + metadata.pop("project_test_results_updated_at", None) + return filter_project_scoped_metadata( + metadata, + list(project_names), + case_sensitive=False, + ) + + +def filter_startup_identity_projects( + metadata: dict[str, object], + project_names: frozenset[str], +) -> None: + raw_identity = metadata.get("startup_identity") + if not isinstance(raw_identity, Mapping): + return + identity = dict(raw_identity) + raw_projects = identity.get("projects") + projects = raw_projects if isinstance(raw_projects, list) else [] + identity["projects"] = [ + project + for project in projects + if isinstance(project, Mapping) and str(project.get("name", "")).strip().casefold() in project_names + ] + identity.pop("fingerprint", None) + identity["fingerprint"] = text_fingerprint(json.dumps(identity, sort_keys=True)) + metadata["startup_identity"] = identity + + +def merge_startup_identities( + metadata: dict[str, object], + indexed_states: Sequence[IndexedState], + *, + project_roots: Mapping[str, object], +) -> None: + identities = [ + dict(identity) + for indexed_state in indexed_states + if isinstance((identity := indexed_state.state.metadata.get("startup_identity")), Mapping) + ] + if not identities: + metadata.pop("startup_identity", None) + return + comparable = [ + {key: value for key, value in identity.items() if key not in {"fingerprint", "projects"}} + for identity in identities + ] + if any(payload != comparable[0] for payload in comparable[1:]): + metadata.pop("startup_identity", None) + return + identity = dict(comparable[0]) + identity["projects"] = [ + {"name": name, "root": str(root)} + for name, root in sorted(project_roots.items(), key=lambda item: item[0].casefold()) + ] + identity["fingerprint"] = text_fingerprint(json.dumps(identity, sort_keys=True)) + metadata["startup_identity"] = identity diff --git a/tests/python/actions/commit_action_test_support.py b/tests/python/actions/commit_action_test_support.py index 2efcadbf..24229bd3 100644 --- a/tests/python/actions/commit_action_test_support.py +++ b/tests/python/actions/commit_action_test_support.py @@ -57,7 +57,7 @@ def run_git(self, _git_root: Path, args: list[str]) -> subprocess.CompletedProce return subprocess.CompletedProcess(args=args, returncode=0, stdout=self._next_pre_stage_status(), stderr="") if args[:3] == ["reset", "-q", "--"]: return subprocess.CompletedProcess(args=args, returncode=self.reset_returncode, stdout="", stderr=self.reset_stderr) - if args[:2] == ["add", "--"]: + if args[:3] == ["add", "--all", "--"]: return subprocess.CompletedProcess(args=args, returncode=0, stdout="", stderr="") if args == ["status", "--porcelain"]: return subprocess.CompletedProcess(args=args, returncode=0, stdout=self.staged_status, stderr="") diff --git a/tests/python/actions/ship_workflow_test_support.py b/tests/python/actions/ship_workflow_test_support.py index 16dca634..4d724317 100644 --- a/tests/python/actions/ship_workflow_test_support.py +++ b/tests/python/actions/ship_workflow_test_support.py @@ -11,6 +11,7 @@ from types import SimpleNamespace from typing import Any +from envctl_engine.actions.action_git_state_support import ExistingPullRequest from envctl_engine.actions.action_ship_support import run_ship_workflow @@ -44,6 +45,7 @@ def run( add_ship_pr_label: Callable[[object, Path, str], int] | None = None, probe_dirty_worktree: Callable[..., object] | None = None, existing_pr_url: Callable[[Path, str], str] | None = None, + existing_pull_request: Callable[[Path, str], ExistingPullRequest | None] | None = None, partition_envctl_protected_paths: Callable[[str], object] | None = None, ordered_unique_paths: Callable[..., list[str]] | None = None, github_pr_checks: Callable[..., dict[str, object]] | None = None, @@ -64,6 +66,7 @@ def run( add_ship_pr_label=add_ship_pr_label or self.add_ship_pr_label, probe_dirty_worktree=probe_dirty_worktree or self.probe_dirty_worktree, existing_pr_url=existing_pr_url or self.existing_pr_url, + existing_pull_request=existing_pull_request, partition_envctl_protected_paths=( partition_envctl_protected_paths or self.partition_envctl_protected_paths ), diff --git a/tests/python/actions/test_action_commit_support.py b/tests/python/actions/test_action_commit_support.py index ae63d96f..d7e483c7 100644 --- a/tests/python/actions/test_action_commit_support.py +++ b/tests/python/actions/test_action_commit_support.py @@ -48,11 +48,99 @@ def _run_git(_git_root: Path, args: list[str]) -> subprocess.CompletedProcess[st ) self.assertEqual(code, 0) - self.assertIn(["add", "--", "app.py"], seen) + self.assertIn(["add", "--all", "--", "app.py"], seen) self.assertTrue(any(args[:2] == ["commit", "-F"] for args in seen)) self.assertIn(["push", "-u", "origin", "feature-branch"], seen) self.assertEqual(ledger.read_text(encoding="utf-8"), "Old summary\n\nNew summary\n\n### Envctl pointer ###\n") + def test_stage_paths_records_unstaged_tracked_deletions(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + git_root = Path(temp_dir) + + def run_git(_git_root: Path, args: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=git_root, + text=True, + capture_output=True, + check=False, + ) + + for args in ( + ["init"], + ["config", "user.email", "envctl-tests@example.invalid"], + ["config", "user.name", "Envctl Tests"], + ): + self.assertEqual(run_git(git_root, args).returncode, 0) + obsolete = git_root / "obsolete.py" + obsolete.write_text("obsolete = True\n", encoding="utf-8") + self.assertEqual(run_git(git_root, ["add", "obsolete.py"]).returncode, 0) + self.assertEqual(run_git(git_root, ["commit", "-m", "initial"]).returncode, 0) + obsolete.unlink() + + runner = action_commit_support.CommitWorkflowRunner( + context=SimpleNamespace(), + dependencies=action_commit_support.CommitWorkflowDependencies( + resolve_git_root=lambda project_root, _repo_root: project_root, + git_available=True, + git_output=lambda _git_root, _args: "", + run_git=run_git, + print_error=lambda _prefix, _result: None, + partition_envctl_protected_paths=partition_envctl_protected_paths, + ordered_unique_paths=lambda *groups: [item for group in groups for item in group], + ), + ) + + self.assertTrue(runner._stage_paths(git_root, ["obsolete.py"])) + staged = run_git(git_root, ["diff", "--cached", "--name-status"]) + self.assertEqual(staged.returncode, 0) + self.assertEqual(staged.stdout.strip(), "D\tobsolete.py") + + def test_stage_candidates_preserves_an_already_staged_deletion(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + git_root = Path(temp_dir) + + def run_git(_git_root: Path, args: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=git_root, + text=True, + capture_output=True, + check=False, + ) + + for args in ( + ["init"], + ["config", "user.email", "envctl-tests@example.invalid"], + ["config", "user.name", "Envctl Tests"], + ): + self.assertEqual(run_git(git_root, args).returncode, 0) + obsolete = git_root / "obsolete.py" + obsolete.write_text("obsolete = True\n", encoding="utf-8") + self.assertEqual(run_git(git_root, ["add", "obsolete.py"]).returncode, 0) + self.assertEqual(run_git(git_root, ["commit", "-m", "initial"]).returncode, 0) + obsolete.unlink() + self.assertEqual(run_git(git_root, ["add", "--all"]).returncode, 0) + + runner = action_commit_support.CommitWorkflowRunner( + context=SimpleNamespace(), + dependencies=action_commit_support.CommitWorkflowDependencies( + resolve_git_root=lambda project_root, _repo_root: project_root, + git_available=True, + git_output=lambda _git_root, _args: "", + run_git=run_git, + print_error=lambda _prefix, _result: None, + partition_envctl_protected_paths=partition_envctl_protected_paths, + ordered_unique_paths=lambda *groups: [item for group in groups for item in group], + ), + ) + + partition = runner._stage_commit_candidates(git_root) + self.assertIsNotNone(partition) + staged = run_git(git_root, ["diff", "--cached", "--name-status"]) + self.assertEqual(staged.returncode, 0) + self.assertEqual(staged.stdout.strip(), "D\tobsolete.py") + def test_read_commit_ledger_segment_reports_empty_pointer(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: ledger = Path(temp_dir) / ".envctl-commit-message.md" diff --git a/tests/python/actions/test_action_git_state_support.py b/tests/python/actions/test_action_git_state_support.py index 30acf4c4..35925306 100644 --- a/tests/python/actions/test_action_git_state_support.py +++ b/tests/python/actions/test_action_git_state_support.py @@ -25,23 +25,85 @@ def fake_git_output(_git_root: Path, args: list[str]) -> str: self.assertTrue(report.untracked) self.assertTrue(report.dirty) - def test_detect_default_branch_prefers_origin_head_then_local_candidates(self) -> None: + def test_detect_default_branch_prefers_dev_then_main_then_master(self) -> None: calls: list[list[str]] = [] - def origin_head(_git_root: Path, args: list[str]) -> str: + def dev_exists(_git_root: Path, args: list[str]) -> str: calls.append(args) - return "origin/develop\n" + return "abc123\n" if args == ["rev-parse", "--verify", "refs/remotes/origin/dev"] else "" - self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=origin_head), "develop") - self.assertEqual(calls, [["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]]) + self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=dev_exists), "dev") + self.assertEqual(calls[0], ["rev-parse", "--verify", "refs/remotes/origin/dev"]) + self.assertTrue(any(args[:3] == ["ls-remote", "--symref", "origin"] for args in calls)) def local_main(_git_root: Path, args: list[str]) -> str: - if args == ["rev-parse", "--verify", "main"]: + if args == ["rev-parse", "--verify", "refs/heads/main"]: return "abc123\n" return "" self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=local_main), "main") + def remote_master(_git_root: Path, args: list[str]) -> str: + if args == ["rev-parse", "--verify", "refs/remotes/origin/master"]: + return "abc123\n" + return "" + + self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=remote_master), "master") + + def test_detect_default_branch_uses_remote_head_only_after_preferred_branches_are_absent(self) -> None: + def remote_head(_git_root: Path, args: list[str]) -> str: + if args == ["symbolic-ref", "--short", "refs/remotes/origin/HEAD"]: + return "origin/develop\n" + return "" + + self.assertEqual(git_state.detect_default_branch(Path("/repo"), git_output=remote_head), "develop") + + def test_detect_default_branch_discovers_dev_omitted_by_local_fetch_refspec(self) -> None: + def single_branch_clone(_git_root: Path, args: list[str]) -> str: + if args == ["rev-parse", "--verify", "refs/remotes/origin/main"]: + return "abc123\n" + if args[:3] == ["ls-remote", "--symref", "origin"]: + return "ref: refs/heads/main\tHEAD\nmainsha\tHEAD\ndevsha\trefs/heads/dev\nmainsha\trefs/heads/main\n" + return "" + + self.assertEqual( + git_state.detect_default_branch(Path("/repo"), git_output=single_branch_clone), + "dev", + ) + + def test_live_remote_inventory_overrides_stale_deleted_dev_tracking_ref(self) -> None: + def stale_dev(_git_root: Path, args: list[str]) -> str: + if args == ["rev-parse", "--verify", "refs/remotes/origin/dev"]: + return "stale-dev-sha\n" + if args == ["rev-parse", "--verify", "refs/remotes/origin/main"]: + return "main-sha\n" + if args[:3] == ["ls-remote", "--symref", "origin"]: + return ( + "ref: refs/heads/main\tHEAD\n" + "main-sha\tHEAD\n" + "main-sha\trefs/heads/main\n" + "master-sha\trefs/heads/master\n" + ) + return "" + + self.assertEqual( + git_state.detect_default_branch(Path("/repo"), git_output=stale_dev), + "main", + ) + + def test_live_remote_default_overrides_stale_dev_when_no_preferred_head_exists(self) -> None: + def no_preferred_remote(_git_root: Path, args: list[str]) -> str: + if args == ["rev-parse", "--verify", "refs/remotes/origin/dev"]: + return "stale-dev-sha\n" + if args[:3] == ["ls-remote", "--symref", "origin"]: + return "ref: refs/heads/feature/alpha\tHEAD\nfeature-sha\tHEAD\n" + return "" + + self.assertEqual( + git_state.detect_default_branch(Path("/repo"), git_output=no_preferred_remote), + "feature/alpha", + ) + def test_existing_pr_url_uses_gh_pr_list_for_open_branch(self) -> None: calls: list[tuple[list[str], str]] = [] @@ -57,7 +119,12 @@ def fake_run( self.assertTrue(capture_output) self.assertFalse(check) calls.append((args, cwd)) - return subprocess.CompletedProcess(args, 0, stdout="https://github.com/acme/repo/pull/9\n", stderr="") + return subprocess.CompletedProcess( + args, + 0, + stdout='[{"url":"https://github.com/acme/repo/pull/9","baseRefName":"dev"}]\n', + stderr="", + ) url = git_state.existing_pr_url(Path("/repo"), "feature", gh_path="/usr/bin/gh", run_process=fake_run) @@ -66,17 +133,95 @@ def fake_run( self.assertEqual(calls[0][0][:4], ["/usr/bin/gh", "pr", "list", "--head"]) self.assertIn("feature", calls[0][0]) + def test_existing_pull_request_preserves_base_metadata(self) -> None: + def fake_run(args: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + args, + 0, + stdout='[{"url":"https://github.com/acme/repo/pull/9","baseRefName":"release"}]', + stderr="", + ) + + existing = git_state.existing_pull_request( + Path("/repo"), + "feature", + gh_path="/usr/bin/gh", + run_process=fake_run, + ) + + self.assertEqual( + existing, + git_state.ExistingPullRequest( + url="https://github.com/acme/repo/pull/9", + base_branch="release", + ), + ) + + def test_update_pull_request_base_uses_pr_url_and_reports_missing_gh(self) -> None: + existing = git_state.ExistingPullRequest(url="https://github.com/acme/repo/pull/9", base_branch="main") + calls: list[list[str]] = [] + + def fake_run(args: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + calls.append(args) + return subprocess.CompletedProcess(args, 0, stdout="updated\n", stderr="") + + updated = git_state.update_pull_request_base( + Path("/repo"), + existing, + "dev", + gh_path="/usr/bin/gh", + run_process=fake_run, + ) + missing = git_state.update_pull_request_base( + Path("/repo"), + existing, + "dev", + gh_path=None, + run_process=lambda *_args, **_kwargs: self.fail("process should not run"), + ) + + self.assertEqual(updated.returncode, 0) + self.assertEqual( + calls, + [["/usr/bin/gh", "pr", "edit", "https://github.com/acme/repo/pull/9", "--base", "dev"]], + ) + self.assertEqual(missing.returncode, 127) + self.assertIn("gh is required", missing.stderr) + def test_existing_pr_url_skips_invalid_inputs_and_failed_gh(self) -> None: def unexpected_run(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: raise AssertionError("run_process should not be called") - self.assertEqual(git_state.existing_pr_url(Path("/repo"), "HEAD", gh_path="/usr/bin/gh", run_process=unexpected_run), "") - self.assertEqual(git_state.existing_pr_url(Path("/repo"), "feature", gh_path=None, run_process=unexpected_run), "") + self.assertEqual( + git_state.existing_pr_url(Path("/repo"), "HEAD", gh_path="/usr/bin/gh", run_process=unexpected_run), "" + ) + self.assertEqual( + git_state.existing_pr_url(Path("/repo"), "feature", gh_path=None, run_process=unexpected_run), "" + ) + + def test_remote_branch_probe_is_noninteractive_and_bounded(self) -> None: + captured: dict[str, object] = {} + + def timed_out(args: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + captured.update(kwargs) + raise subprocess.TimeoutExpired(args, timeout=float(kwargs["timeout"])) + + result = git_state.run_git( + Path("/repo"), + ["ls-remote", "--heads", "origin", "refs/heads/dev"], + run_process=timed_out, + ) + + self.assertEqual(result.returncode, 124) + self.assertEqual(captured["timeout"], 10) + self.assertEqual(captured["env"]["GIT_TERMINAL_PROMPT"], "0") # type: ignore[index] def failed_run(*args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: return subprocess.CompletedProcess(args, 1, stdout="", stderr="not found") - self.assertEqual(git_state.existing_pr_url(Path("/repo"), "feature", gh_path="/usr/bin/gh", run_process=failed_run), "") + self.assertEqual( + git_state.existing_pr_url(Path("/repo"), "feature", gh_path="/usr/bin/gh", run_process=failed_run), "" + ) if __name__ == "__main__": diff --git a/tests/python/actions/test_action_pr_workflow_support.py b/tests/python/actions/test_action_pr_workflow_support.py index 9ccaf606..0020ae76 100644 --- a/tests/python/actions/test_action_pr_workflow_support.py +++ b/tests/python/actions/test_action_pr_workflow_support.py @@ -7,9 +7,65 @@ from types import SimpleNamespace from envctl_engine.actions import action_pr_workflow_support as support +from envctl_engine.actions.action_git_state_support import ExistingPullRequest class ActionPrWorkflowSupportTests(unittest.TestCase): + def test_pr_workflow_rejects_default_branch_as_the_head(self) -> None: + context = SimpleNamespace( + repo_root=Path("/repo"), + project_root=Path("/repo"), + project_name="Main", + ) + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "dev\n", + resolve_base_branch_fn=lambda _context, _git_root: "dev", + existing_pull_request_fn=lambda *_args: None, + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), + probe_dirty_worktree_fn=lambda *_args: self.fail("worktree should not be probed"), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: self.fail("title should not be built"), + pr_body_fn=lambda *_args: self.fail("body should not be built"), + write_pr_body_file_fn=lambda _body: self.fail("body file should not be written"), + print_process_output_fn=lambda _result: None, + gh_path="/usr/bin/gh", + ) + + self.assertEqual(code, 1) + + def test_pr_workflow_rediscovers_existing_pr_before_self_base_guard(self) -> None: + context = SimpleNamespace( + repo_root=Path("/repo"), + project_root=Path("/repo"), + project_name="Main", + ) + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "dev\n", + resolve_base_branch_fn=lambda _context, _git_root: "dev", + existing_pull_request_fn=lambda _git_root, _branch: ExistingPullRequest( + url="https://github.com/acme/repo/pull/12", + base_branch="dev", + ), + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), + probe_dirty_worktree_fn=lambda *_args: self.fail("worktree should not be probed"), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: self.fail("title should not be built"), + pr_body_fn=lambda *_args: self.fail("body should not be built"), + write_pr_body_file_fn=lambda _body: self.fail("body file should not be written"), + print_process_output_fn=lambda _result: None, + gh_path="/usr/bin/gh", + ) + + self.assertEqual(code, 0) + def test_pr_workflow_skips_existing_pull_request(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo_root = Path(tmpdir) @@ -22,7 +78,11 @@ def test_pr_workflow_skips_existing_pull_request(self) -> None: git_available=True, git_output_fn=lambda _git_root, _args: "feature/demo\n", resolve_base_branch_fn=lambda _context, _git_root: "main", - existing_pr_url_fn=lambda _git_root, _branch: "https://github.com/acme/repo/pull/12", + existing_pull_request_fn=lambda _git_root, _branch: ExistingPullRequest( + url="https://github.com/acme/repo/pull/12", + base_branch="main", + ), + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), probe_dirty_worktree_fn=lambda *_args: self.fail("dirty worktree should not be probed"), run_commit_action_fn=lambda _context: self.fail("commit should not run"), pr_title_fn=lambda *_args: self.fail("title should not be built"), @@ -46,7 +106,9 @@ def test_pr_workflow_creates_with_gh_and_removes_body_file(self) -> None: def run_process(args: list[str], **_kwargs): # noqa: ANN001 calls.append([str(token) for token in args]) self.assertTrue(body_file.is_file()) - return subprocess.CompletedProcess(args=args, returncode=0, stdout="https://example.test/pr/1\n", stderr="") + return subprocess.CompletedProcess( + args=args, returncode=0, stdout="https://example.test/pr/1\n", stderr="" + ) def write_body(_body: str) -> Path: body_file.write_text("Body", encoding="utf-8") @@ -58,7 +120,8 @@ def write_body(_body: str) -> Path: git_available=True, git_output_fn=lambda _git_root, _args: "feature/demo\n", resolve_base_branch_fn=lambda _context, _git_root: "main", - existing_pr_url_fn=lambda _git_root, _branch: "", + existing_pull_request_fn=lambda _git_root, _branch: None, + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), probe_dirty_worktree_fn=lambda *_args: SimpleNamespace(dirty=False), run_commit_action_fn=lambda _context: self.fail("commit should not run"), pr_title_fn=lambda _context, _git_root, _head_branch: "Demo PR", @@ -75,6 +138,157 @@ def write_body(_body: str) -> Path: self.assertIn("--base", calls[0]) self.assertIn("main", calls[0]) + def test_explicit_base_updates_existing_pr_and_verifies_result(self) -> None: + context = SimpleNamespace( + repo_root=Path("/repo"), + project_root=Path("/repo"), + project_name="Main", + env={"ENVCTL_PR_BASE": "release"}, + ) + existing_values = iter( + [ + ExistingPullRequest("https://github.com/acme/repo/pull/12", "main"), + ExistingPullRequest("https://github.com/acme/repo/pull/12", "release"), + ] + ) + update_calls: list[tuple[ExistingPullRequest, str]] = [] + + def update_base( + _git_root: Path, + existing: ExistingPullRequest, + base_branch: str, + ) -> subprocess.CompletedProcess[str]: + update_calls.append((existing, base_branch)) + return subprocess.CompletedProcess([], 0, stdout="updated\n", stderr="") + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "feature/demo\n", + resolve_base_branch_fn=lambda *_args: self.fail("implicit resolver should not run"), + existing_pull_request_fn=lambda *_args: next(existing_values), + update_pull_request_base_fn=update_base, + probe_dirty_worktree_fn=lambda *_args: self.fail("worktree should not be probed"), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: self.fail("title should not be built"), + pr_body_fn=lambda *_args: self.fail("body should not be built"), + write_pr_body_file_fn=lambda _body: self.fail("body file should not be written"), + print_process_output_fn=lambda _result: None, + gh_path="/usr/bin/gh", + ) + + self.assertEqual(code, 0) + self.assertEqual(len(update_calls), 1) + self.assertEqual(update_calls[0][1], "release") + + def test_implicit_default_preserves_existing_pr_base(self) -> None: + context = SimpleNamespace( + repo_root=Path("/repo"), + project_root=Path("/repo"), + project_name="Main", + env={}, + ) + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "feature/demo\n", + resolve_base_branch_fn=lambda *_args: self.fail("implicit resolver should not run for an existing PR"), + existing_pull_request_fn=lambda *_args: ExistingPullRequest( + "https://github.com/acme/repo/pull/12", + "main", + ), + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), + probe_dirty_worktree_fn=lambda *_args: self.fail("worktree should not be probed"), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: self.fail("title should not be built"), + pr_body_fn=lambda *_args: self.fail("body should not be built"), + write_pr_body_file_fn=lambda _body: self.fail("body file should not be written"), + print_process_output_fn=lambda _result: None, + gh_path="/usr/bin/gh", + ) + + self.assertEqual(code, 0) + + def test_failed_gh_create_recovers_concurrent_matching_pr(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + repo_root = Path(tmpdir) + context = SimpleNamespace(repo_root=repo_root, project_root=repo_root, project_name="Main", env={}) + existing_values = iter( + [ + None, + ExistingPullRequest("https://github.com/acme/repo/pull/13", "dev"), + ] + ) + body_file = repo_root / "body.md" + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "feature/race\n", + resolve_base_branch_fn=lambda *_args: "dev", + existing_pull_request_fn=lambda *_args: next(existing_values), + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), + probe_dirty_worktree_fn=lambda *_args: SimpleNamespace(dirty=False), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: "Race", + pr_body_fn=lambda *_args: "Body", + write_pr_body_file_fn=lambda _body: body_file, + print_process_output_fn=lambda _result: self.fail("recovered create error should stay quiet"), + gh_path="/usr/bin/gh", + run_process_fn=lambda args, **_kwargs: subprocess.CompletedProcess( + args, + 1, + stdout="", + stderr="a pull request already exists", + ), + ) + + self.assertEqual(code, 0) + + def test_failed_helper_create_recovers_concurrent_matching_pr(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + repo_root = Path(tmpdir) + helper = repo_root / "utils" / "create-pr.sh" + helper.parent.mkdir(parents=True) + helper.write_text("#!/bin/sh\n", encoding="utf-8") + helper.chmod(0o755) + context = SimpleNamespace(repo_root=repo_root, project_root=repo_root, project_name="Main", env={}) + existing_values = iter( + [ + None, + ExistingPullRequest("https://github.com/acme/repo/pull/14", "dev"), + ] + ) + + code = support.run_pr_workflow( + context, + resolve_git_root_fn=lambda project_root, _repo_root: project_root, + git_available=True, + git_output_fn=lambda _git_root, _args: "feature/race\n", + resolve_base_branch_fn=lambda *_args: "dev", + existing_pull_request_fn=lambda *_args: next(existing_values), + update_pull_request_base_fn=lambda *_args: self.fail("base update should not run"), + probe_dirty_worktree_fn=lambda *_args: SimpleNamespace(dirty=False), + run_commit_action_fn=lambda _context: self.fail("commit should not run"), + pr_title_fn=lambda *_args: self.fail("title should not be built"), + pr_body_fn=lambda *_args: self.fail("body should not be built"), + write_pr_body_file_fn=lambda _body: self.fail("body file should not be written"), + print_process_output_fn=lambda _result: self.fail("recovered helper error should stay quiet"), + gh_path=None, + run_process_fn=lambda args, **_kwargs: subprocess.CompletedProcess( + args, + 1, + stdout="", + stderr="a pull request already exists", + ), + ) + + self.assertEqual(code, 0) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/actions/test_action_ship_support.py b/tests/python/actions/test_action_ship_support.py index df0ece67..36da1e92 100644 --- a/tests/python/actions/test_action_ship_support.py +++ b/tests/python/actions/test_action_ship_support.py @@ -20,7 +20,7 @@ def test_public_run_ship_workflow_is_thin_compatibility_wrapper() -> None: source = inspect.getsource(run_ship_workflow) assert "ShipWorkflowRunner" in source - assert len(source.splitlines()) <= 35 + assert len(source.splitlines()) <= 40 def test_ship_workflow_runner_exposes_named_phases() -> None: @@ -58,6 +58,7 @@ def test_ship_workflow_dependencies_group_injected_collaborators() -> None: "partition_envctl_protected_paths", "ordered_unique_paths", "github_pr_checks", + "existing_pull_request", } runner_fields = set(getattr(ship_support.ShipWorkflowRunner, "__dataclass_fields__", {})) diff --git a/tests/python/actions/test_action_ship_workflow_lifecycle.py b/tests/python/actions/test_action_ship_workflow_lifecycle.py index ca8a8661..c53019a9 100644 --- a/tests/python/actions/test_action_ship_workflow_lifecycle.py +++ b/tests/python/actions/test_action_ship_workflow_lifecycle.py @@ -3,6 +3,7 @@ from pathlib import Path from types import SimpleNamespace +from envctl_engine.actions.action_git_state_support import ExistingPullRequest from tests.python.actions.ship_workflow_test_support import ship_workflow_fixture @@ -55,6 +56,82 @@ def github_pr_checks( assert result.payload["checks_expected_head_sha"] == "abc123" +def test_run_ship_workflow_reconciles_explicit_base_before_reusing_existing_pr() -> None: + with ship_workflow_fixture() as fixture: + fixture.context.env["ENVCTL_PR_BASE"] = "dev" + pr_calls: list[object] = [] + resolved_bases: list[str] = [] + + def run_pr_action(context: object) -> int: + pr_calls.append(context) + return 0 + + def resolve_base_ref(_git_root: Path, base_branch: str) -> str: + resolved_bases.append(base_branch) + return f"origin/{base_branch}" + + result = fixture.run(run_pr_action=run_pr_action, resolve_base_ref=resolve_base_ref) + + assert result.code == 0 + assert pr_calls == [fixture.context] + assert resolved_bases == ["dev"] + assert result.payload["pr_created"] is False + assert result.payload["pr_url"] == "https://github.com/acme/repo/pull/7" + assert result.payload["step_statuses"] == ["clean_no_changes", "pr_exists", "checks_passed"] + + +def test_run_ship_workflow_predicts_conflicts_against_implicit_existing_pr_base() -> None: + with ship_workflow_fixture() as fixture: + resolved_bases: list[str] = [] + + def resolve_base_branch(_context: object, _git_root: Path) -> str: + raise AssertionError("default resolver must not replace an existing PR base") + + def resolve_base_ref(_git_root: Path, base_branch: str) -> str: + resolved_bases.append(base_branch) + return f"origin/{base_branch}" + + result = fixture.run( + existing_pull_request=lambda _git_root, _branch: ExistingPullRequest( + url="https://github.com/acme/repo/pull/7", + base_branch="main", + ), + resolve_base_branch=resolve_base_branch, + resolve_base_ref=resolve_base_ref, + ) + + assert result.code == 0 + assert resolved_bases == ["main"] + assert result.payload["merge_conflicts"]["base_branch"] == "main" + + +def test_run_ship_workflow_tracks_concurrent_pr_winner_after_pr_action_recovers() -> None: + with ship_workflow_fixture() as fixture: + existing_calls = 0 + pr_calls = 0 + + def existing_pr_url(_git_root: Path, _branch: str) -> str: + nonlocal existing_calls + existing_calls += 1 + return "" if existing_calls == 1 else "https://github.com/acme/repo/pull/9" + + def run_pr_action(_context: object) -> int: + nonlocal pr_calls + pr_calls += 1 + return 0 + + result = fixture.run( + existing_pr_url=existing_pr_url, + run_pr_action=run_pr_action, + ) + + assert result.code == 0 + assert pr_calls == 1 + assert existing_calls >= 2 + assert result.payload["pr_url"] == "https://github.com/acme/repo/pull/9" + assert result.payload["step_statuses"] == ["clean_no_changes", "pr_created", "checks_passed"] + + def test_run_ship_workflow_pushes_clean_local_head_when_existing_pr_branch_is_stale() -> None: with ship_workflow_fixture() as fixture: run_git_calls: list[list[str]] = [] @@ -357,6 +434,4 @@ def git_output(_git_root: Path, args: list[str]) -> str: assert result.payload["committed"] is False assert result.payload["pushed"] is False assert result.payload["protected_local_artifacts_skipped"] == [".envctl-state/code-intelligence.json"] - assert result.stderr.splitlines() == [ - "ship: PR already exists for Main: https://github.com/acme/repo/pull/7" - ] + assert result.stderr.splitlines() == ["ship: PR already exists for Main: https://github.com/acme/repo/pull/7"] diff --git a/tests/python/actions/test_action_worktree_runner.py b/tests/python/actions/test_action_worktree_runner.py index 200743db..46ff7778 100644 --- a/tests/python/actions/test_action_worktree_runner.py +++ b/tests/python/actions/test_action_worktree_runner.py @@ -74,9 +74,9 @@ def test_worktree_runner_uses_named_owners_for_target_resolution_and_delete_exec target_resolution_source = ( REPO_ROOT / "python/envctl_engine/actions/action_worktree_target_resolution.py" ).read_text(encoding="utf-8") - self_destruct_source = ( - REPO_ROOT / "python/envctl_engine/actions/action_worktree_self_destruct.py" - ).read_text(encoding="utf-8") + self_destruct_source = (REPO_ROOT / "python/envctl_engine/actions/action_worktree_self_destruct.py").read_text( + encoding="utf-8" + ) self.assertIn("class ActionWorktreeDeleteRunner", source) self.assertIn("return ActionWorktreeDeleteRunner(", source) @@ -106,9 +106,9 @@ def test_resolve_current_worktree_target_discovers_matching_current_tree(self) - runtime=runtime, require_configured_main_root=True, current_cwd=lambda: Path("/should/not/be/used"), - discover_tree_projects_fn=lambda repo_root, trees_dir_name: [("feature-a-1", tree_root)] - if repo_root == repo and trees_dir_name == "trees" - else [], + discover_tree_projects_fn=lambda repo_root, trees_dir_name: ( + [("feature-a-1", tree_root)] if repo_root == repo and trees_dir_name == "trees" else [] + ), main_repo_root_for_linked_worktree_fn=lambda _worktree_root: None, git_main_repo_root_for_worktree_fn=lambda _worktree_root, trees_dir_name=None: repo, ) @@ -305,15 +305,14 @@ def test_run_self_destruct_worktree_action_uses_explicit_current_worktree_safety calls: list[tuple[str, object]] = [] runtime = SimpleNamespace( - _blast_worktree_before_delete=lambda **kwargs: ( - calls.append(("cleanup", kwargs)) or ["cleanup warning"] - ), + _blast_worktree_before_delete=lambda **kwargs: calls.append(("cleanup", kwargs)) or ["cleanup warning"], _trees_root_for_worktree=lambda value: calls.append(("trees_root", value)) or tree_root.parent, ) orchestrator = SimpleNamespace( runtime=runtime, - resolve_targets=lambda route, *, trees_only: calls.append(("resolve", (route.command, trees_only))) - or ([target], None), + resolve_targets=lambda route, *, trees_only: ( + calls.append(("resolve", (route.command, trees_only))) or ([target], None) + ), _main_repo_root_for_worktree=lambda value: calls.append(("main_repo", value)) or tree_root.parent, _spawn_self_destruct_helper=lambda **kwargs: calls.append(("spawn", kwargs)) or True, ) diff --git a/tests/python/actions/test_actions_cli_commit_protected_artifacts.py b/tests/python/actions/test_actions_cli_commit_protected_artifacts.py index f7367ad1..92e6a2e4 100644 --- a/tests/python/actions/test_actions_cli_commit_protected_artifacts.py +++ b/tests/python/actions/test_actions_cli_commit_protected_artifacts.py @@ -16,7 +16,7 @@ def test_commit_action_skips_envctl_local_artifacts_and_commits_normal_changes(t assert result.code == 0 assert "Skipping envctl-local artifacts: .envctl-commit-message.md, MAIN_TASK.md" in result.output - assert ["add", "--", "app.py"] in harness.seen_git_args + assert ["add", "--all", "--", "app.py"] in harness.seen_git_args assert ["push", "-u", "origin", "feature/demo"] in harness.seen_git_args @@ -57,7 +57,7 @@ def test_commit_action_recovers_when_envctl_local_artifacts_are_already_staged(t assert "Skipping envctl-local artifacts: MAIN_TASK.md, .envctl-commit-message.md" in result.output assert "Refusing to commit" not in result.output assert ["reset", "-q", "--", "MAIN_TASK.md"] in harness.seen_git_args - assert ["add", "--", "app.py"] in harness.seen_git_args + assert ["add", "--all", "--", "app.py"] in harness.seen_git_args assert ["push", "-u", "origin", "feature/demo"] in harness.seen_git_args diff --git a/tests/python/actions/test_actions_cli_pr.py b/tests/python/actions/test_actions_cli_pr.py index e92c9dd7..cf5522e0 100644 --- a/tests/python/actions/test_actions_cli_pr.py +++ b/tests/python/actions/test_actions_cli_pr.py @@ -2,6 +2,7 @@ from __future__ import annotations from tests.python.actions.actions_cli_test_support import * # noqa: F403,F405 +from envctl_engine.actions.action_git_state_support import ExistingPullRequest class ActionsCliPrTests(unittest.TestCase): @@ -34,8 +35,11 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), patch( - "envctl_engine.actions.project_action_domain.existing_pr_url", - return_value="https://github.com/acme/supportopia/pull/42", + "envctl_engine.actions.project_action_domain.existing_pull_request", + return_value=ExistingPullRequest( + url="https://github.com/acme/supportopia/pull/42", + base_branch="main", + ), ), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), @@ -95,7 +99,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.probe_dirty_worktree") as dirty_probe, patch("envctl_engine.actions.project_action_domain.run_commit_action", return_value=0) as commit_action, patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -146,7 +150,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.probe_dirty_worktree") as dirty_probe, patch("envctl_engine.actions.project_action_domain.run_commit_action", return_value=1) as commit_action, patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), @@ -223,7 +227,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -302,7 +306,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -369,7 +373,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -431,7 +435,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 patch.dict(os.environ, {"ENVCTL_PR_BODY": "Typed PR body\n\nWith details."}, clear=False), patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -495,7 +499,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 patch("builtins.input", side_effect=AssertionError("input() should not be called for pr action")), patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -553,7 +557,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -602,7 +606,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): @@ -659,7 +663,7 @@ def fake_run(args: list[str], **_kwargs): # noqa: ANN001 with ( patch("envctl_engine.actions.project_action_domain.detect_default_branch", return_value="main"), patch("envctl_engine.actions.project_action_domain._git_output", side_effect=fake_git_output), - patch("envctl_engine.actions.project_action_domain.existing_pr_url", return_value=""), + patch("envctl_engine.actions.project_action_domain.existing_pull_request", return_value=None), patch("envctl_engine.actions.project_action_domain.shutil.which", side_effect=fake_which), patch("envctl_engine.actions.project_action_domain.subprocess.run", side_effect=fake_run), ): diff --git a/tests/python/actions/test_actions_cli_ship.py b/tests/python/actions/test_actions_cli_ship.py index 569e6da2..166a90a6 100644 --- a/tests/python/actions/test_actions_cli_ship.py +++ b/tests/python/actions/test_actions_cli_ship.py @@ -1,6 +1,8 @@ # ruff: noqa: F403,F405 from __future__ import annotations +from envctl_engine.actions.action_git_state_support import ExistingPullRequest +from envctl_engine.actions.action_protected_artifacts import unstaged_stageable_paths from tests.python.actions.actions_cli_test_support import * # noqa: F403,F405 @@ -34,6 +36,21 @@ def test_partition_envctl_protected_paths_separates_staged_skipped_and_stageable ) self.assertEqual(partition.stageable_paths, ["app.py", "docs/reference/commands.md", "new_name.py"]) + def test_unstaged_stageable_paths_excludes_staged_only_changes_and_keeps_rename_source(self) -> None: + self.assertEqual( + unstaged_stageable_paths( + "D already-deleted.py\n" + "M already-modified.py\n" + " D deleted.py\n" + " M modified.py\n" + "MM both.py\n" + " R old_name.py -> new_name.py\n" + "?? new.py\n" + " M MAIN_TASK.md\n" + ), + ["deleted.py", "modified.py", "both.py", "old_name.py", "new_name.py", "new.py"], + ) + def test_ship_action_reuses_commit_and_pr_then_reports_passed_checks_json(self) -> None: domain = importlib.import_module("envctl_engine.actions.project_action_domain") with tempfile.TemporaryDirectory() as tmpdir: @@ -80,7 +97,14 @@ def fake_commit_action(_context: object) -> int: patch("envctl_engine.actions.project_action_domain.add_ship_pr_label", return_value=0) as label_pr, patch( "envctl_engine.actions.project_action_domain.existing_pr_url", - side_effect=["", "https://github.com/acme/repo/pull/7"], + return_value="https://github.com/acme/repo/pull/7", + ), + patch( + "envctl_engine.actions.project_action_domain.existing_pull_request", + side_effect=[ + None, + ExistingPullRequest("https://github.com/acme/repo/pull/7", "main"), + ], ), patch( "envctl_engine.actions.project_action_domain._github_pr_checks", @@ -169,6 +193,10 @@ def fake_run_git(_git_root: Path, args: list[str]) -> subprocess.CompletedProces "envctl_engine.actions.project_action_domain.existing_pr_url", return_value="https://github.com/acme/repo/pull/7", ), + patch( + "envctl_engine.actions.project_action_domain.existing_pull_request", + return_value=ExistingPullRequest("https://github.com/acme/repo/pull/7", "main"), + ), patch( "envctl_engine.actions.project_action_domain._github_pr_checks", return_value={ @@ -245,6 +273,10 @@ def fake_run(git_root: Path, args: list[str]): # noqa: ANN001 "envctl_engine.actions.project_action_domain.existing_pr_url", return_value="https://github.com/acme/repo/pull/7", ), + patch( + "envctl_engine.actions.project_action_domain.existing_pull_request", + return_value=ExistingPullRequest("https://github.com/acme/repo/pull/7", "main"), + ), redirect_stdout(StringIO()) as stdout, ): code = domain.run_ship_action(context) @@ -284,11 +316,7 @@ def fake_git_output(_git_root: Path, args: list[str]) -> str: if args == ["diff", "--name-only", "--diff-filter=U"]: return "src/service.py\n" if args == ["ls-files", "-u"]: - return ( - "100644 aaa 1\tsrc/service.py\n" - "100644 bbb 2\tsrc/service.py\n" - "100644 ccc 3\tsrc/service.py\n" - ) + return "100644 aaa 1\tsrc/service.py\n100644 bbb 2\tsrc/service.py\n100644 ccc 3\tsrc/service.py\n" return "" with ( diff --git a/tests/python/planning/plan_agent_launch_support_test_support.py b/tests/python/planning/plan_agent_launch_support_test_support.py index 48ffaaca..2c611020 100644 --- a/tests/python/planning/plan_agent_launch_support_test_support.py +++ b/tests/python/planning/plan_agent_launch_support_test_support.py @@ -127,8 +127,14 @@ class _StateRepositoryHarness: def __init__(self, state: RunState | None) -> None: self.state = state - def load_latest(self, *, mode: str | None = None, strict_mode_match: bool = False) -> RunState | None: - _ = mode, strict_mode_match + def load_latest( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: list[str] | None = None, + ) -> RunState | None: + _ = mode, strict_mode_match, project_names return self.state diff --git a/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py b/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py index 127f9fcd..20e6c844 100644 --- a/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py +++ b/tests/python/planning/test_plan_agent_workflow_runtime_addresses.py @@ -24,8 +24,14 @@ class _StateRepositoryHarness: def __init__(self, state: RunState | None) -> None: self.state = state - def load_latest(self, *, mode: str | None = None, strict_mode_match: bool = False) -> RunState | None: - _ = mode, strict_mode_match + def load_latest( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: list[str] | None = None, + ) -> RunState | None: + _ = mode, strict_mode_match, project_names return self.state @@ -37,6 +43,43 @@ def __init__(self, state: RunState | None) -> None: class PlanAgentWorkflowRuntimeAddressTests(unittest.TestCase): + def test_worktree_address_lookup_retries_unfiltered_for_base_project_state(self) -> None: + calls: list[list[str] | None] = [] + state = RunState( + run_id="run-base", + mode="trees", + requirements={ + "feature-a": RequirementsResult( + project="feature-a", + redis={"enabled": True, "final": 6381}, + ) + }, + ) + + class BaseProjectRepository: + def load_latest( + self, + *, + mode: str | None = None, + strict_mode_match: bool = False, + project_names: list[str] | None = None, + ) -> RunState | None: + _ = mode, strict_mode_match + calls.append(project_names) + return state if project_names is None else None + + runtime = SimpleNamespace(state_repository=BaseProjectRepository()) + worktree = CreatedPlanWorktree( + name="feature-a-1", + root=Path("/repo/trees/feature-a/1"), + plan_file="feature-a.md", + ) + + section = _runtime_addresses_prompt_section(runtime, worktree=worktree) + + self.assertEqual(calls, [["feature-a-1"], None]) + self.assertIn("Redis (feature-a): redis://localhost:6381", section) + def test_builder_renders_dependency_and_service_addresses_for_matching_worktree(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: worktree_root = Path(tmpdir) / "repo" / "trees" / "feature-a-1" @@ -127,9 +170,10 @@ def test_matching_and_port_helpers_cover_stale_or_partial_state(self) -> None: self.assertTrue(_state_service_matches_worktree(matching_service, worktree)) self.assertFalse(_state_service_matches_worktree(stale_service, worktree)) self.assertEqual(_component_port({"requested": "0", "resources": {"primary": "49152"}}), 49152) - self.assertEqual(_service_address_lines(RunState(run_id="run-1", mode="trees", services={"api": matching_service})), [ - "Backend (backend): http://localhost:8000" - ]) + self.assertEqual( + _service_address_lines(RunState(run_id="run-1", mode="trees", services={"api": matching_service})), + ["Backend (backend): http://localhost:8000"], + ) def test_runtime_section_uses_legacy_state_loader_when_repository_is_unavailable(self) -> None: state = RunState( diff --git a/tests/python/planning/test_planning_worktree_setup_provenance.py b/tests/python/planning/test_planning_worktree_setup_provenance.py index 8fbd5cc9..c10e2e1f 100644 --- a/tests/python/planning/test_planning_worktree_setup_provenance.py +++ b/tests/python/planning/test_planning_worktree_setup_provenance.py @@ -1,10 +1,62 @@ # ruff: noqa: F403,F405 from __future__ import annotations +from types import SimpleNamespace + +from envctl_engine.planning.worktree_provenance import detect_default_branch from tests.python.planning.planning_worktree_setup_test_support import * class PlanningWorktreeSetupProvenanceTests(PlanningWorktreeSetupTestCase): + def test_detached_default_branch_uses_live_remote_over_stale_tracking_ref(self) -> None: + calls: list[dict[str, object]] = [] + + def fake_run(cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 + args = [str(token) for token in cmd] + calls.append({"args": args, "cwd": cwd, "env": env, "timeout": timeout}) + git_args = args[3:] + if git_args == ["rev-parse", "--verify", "refs/remotes/origin/dev"]: + return subprocess.CompletedProcess(args, 0, stdout="stale-dev\n", stderr="") + if git_args == ["rev-parse", "--verify", "refs/remotes/origin/main"]: + return subprocess.CompletedProcess(args, 0, stdout="main\n", stderr="") + if git_args[:3] == ["ls-remote", "--symref", "origin"]: + return subprocess.CompletedProcess( + args, + 0, + stdout=("ref: refs/heads/main\tHEAD\nmain-sha\tHEAD\nmain-sha\trefs/heads/main\n"), + stderr="", + ) + return subprocess.CompletedProcess(args, 1, stdout="", stderr="missing") + + runtime = SimpleNamespace( + config=SimpleNamespace(base_dir=Path("/repo")), + process_runner=SimpleNamespace(run=fake_run), + _command_env=lambda *, port: {"BASE": "1", "PORT": str(port)}, + ) + + self.assertEqual(detect_default_branch(runtime), "main") + live_probe = next(call for call in calls if call["args"][3] == "ls-remote") + self.assertEqual(live_probe["timeout"], 10.0) + self.assertEqual(live_probe["env"]["GIT_TERMINAL_PROMPT"], "0") + self.assertEqual(live_probe["env"]["BASE"], "1") + + def test_detached_default_branch_falls_back_to_local_tracking_ref_offline(self) -> None: + def fake_run(cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 + _ = cwd, env, timeout + args = [str(token) for token in cmd] + git_args = args[3:] + if git_args == ["rev-parse", "--verify", "refs/remotes/origin/dev"]: + return subprocess.CompletedProcess(args, 0, stdout="local-dev\n", stderr="") + return subprocess.CompletedProcess(args, 1, stdout="", stderr="offline") + + runtime = SimpleNamespace( + config=SimpleNamespace(base_dir=Path("/repo")), + process_runner=SimpleNamespace(run=fake_run), + _command_env=lambda *, port: {"PORT": str(port)}, + ) + + self.assertEqual(detect_default_branch(runtime), "dev") + def test_setup_worktree_creation_writes_provenance_file(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -57,9 +109,7 @@ def fake_run(cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 'project_name: "repo"\n', ) self.assertEqual( - (repo / "trees" / "feature-a" / "1" / ".serena" / "project.local.yml").read_text( - encoding="utf-8" - ), + (repo / "trees" / "feature-a" / "1" / ".serena" / "project.local.yml").read_text(encoding="utf-8"), 'project_name: "repo-feature-a-1"\n', ) self.assertEqual( @@ -233,7 +283,9 @@ def fake_run(cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 if git_cwd == str(unrelated.resolve()) and command[3:] == ["rev-parse", "--show-toplevel"]: return subprocess.CompletedProcess(args=command, returncode=0, stdout=f"{unrelated}\n", stderr="") if git_cwd == str(repo.resolve()) and command[3:] == ["rev-parse", "--git-common-dir"]: - return subprocess.CompletedProcess(args=command, returncode=0, stdout=f"{repo / '.git'}\n", stderr="") + return subprocess.CompletedProcess( + args=command, returncode=0, stdout=f"{repo / '.git'}\n", stderr="" + ) if git_cwd == str(unrelated.resolve()) and command[3:] == ["rev-parse", "--git-common-dir"]: return subprocess.CompletedProcess( args=command, returncode=0, stdout=f"{unrelated / '.git'}\n", stderr="" diff --git a/tests/python/runtime/engine_runtime_real_startup_test_support.py b/tests/python/runtime/engine_runtime_real_startup_test_support.py index 7f46ce70..7da95859 100644 --- a/tests/python/runtime/engine_runtime_real_startup_test_support.py +++ b/tests/python/runtime/engine_runtime_real_startup_test_support.py @@ -102,6 +102,7 @@ def __init__(self) -> None: self._next_pid = 4000 self.auto_track_listener_ports = False self._pid_listener_ports: dict[int, int] = {} + self._terminated_pids: set[int] = set() self.wait_for_port_result = False self.wait_for_port_calls: list[tuple[int, float]] = [] self.wait_for_pid_port_result = False @@ -199,6 +200,7 @@ def start_background(self, cmd, *, cwd=None, env=None, stdout_path=None, stderr_ _ = stderr_path self._next_pid += 1 pid = self._next_pid + self._terminated_pids.discard(pid) if parsed_port is not None and parsed_port in self.fail_start_ports: if stdout_path: @@ -269,6 +271,8 @@ def find_pid_listener_port( return self.find_pid_listener_port_result def is_pid_running(self, _pid: int) -> bool: + if _pid in self._terminated_pids: + return False if self.auto_track_listener_ports and _pid in self._pid_listener_ports: return True return self.pid_running_result @@ -281,6 +285,7 @@ def pid_owns_port(self, _pid: int, _port: int) -> bool: def terminate(self, _pid: int, *, term_timeout: float = 2.0, kill_timeout: float = 1.0) -> bool: _ = term_timeout, kill_timeout + self._terminated_pids.add(_pid) if self.auto_track_listener_ports: self._pid_listener_ports.pop(_pid, None) return True diff --git a/tests/python/runtime/lifecycle_parity_test_support.py b/tests/python/runtime/lifecycle_parity_test_support.py index 6b1c1537..8126f913 100644 --- a/tests/python/runtime/lifecycle_parity_test_support.py +++ b/tests/python/runtime/lifecycle_parity_test_support.py @@ -33,7 +33,6 @@ dump_state = state_module.dump_state - class _NoopPlanner: def __init__(self) -> None: self.released = False @@ -46,6 +45,7 @@ def release(self, port: int, owner: str | None = None) -> None: _ = owner self.released_ports.append(port) + class _TrackingPlanner: def __init__(self) -> None: self.released_session = False @@ -57,6 +57,7 @@ def release_session(self) -> None: def release_all(self) -> None: self.released_all = True + class _TrackingRunner: def __init__(self) -> None: self.terminated: list[int] = [] @@ -72,7 +73,7 @@ def terminate(self, pid: int, *, term_timeout: float = 2.0, kill_timeout: float return True def is_pid_running(self, pid: int) -> bool: - return pid > 0 + return pid > 0 and pid not in self.terminated def pid_owns_port(self, pid: int, port: int) -> bool: _ = pid, port @@ -94,11 +95,13 @@ def run(self, cmd, *, cwd=None, env=None, timeout=None): # noqa: ANN001 return type("Result", (), {"returncode": 0, "stdout": stdout, "stderr": ""})() return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + class _OwnershipDenyRunner(_TrackingRunner): def pid_owns_port(self, pid: int, port: int) -> bool: _ = pid, port return False + class _ResumeRestoreRunner: def __init__(self) -> None: self.run_calls: list[tuple[str, ...]] = [] @@ -175,6 +178,7 @@ def terminate(self, pid: int, *, term_timeout: float = 2.0, kill_timeout: float self._running.discard(pid) return True + __all__ = [ "Path", "PYTHON_ROOT", diff --git a/tests/python/runtime/test_engine_runtime_artifacts.py b/tests/python/runtime/test_engine_runtime_artifacts.py index cf3973fe..05c3f32e 100644 --- a/tests/python/runtime/test_engine_runtime_artifacts.py +++ b/tests/python/runtime/test_engine_runtime_artifacts.py @@ -107,6 +107,34 @@ def test_persist_events_snapshot_updates_bound_run_directory(self) -> None: self.assertEqual((legacy_root / "events.jsonl").read_text(encoding="utf-8"), expected) self.assertEqual((run_dir / "events.jsonl").read_text(encoding="utf-8"), expected) + def test_persist_events_snapshot_uses_repository_fence_for_bound_run(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_root = Path(tmpdir) / "runtime" + legacy_root = Path(tmpdir) / "legacy" + calls: list[dict[str, object]] = [] + runtime = SimpleNamespace( + events=[{"event": "test"}], + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + env={"ENVCTL_DEBUG_UI_RUN_ID": "run-1"}, + state_repository=SimpleNamespace(write_runtime_artifact=lambda **kwargs: calls.append(kwargs) or False), + ) + + persist_events_snapshot(runtime) + + self.assertEqual( + calls, + [ + { + "run_id": "run-1", + "artifact_name": "events.jsonl", + "text": '{"event": "test"}\n', + } + ], + ) + self.assertFalse((runtime_root / "events.jsonl").exists()) + self.assertFalse((legacy_root / "events.jsonl").exists()) + def test_write_artifacts_writes_runtime_readiness_report_synchronously_without_explicit_async_opt_in(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: runtime_root = Path(tmpdir) / "runtime" @@ -235,6 +263,51 @@ def test_write_runtime_readiness_report_writes_runtime_and_run_reports(self) -> self.assertEqual(run_report["gap_report"]["sha256"], "gap123") self.assertEqual(run_report["feature_matrix"]["sha256"], "matrix123") + def test_readiness_report_does_not_bypass_repository_when_stale_run_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_root = Path(tmpdir) / "runtime" + legacy_root = Path(tmpdir) / "legacy" + run_dir = runtime_root / "runs" / "run-retired" + calls: list[dict[str, object]] = [] + result = RuntimeReadinessResult( + passed=True, + report_path=Path(tmpdir) / "gap.json", + report_generated_at="now", + report_sha256="gap", + matrix_path=Path(tmpdir) / "matrix.json", + matrix_generated_at="now", + matrix_sha256="matrix", + parity_manifest_path=Path(tmpdir) / "manifest.json", + parity_manifest_generated_at="now", + parity_manifest_sha256="manifest", + blocking_gap_count=0, + high_gap_count=0, + medium_gap_count=0, + low_gap_count=0, + total_gap_count=0, + errors=[], + warnings=[], + ) + runtime = SimpleNamespace( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + config=SimpleNamespace(base_dir=Path(tmpdir)), + state_repository=SimpleNamespace(write_runtime_artifact=lambda **kwargs: calls.append(kwargs) or False), + ) + + persisted = write_runtime_readiness_report( + runtime, + run_dir=run_dir, + readiness_result=result, + ) + + self.assertFalse(persisted) + self.assertEqual(calls[0]["run_id"], "run-retired") + self.assertEqual(calls[0]["artifact_name"], "runtime_readiness_report.json") + self.assertFalse((runtime_root / "runtime_readiness_report.json").exists()) + self.assertFalse((legacy_root / "runtime_readiness_report.json").exists()) + self.assertFalse((run_dir / "runtime_readiness_report.json").exists()) + def test_write_runtime_readiness_report_reuses_cached_payload_when_report_hash_matches(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo_root = Path(tmpdir) / "repo" @@ -249,10 +322,16 @@ def test_write_runtime_readiness_report_reuses_cached_payload_when_report_hash_m run_dir.mkdir() report_text = json.dumps({"generated_at": "now", "summary": {"total_gap_count": 0}}, sort_keys=True) report_hash = hashlib.sha256(report_text.encode("utf-8")).hexdigest() - matrix_text = json.dumps({"generated_at": "now", "features": [], "summary": {"feature_count": 0}}, sort_keys=True) - matrix_hash = hashlib.sha256((json.dumps(json.loads(matrix_text), indent=2, sort_keys=True) + "\n").encode("utf-8")).hexdigest() + matrix_text = json.dumps( + {"generated_at": "now", "features": [], "summary": {"feature_count": 0}}, sort_keys=True + ) + matrix_hash = hashlib.sha256( + (json.dumps(json.loads(matrix_text), indent=2, sort_keys=True) + "\n").encode("utf-8") + ).hexdigest() gap_report_path.write_text(report_text, encoding="utf-8") - matrix_path.write_text(json.dumps(json.loads(matrix_text), indent=2, sort_keys=True) + "\n", encoding="utf-8") + matrix_path.write_text( + json.dumps(json.loads(matrix_text), indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) manifest_path.write_text( json.dumps({"generated_at": "now", "commands": {}, "modes": {}}, sort_keys=True), encoding="utf-8" ) diff --git a/tests/python/runtime/test_engine_runtime_command_parity_state_config.py b/tests/python/runtime/test_engine_runtime_command_parity_state_config.py index a63b11e1..374fcbed 100644 --- a/tests/python/runtime/test_engine_runtime_command_parity_state_config.py +++ b/tests/python/runtime/test_engine_runtime_command_parity_state_config.py @@ -2,10 +2,10 @@ from __future__ import annotations from tests.python.runtime.engine_runtime_command_parity_test_support import * +from envctl_engine.state.run_index import StateSelector class EngineRuntimeCommandParityStateConfigTests(EngineRuntimeCommandParityTestCase): - def test_list_targets_discovers_projects_in_main_mode(self) -> None: tmpdir = tempfile.TemporaryDirectory() self.addCleanup(tmpdir.cleanup) @@ -82,7 +82,11 @@ def test_list_trees_json_reports_selected_state_presence_and_service_truth(self) state=RunState( run_id="run-prev", mode="trees", - services={"feature-1 Backend": ServiceRecord(name="feature-1 Backend", type="backend", cwd=".", status="unknown")}, + services={ + "feature-1 Backend": ServiceRecord( + name="feature-1 Backend", type="backend", cwd=".", status="unknown" + ) + }, requirements={}, metadata={"repo_scope_id": config.runtime_scope_id, "project_roots": {"feature-1": str(feature_dir)}}, ), @@ -104,6 +108,66 @@ def test_list_trees_json_reports_selected_state_presence_and_service_truth(self) self.assertFalse(feature["services_running"]) self.assertFalse(feature["running"]) + def test_tree_list_json_reports_discovery_and_integrity_warning_for_tampered_active_run(self) -> None: + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + repo = Path(tmpdir.name) / "repo" + runtime_root = Path(tmpdir.name) / "runtime" + (repo / ".git").mkdir(parents=True, exist_ok=True) + feature_dirs = { + name: repo / "trees" / "feature" / suffix for name, suffix in (("feature-1", "1"), ("feature-2", "2")) + } + for feature_dir in feature_dirs.values(): + feature_dir.mkdir(parents=True, exist_ok=True) + (feature_dir / ".envctl").write_text("ENVCTL_DEFAULT_MODE=trees\n") + config = load_config({"RUN_REPO_ROOT": str(repo), "RUN_SH_RUNTIME_DIR": str(runtime_root)}) + runtime = PythonEngineRuntime(config, env={}) + + for run_id, project_name in (("run-valid", "feature-1"), ("run-tampered", "feature-2")): + runtime.state_repository.save_resume_state( + state=RunState( + run_id=run_id, + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=str(feature_dirs[project_name]), + project=project_name, + ) + }, + requirements={}, + metadata={ + "repo_scope_id": config.runtime_scope_id, + "project_roots": {project_name: str(feature_dirs[project_name])}, + }, + ), + emit=lambda *args, **kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + tampered = next( + candidate + for candidate in runtime.state_repository.run_index.candidates( + StateSelector(mode="trees", project_names=()) + ) + if candidate.run_id == "run-tampered" + ) + tampered.state_path.write_text('{"broken":', encoding="utf-8") + + for args in (("--list-trees", "--json"), ("--list-targets", "--trees", "--json")): + with self.subTest(args=args): + buffer = StringIO() + with redirect_stdout(buffer): + code = runtime.dispatch(parse_route(list(args), env={})) + + self.assertEqual(code, 0) + payload = json.loads(buffer.getvalue()) + self.assertEqual({project["name"] for project in payload["projects"]}, set(feature_dirs)) + self.assertEqual(payload["state_error"]["code"], "runtime_state_integrity_error") + self.assertIn("run-tampered", payload["state_error"]["message"]) + self.assertEqual(payload["warnings"], [payload["state_error"]]) + self.assertFalse(any(project["state_present"] for project in payload["projects"])) + def test_list_trees_json_reports_imported_branch_worktrees_individually(self) -> None: tmpdir = tempfile.TemporaryDirectory() self.addCleanup(tmpdir.cleanup) @@ -289,7 +353,10 @@ def test_show_config_json_reports_additional_service_validation_errors_and_metad code = runtime.dispatch(parse_route(["--show-config", "--json"], env={})) self.assertEqual(code, 0) payload = json.loads(buffer.getvalue()) - self.assertIn("additional service dependency cycle: voice-runtime -> worker -> voice-runtime", payload["additional_service_errors"]) + self.assertIn( + "additional service dependency cycle: voice-runtime -> worker -> voice-runtime", + payload["additional_service_errors"], + ) self.assertTrue(payload["service_dependency_env_section_present"]["voice-runtime"]) self.assertEqual(payload["service_dependency_env_templates"]["voice-runtime"][0]["name"], "VOICE_PUBLIC") diff --git a/tests/python/runtime/test_engine_runtime_commands.py b/tests/python/runtime/test_engine_runtime_commands.py index 9b1af759..81cdd8cf 100644 --- a/tests/python/runtime/test_engine_runtime_commands.py +++ b/tests/python/runtime/test_engine_runtime_commands.py @@ -2,6 +2,7 @@ import os from pathlib import Path +import sys import tempfile import unittest from types import SimpleNamespace @@ -49,6 +50,22 @@ def test_command_env_sets_port_and_overrides(self) -> None: self.assertEqual(env["PORT"], "9000") self.assertEqual(env["A"], "1") self.assertEqual(env["B"], "2") + self.assertEqual(env["ENVCTL_PYTHON_EXECUTABLE"], sys.executable) + + def test_command_env_keeps_running_interpreter_authoritative(self) -> None: + runtime = SimpleNamespace(env={"ENVCTL_PYTHON_EXECUTABLE": "/stale/runtime/python"}) + + with patch( + "envctl_engine.runtime.engine_runtime_commands.sys.executable", + "/opt/envctl/bin/python", + ): + env = command_env( + runtime, + port=9000, + extra={"ENVCTL_PYTHON_EXECUTABLE": "/spoofed/project/python"}, + ) + + self.assertEqual(env["ENVCTL_PYTHON_EXECUTABLE"], "/opt/envctl/bin/python") def test_command_env_strips_github_actions_cleanup_variables(self) -> None: runtime = SimpleNamespace(env={"A": "1"}) @@ -113,12 +130,32 @@ def test_split_command_accepts_relative_executable_from_cwd(self) -> None: script = root / "scripts" / "start-service.sh" script.parent.mkdir(parents=True, exist_ok=True) script.write_text("#!/usr/bin/env sh\n", encoding="utf-8") + script.chmod(0o755) runtime = SimpleNamespace(_command_exists=lambda executable: False) parsed = split_command(runtime, "scripts/start-service.sh {port}", port=8123, cwd=root) self.assertEqual(parsed, ["scripts/start-service.sh", "8123"]) + def test_split_command_rejects_non_executable_file_and_directory(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + file_path = root / "start-service.sh" + file_path.write_text("#!/usr/bin/env sh\n", encoding="utf-8") + directory_path = root / "service-dir" + directory_path.mkdir() + runtime = SimpleNamespace(_command_exists=lambda _executable: True) + + for command in (str(file_path), str(directory_path)): + with ( + self.subTest(command=command), + self.assertRaisesRegex( + RuntimeError, + "Resolved command executable not found", + ), + ): + split_command(runtime, command, cwd=root) + def test_default_python_executable_falls_back_to_python3(self) -> None: runtime = SimpleNamespace(env={}, _command_exists=lambda executable: False) diff --git a/tests/python/runtime/test_engine_runtime_dispatch.py b/tests/python/runtime/test_engine_runtime_dispatch.py index ec0b6156..27d5a436 100644 --- a/tests/python/runtime/test_engine_runtime_dispatch.py +++ b/tests/python/runtime/test_engine_runtime_dispatch.py @@ -1,29 +1,40 @@ from __future__ import annotations import importlib -from pathlib import Path +import tempfile +import threading import unittest -from io import StringIO from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path from types import SimpleNamespace -import tempfile from unittest.mock import patch +from envctl_engine.state.repository import RuntimeStateRepository + REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" dispatch_module = importlib.import_module("envctl_engine.runtime.engine_runtime_dispatch") +command_policy_module = importlib.import_module("envctl_engine.runtime.command_policy") +lifecycle_lease_module = importlib.import_module("envctl_engine.runtime.lifecycle_operation_lease") utility_module = importlib.import_module("envctl_engine.runtime.utility_command_support") dispatch = dispatch_module.dispatch dispatch_command = dispatch_module.dispatch_command class EngineRuntimeDispatchTests(unittest.TestCase): + def _runtime_root(self) -> Path: + temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(temporary_directory.cleanup) + return Path(temporary_directory.name) + def test_dispatch_entry_configures_probe_debug_recorder_and_route_events(self) -> None: events: list[tuple[str, dict[str, object]]] = [] calls: list[str] = [] route = SimpleNamespace(command="start", mode="main") runtime = SimpleNamespace( + runtime_root=self._runtime_root(), process_probe=None, _build_process_probe_backend=lambda: calls.append("probe_backend") or "backend", _effective_start_mode=lambda value: calls.append(f"effective:{value.command}") or "trees", @@ -87,6 +98,213 @@ def test_dispatch_entry_uses_route_mode_for_non_startup_commands(self) -> None: self.assertEqual(code, 4) self.assertTrue(all(payload["effective_mode"] == "trees" for _event, payload in events)) + def test_exclusive_operation_commands_follow_mutating_command_policy_sets(self) -> None: + expected = { + *command_policy_module.LIFECYCLE_CLEANUP_COMMANDS, + *command_policy_module.STARTUP_COMMANDS, + "blast-worktree", + "delete-worktree", + "ensure-worktree", + "resume", + "self-destruct-worktree", + } + + self.assertEqual(dispatch_module._EXCLUSIVE_OPERATION_COMMANDS, frozenset(expected)) + self.assertIn("import", expected) + self.assertIn("self-destruct-worktree", expected) + self.assertNotIn("test-focused", expected) + self.assertNotIn("ship", expected) + + def test_nested_lifecycle_dispatch_is_reentrant_for_owning_runtime_thread(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + commands: list[str] = [] + runtime = SimpleNamespace( + runtime_root=self._runtime_root(), + process_probe=None, + _build_process_probe_backend=lambda: "backend", + _effective_start_mode=lambda route: route.mode, + _configure_debug_recorder=lambda _route: None, + _emit=lambda event, **payload: events.append((event, payload)), + ) + outer_route = SimpleNamespace(command="start", mode="trees") + nested_route = SimpleNamespace(command="stop", mode="trees") + + def command_dispatch(_runtime: object, route: object) -> int: + command = str(getattr(route, "command", "")) + commands.append(command) + if command == "start": + return dispatch(runtime, nested_route) + return 7 + + with ( + patch("envctl_engine.runtime.engine_runtime_dispatch.ProcessProbe", return_value="probe"), + patch( + "envctl_engine.runtime.engine_runtime_dispatch.dispatch_command", + side_effect=command_dispatch, + ), + ): + code = dispatch(runtime, outer_route) + + self.assertEqual(code, 7) + self.assertEqual(commands, ["start", "stop"]) + self.assertFalse(any(event == "lifecycle.operation.busy" for event, _payload in events)) + self.assertIsNone(getattr(runtime, "_lifecycle_operation_owner", None)) + self.assertEqual(getattr(runtime, "_lifecycle_operation_depth", 0), 0) + + def test_concurrent_lifecycle_dispatch_on_another_thread_fails_busy_without_overlap(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + commands: list[str] = [] + entered = threading.Event() + release = threading.Event() + results: list[int] = [] + errors: list[BaseException] = [] + runtime = SimpleNamespace( + runtime_root=self._runtime_root(), + process_probe=None, + _build_process_probe_backend=lambda: "backend", + _effective_start_mode=lambda route: route.mode, + _configure_debug_recorder=lambda _route: None, + _emit=lambda event, **payload: events.append((event, payload)), + ) + outer_route = SimpleNamespace(command="start", mode="trees") + concurrent_route = SimpleNamespace(command="stop", mode="trees") + + def command_dispatch(_runtime: object, route: object) -> int: + command = str(getattr(route, "command", "")) + commands.append(command) + if command == "start": + entered.set() + if not release.wait(timeout=2): + raise AssertionError("timed out waiting to release lifecycle operation") + return 0 + + def run_outer() -> None: + try: + results.append(dispatch(runtime, outer_route)) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + with ( + patch("envctl_engine.runtime.engine_runtime_dispatch.ProcessProbe", return_value="probe"), + patch( + "envctl_engine.runtime.engine_runtime_dispatch.dispatch_command", + side_effect=command_dispatch, + ), + ): + thread = threading.Thread(target=run_outer, name="envctl-dispatch-lock-test") + thread.start() + try: + self.assertTrue(entered.wait(timeout=2)) + output = StringIO() + with redirect_stdout(output): + concurrent_code = dispatch(runtime, concurrent_route) + finally: + release.set() + thread.join(timeout=2) + + self.assertFalse(thread.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(results, [0]) + self.assertEqual(concurrent_code, 1) + self.assertEqual(commands, ["start"]) + self.assertIn("Another envctl lifecycle operation is already active", output.getvalue()) + self.assertTrue(any(event == "lifecycle.operation.busy" for event, _payload in events)) + + def test_committed_start_releases_lease_before_interactive_wait_so_stop_can_run(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + commands: list[str] = [] + interactive_wait_entered = threading.Event() + release_interactive_wait = threading.Event() + results: list[int] = [] + runtime = SimpleNamespace( + runtime_root=self._runtime_root(), + process_probe=None, + _build_process_probe_backend=lambda: "backend", + _effective_start_mode=lambda route: route.mode, + _configure_debug_recorder=lambda _route: None, + _emit=lambda event, **payload: events.append((event, payload)), + ) + start_route = SimpleNamespace(command="start", mode="trees") + stop_route = SimpleNamespace(command="stop", mode="trees") + + def command_dispatch(_runtime: object, route: object) -> int: + command = str(getattr(route, "command", "")) + commands.append(command) + if command == "start": + released = lifecycle_lease_module.release_lifecycle_operation(runtime) + self.assertTrue(released) + interactive_wait_entered.set() + self.assertTrue(release_interactive_wait.wait(timeout=2)) + return 0 + + def run_start() -> None: + results.append(dispatch(runtime, start_route)) + + with ( + patch("envctl_engine.runtime.engine_runtime_dispatch.ProcessProbe", return_value="probe"), + patch( + "envctl_engine.runtime.engine_runtime_dispatch.dispatch_command", + side_effect=command_dispatch, + ), + ): + thread = threading.Thread(target=run_start, name="envctl-interactive-start-test") + thread.start() + try: + self.assertTrue(interactive_wait_entered.wait(timeout=2)) + stop_code = dispatch(runtime, stop_route) + finally: + release_interactive_wait.set() + thread.join(timeout=2) + + self.assertFalse(thread.is_alive()) + self.assertEqual(results, [0]) + self.assertEqual(stop_code, 0) + self.assertEqual(commands, ["start", "stop"]) + self.assertFalse(any(event == "lifecycle.operation.busy" for event, _payload in events)) + + def test_operation_lock_rejects_retargeted_repository_root_before_external_write(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + runtime_root.mkdir(parents=True) + legacy_root.mkdir() + repository = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + external = Path(tmpdir) / "external" + external.mkdir() + sentinel = external / "sentinel.txt" + sentinel.write_text("untouched", encoding="utf-8") + runtime_root.rmdir() + runtime_root.symlink_to(external, target_is_directory=True) + runtime = SimpleNamespace( + runtime_root=runtime_root, + state_repository=repository, + process_probe=None, + _build_process_probe_backend=lambda: "backend", + _effective_start_mode=lambda route: route.mode, + _configure_debug_recorder=lambda _route: None, + _emit=lambda *_args, **_kwargs: None, + ) + + with ( + patch("envctl_engine.runtime.engine_runtime_dispatch.ProcessProbe", return_value="probe"), + patch( + "envctl_engine.runtime.engine_runtime_dispatch.dispatch_command", + side_effect=AssertionError("command must not run"), + ), + self.assertRaisesRegex(RuntimeError, "symlink"), + ): + dispatch(runtime, SimpleNamespace(command="start", mode="trees")) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "untouched") + self.assertEqual(list(external.iterdir()), [sentinel]) + def test_list_commands_dispatch_prints_supported_commands(self) -> None: runtime = SimpleNamespace() route = SimpleNamespace(command="list-commands", mode="main") diff --git a/tests/python/runtime/test_engine_runtime_event_support.py b/tests/python/runtime/test_engine_runtime_event_support.py index c0812ebe..04db99b7 100644 --- a/tests/python/runtime/test_engine_runtime_event_support.py +++ b/tests/python/runtime/test_engine_runtime_event_support.py @@ -53,6 +53,32 @@ def test_emit_records_event_and_notifies_listener(self) -> None: self.assertEqual(received[0][0], "ui.input.submit") self.assertIn("command_hash", received[0][1]) + def test_emit_disables_failed_debug_recorder_without_interrupting_lifecycle(self) -> None: + received: list[str] = [] + + class FailedRecorder: + def record(self, *_args: object, **_kwargs: object) -> None: + raise OSError("debug disk unavailable") + + runtime = SimpleNamespace( + _active_command_id="cmd-1", + _debug_hash_salt="salt", + _emit_lock=threading.Lock(), + events=[], + _debug_recorder=FailedRecorder(), + _debug_recorder_failure=None, + _emit_listeners=[lambda event, _payload: received.append(event)], + env={}, + config=SimpleNamespace(raw={}), + ) + + emit(runtime, "service.start", project="Main") + + self.assertEqual(runtime.events[0]["event"], "service.start") + self.assertEqual(received, ["service.start"]) + self.assertIsNone(runtime._debug_recorder) + self.assertEqual(runtime._debug_recorder_failure, "debug disk unavailable") + def test_debug_should_auto_pack_policies(self) -> None: runtime = SimpleNamespace(env={"ENVCTL_DEBUG_AUTO_PACK": "anomaly"}, config=SimpleNamespace(raw={})) diff --git a/tests/python/runtime/test_engine_runtime_lifecycle_support.py b/tests/python/runtime/test_engine_runtime_lifecycle_support.py index 78647f16..6ca190ec 100644 --- a/tests/python/runtime/test_engine_runtime_lifecycle_support.py +++ b/tests/python/runtime/test_engine_runtime_lifecycle_support.py @@ -36,12 +36,17 @@ def test_release_requirement_ports_releases_enabled_final_ports(self) -> None: project="Main", db={"enabled": True, "final": 5432}, redis={"enabled": False, "final": 6379}, + supabase={ + "enabled": True, + "final": 15432, + "resources": {"db": 15432, "api": 15433}, + }, n8n={"enabled": True, "final": 5678}, ) release_requirement_ports(runtime, requirements) - self.assertEqual(released, [5432, 5678]) + self.assertEqual(released, [5432, 5678, 15432, 15433]) def test_release_requirement_ports_skips_external_dependency_ports(self) -> None: released: list[int] = [] @@ -108,10 +113,16 @@ def test_terminate_services_from_state_releases_ports_for_terminated_services(se def test_terminate_service_record_prefers_process_group_termination(self) -> None: calls: list[tuple[str, int]] = [] + live_pids = {3210} runtime = SimpleNamespace( _emit=lambda *args, **kwargs: None, process_runner=SimpleNamespace( - terminate_process_group=lambda pid, **kwargs: calls.append(("group", pid)) or True, + is_pid_running=lambda pid: pid in live_pids, + terminate_process_group=lambda pid, **kwargs: ( + calls.append(("group", pid)), + live_pids.discard(pid), + True, + )[-1], terminate=lambda pid, **kwargs: calls.append(("single", pid)) or True, ), ) @@ -129,6 +140,7 @@ def test_terminate_service_record_prefers_process_group_termination(self) -> Non def test_terminate_service_record_accepts_verified_listener_child_ownership(self) -> None: calls: list[tuple[str, int]] = [] skips: list[dict[str, object]] = [] + live_pids = {111, 222} def pid_owns_port(pid: int, port: int) -> bool: return pid == 222 and port == 9000 @@ -136,8 +148,13 @@ def pid_owns_port(pid: int, port: int) -> bool: runtime = SimpleNamespace( _emit=lambda event, **payload: skips.append(payload) if event == "cleanup.skip" else None, process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, pid_owns_port=pid_owns_port, - terminate_process_group=lambda pid, **kwargs: calls.append(("group", pid)) or True, + terminate_process_group=lambda pid, **kwargs: ( + calls.append(("group", pid)), + live_pids.discard(pid), + True, + )[-1], terminate=lambda pid, **kwargs: calls.append(("single", pid)) or True, ), ) @@ -150,7 +167,7 @@ def pid_owns_port(pid: int, port: int) -> bool: ) self.assertTrue(terminated) - self.assertEqual(calls, [("group", 111)]) + self.assertEqual(calls, [("group", 111), ("group", 222)]) self.assertEqual(skips, []) def test_blast_worktree_before_delete_updates_state_and_emits_finish(self) -> None: @@ -366,7 +383,10 @@ def _resolve_proc_path(path_obj: Path, strict: bool = False) -> Path: # noqa: A return path_obj with ( - patch("envctl_engine.runtime.engine_runtime_lifecycle_support.Path.iterdir", return_value=[proc_root / "111", proc_root / "222"]), + patch( + "envctl_engine.runtime.engine_runtime_lifecycle_support.Path.iterdir", + return_value=[proc_root / "111", proc_root / "222"], + ), patch("pathlib.Path.resolve", autospec=True, side_effect=_resolve_proc_path), patch( "envctl_engine.runtime.engine_runtime_lifecycle_support._remove_tree_containers", diff --git a/tests/python/runtime/test_engine_runtime_plan_resume_startup.py b/tests/python/runtime/test_engine_runtime_plan_resume_startup.py index 1c7b3717..1a239fd6 100644 --- a/tests/python/runtime/test_engine_runtime_plan_resume_startup.py +++ b/tests/python/runtime/test_engine_runtime_plan_resume_startup.py @@ -100,12 +100,14 @@ def fake_start_project_context(*, context, mode, route, run_id): # noqa: ANN001 with ( patch.object(engine, "_try_load_existing_state", return_value=state), patch.object(engine, "_resume", return_value=1) as resume_mock, + patch.object(engine, "_terminate_services_from_state", return_value=set()) as terminate_mock, patch.object(engine, "_start_project_context", side_effect=fake_start_project_context), ): code = engine.dispatch(route) self.assertEqual(code, 0) self.assertEqual(resume_mock.call_count, 1) + self.assertEqual(terminate_mock.call_count, 1) self.assertEqual(started_projects, ["feature-a-1"]) def test_plan_auto_resumes_existing_run_when_selected_projects_are_subset_of_state(self) -> None: @@ -282,6 +284,7 @@ def test_plan_no_resume_flag_disables_auto_resume(self) -> None: with ( patch.object(engine, "_try_load_existing_state", return_value=state), patch.object(engine, "_resume", return_value=0) as resume_mock, + patch.object(engine, "_terminate_services_from_state", return_value=set()) as terminate_mock, patch.object(engine, "_start_project_context", side_effect=RuntimeError("expected startup path")), ): out = StringIO() @@ -290,6 +293,7 @@ def test_plan_no_resume_flag_disables_auto_resume(self) -> None: self.assertEqual(code, 1) self.assertEqual(resume_mock.call_count, 0) + self.assertEqual(terminate_mock.call_count, 1) self.assertIn("expected startup path", out.getvalue()) def test_exact_tree_plan_match_reuses_prior_run_id(self) -> None: @@ -339,4 +343,3 @@ def fake_result(context: ProjectContext) -> ProjectStartupResult: assert second_state is not None self.assertEqual(second_state.run_id, first_state.run_id) self.assertNotEqual(first_engine._current_session_id(), second_session_id) - diff --git a/tests/python/runtime/test_engine_runtime_real_startup.py b/tests/python/runtime/test_engine_runtime_real_startup.py index e65fcd7a..00310d1a 100644 --- a/tests/python/runtime/test_engine_runtime_real_startup.py +++ b/tests/python/runtime/test_engine_runtime_real_startup.py @@ -253,10 +253,11 @@ def capture_emit(event: str, **payload: object) -> None: rendered = out.getvalue() self.assertIn("\x1b]8;;file://", rendered) visible = strip_ansi(rendered) - self.assertIn("backend listener not detected", visible) + self.assertIn("backend is missing a required executable or module", visible) self.assertIn("log_path:", visible) failure_events = [payload for event, payload in captured_events if event == "service.failure"] self.assertTrue(failure_events) + self.assertEqual(failure_events[0]["failure_class"], "dependency_missing") detail = str(failure_events[0]["detail"]) self.assertIn("log_path:", detail) self.assertNotIn("\x1b]8;;", detail) diff --git a/tests/python/runtime/test_engine_runtime_resume_startup.py b/tests/python/runtime/test_engine_runtime_resume_startup.py index b7a62022..8bc3e98c 100644 --- a/tests/python/runtime/test_engine_runtime_resume_startup.py +++ b/tests/python/runtime/test_engine_runtime_resume_startup.py @@ -214,7 +214,9 @@ def fake_result(context: ProjectContext) -> ProjectStartupResult: first_engine = PythonEngineRuntime(self._config(repo, runtime), env={}) first_engine._reconcile_state_truth = lambda _state: [] # type: ignore[assignment] - with patch.object(first_engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"])): + with patch.object( + first_engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"]) + ): first_code = first_engine.dispatch(parse_route(["--plan", "feature-a", "--batch"], env={})) self.assertEqual(first_code, 0) first_state = first_engine.state_repository.load_latest(mode="trees", strict_mode_match=True) @@ -223,10 +225,20 @@ def fake_result(context: ProjectContext) -> ProjectStartupResult: second_engine = PythonEngineRuntime(self._config(repo, runtime, extra={"BACKEND_DIR": "api"}), env={}) second_engine._reconcile_state_truth = lambda _state: [] # type: ignore[assignment] - with patch.object(second_engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"])): + with ( + patch.object( + second_engine, + "_terminate_services_from_state", + return_value=set(), + ) as terminate_mock, + patch.object( + second_engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"]) + ), + ): second_code = second_engine.dispatch(parse_route(["--plan", "feature-a", "--batch"], env={})) self.assertEqual(second_code, 0) + self.assertEqual(terminate_mock.call_count, 1) second_state = second_engine.state_repository.load_latest(mode="trees", strict_mode_match=True) self.assertIsNotNone(second_state) assert second_state is not None @@ -242,7 +254,9 @@ def test_project_root_change_forces_fresh_run(self) -> None: (tree / "frontend").mkdir(parents=True, exist_ok=True) engine = PythonEngineRuntime(self._config(repo, runtime), env={}) context = engine._discover_projects(mode="trees")[0] - wrong_root_context = ProjectContext(name=context.name, root=repo / "trees" / "feature-a" / "99", ports=context.ports) + wrong_root_context = ProjectContext( + name=context.name, root=repo / "trees" / "feature-a" / "99", ports=context.ports + ) metadata = startup_support.build_startup_identity_metadata( engine, runtime_mode="trees", @@ -287,7 +301,9 @@ def fake_result(selected_context: ProjectContext) -> ProjectStartupResult: ) engine._reconcile_state_truth = lambda _state: [] # type: ignore[assignment] - with patch.object(engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"])): + with patch.object( + engine, "_start_project_context", side_effect=lambda **kwargs: fake_result(kwargs["context"]) + ): code = engine.dispatch(parse_route(["--plan", "feature-a", "--batch"], env={})) self.assertEqual(code, 0) @@ -324,6 +340,9 @@ def test_resume_rejects_state_without_resumable_services(self) -> None: with ( patch.object(engine, "_try_load_existing_state", return_value=state), patch.object(engine, "_run_interactive_dashboard_loop", return_value=0) as loop_mock, + patch( + "envctl_engine.startup.resume_orchestrator.release_lifecycle_operation" + ) as release_lifecycle_operation_mock, patch("sys.stdin.isatty", return_value=True), patch("sys.stdout.isatty", return_value=True), patch.dict(os.environ, {"TERM": "xterm-256color"}, clear=False), @@ -333,6 +352,7 @@ def test_resume_rejects_state_without_resumable_services(self) -> None: self.assertEqual(code, 1) self.assertEqual(loop_mock.call_count, 0) + release_lifecycle_operation_mock.assert_not_called() self.assertIn("No active services to resume.", output.getvalue()) def test_dashboard_interactive_flag_enters_loop(self) -> None: @@ -441,4 +461,3 @@ def test_resume_batch_flag_skips_interactive_loop(self) -> None: self.assertEqual(code, 0) self.assertEqual(loop_mock.call_count, 0) - diff --git a/tests/python/runtime/test_engine_runtime_service_truth.py b/tests/python/runtime/test_engine_runtime_service_truth.py index a02e39b0..24d04dcf 100644 --- a/tests/python/runtime/test_engine_runtime_service_truth.py +++ b/tests/python/runtime/test_engine_runtime_service_truth.py @@ -16,6 +16,7 @@ listener_pids_for_port, rebind_stale_service_pid, refresh_service_listener_pids, + service_listener_failure_class, service_listener_failure_detail, service_truth_fallback_enabled, service_truth_status, @@ -34,6 +35,7 @@ def __init__(self) -> None: self.listener_pids_result: list[int] = [] self.process_tree_listener_pids_result: list[int] = [] self.running_result = True + self.wait_for_port_calls: list[tuple[int, float]] = [] def wait_for_pid_port( self, @@ -49,6 +51,7 @@ def wait_for_pid_port( def wait_for_port(self, port: int, *, host: str = "127.0.0.1", timeout: float = 30.0) -> bool: # noqa: ARG002 _ = host + self.wait_for_port_calls.append((port, timeout)) return self.wait_for_port_result def find_pid_listener_port(self, pid: int, requested_port: int, *, max_delta: int) -> int | None: # noqa: ARG002 @@ -120,6 +123,16 @@ def test_service_listener_failure_detail_reports_startup_progress_without_callin self.assertIn("startup still in progress: INFO: Waiting for application startup.", detail) self.assertNotIn("log: INFO: Waiting for application startup.", detail) + def test_service_listener_failure_class_recognizes_missing_nested_executable(self) -> None: + detail = ( + "process 2304354 exited; log_path: /tmp/voice-runtime.txt; " + "log: scripts/envctl/start-voice-runtime.sh: line 20: exec: python: not found" + ) + + self.assertEqual(service_listener_failure_class(detail), "dependency_missing") + self.assertEqual(service_listener_failure_class("process 42 exited"), "process_exited") + self.assertEqual(service_listener_failure_class("listener timed out"), "listener_not_detected") + def test_wait_for_service_listener_uses_port_probe_fallback(self) -> None: events: list[tuple[str, dict[str, object]]] = [] runner = _RunnerStub() @@ -138,6 +151,25 @@ def test_wait_for_service_listener_uses_port_probe_fallback(self) -> None: self.assertTrue(ok) self.assertEqual(events[0][0], "service.bind.port_fallback") self.assertEqual(events[0][1]["service"], "Main Backend") + self.assertEqual(runner.wait_for_port_calls, [(8000, 2.0)]) + + def test_wait_for_service_listener_never_falls_back_after_launch_pid_exits(self) -> None: + runner = _RunnerStub() + runner.running_result = False + runner.wait_for_port_result = True + runner.process_tree_probe_supported = False + runtime = SimpleNamespace( + process_runner=runner, + config=SimpleNamespace(runtime_truth_mode="auto"), + _listener_probe_supported=True, + _service_listener_timeout=lambda: 300.0, + _emit=lambda *_args, **_kwargs: None, + ) + + ok = wait_for_service_listener(runtime, 2304354, 8251, service_name="voice-runtime") + + self.assertFalse(ok) + self.assertEqual(runner.wait_for_port_calls, []) def test_wait_for_service_listener_rejects_generic_port_when_process_tree_probe_is_available(self) -> None: events: list[tuple[str, dict[str, object]]] = [] @@ -236,8 +268,7 @@ def wait_for_pid_port( self.wait_calls += 1 if self.wait_calls == 2: self.log_path.write_text( - "INFO: Waiting for application startup.\n" - "INFO: Application startup complete.\n", + "INFO: Waiting for application startup.\nINFO: Application startup complete.\n", encoding="utf-8", ) return True @@ -309,9 +340,7 @@ def wait_for_pid_port( ) self.assertEqual(detected, 8000) - self.assertTrue( - any(event == "service.startup.progress.listener_accepted" for event, _payload in events) - ) + self.assertTrue(any(event == "service.startup.progress.listener_accepted" for event, _payload in events)) self.assertFalse(any(event == "service.startup.progress.timeout" for event, _payload in events)) self.assertFalse(any(event == "service.bind.actual.discovered" for event, _payload in events)) diff --git a/tests/python/runtime/test_engine_runtime_startup_support.py b/tests/python/runtime/test_engine_runtime_startup_support.py index ca8a7390..3c808f71 100644 --- a/tests/python/runtime/test_engine_runtime_startup_support.py +++ b/tests/python/runtime/test_engine_runtime_startup_support.py @@ -61,17 +61,15 @@ def _reuse_runtime( "backend": backend_enabled, "frontend": frontend_enabled, }.get(str(service_name).strip().lower(), False), - requirement_enabled_for_mode=lambda _mode, requirement_name: str(requirement_name).strip().lower() - in enabled_dependencies, + requirement_enabled_for_mode=lambda _mode, requirement_name: ( + str(requirement_name).strip().lower() in enabled_dependencies + ), ) return SimpleNamespace( env=dict(external_env or {}), config=config, _try_load_existing_state=lambda **_kwargs: state, - _project_name_from_service=lambda name: str(name) - .replace(" Backend", "") - .replace(" Frontend", "") - .strip(), + _project_name_from_service=lambda name: str(name).replace(" Backend", "").replace(" Frontend", "").strip(), ) def test_effective_start_mode_switches_main_to_trees_for_setup_worktree(self) -> None: @@ -144,6 +142,61 @@ def test_run_reuse_accepts_exact_project_roots_and_startup_identity_match(self) self.assertEqual(decision.reason, "exact_match") self.assertIs(decision.candidate_state, state) + def test_run_reuse_forwards_selected_project_names_to_state_lookup(self) -> None: + context = ProjectContext(name="feature-a", root=Path("/repo/trees/feature-a"), ports={}) + state = RunState( + run_id="run-a", + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/repo/trees/feature-a/backend", + ) + }, + metadata={"project_roots": {"feature-a": str(context.root)}}, + ) + runtime = self._reuse_runtime(state=state) + calls: list[dict[str, object]] = [] + runtime._try_load_existing_state = lambda **kwargs: calls.append(kwargs) or state + + startup_support.evaluate_run_reuse( + runtime, + runtime_mode="trees", + route=Route(command="start", mode="trees"), + contexts=[context], + ) + + self.assertEqual( + calls, + [{"mode": "trees", "strict_mode_match": True, "project_names": ["feature-a"]}], + ) + + def test_explicit_no_resume_keeps_matching_candidate_for_safe_replacement(self) -> None: + context = ProjectContext(name="feature-a", root=Path("/repo/trees/feature-a"), ports={}) + state = RunState( + run_id="run-a", + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/repo/trees/feature-a/backend", + ) + }, + metadata={"project_roots": {"feature-a": str(context.root)}}, + ) + decision = startup_support.evaluate_run_reuse( + self._reuse_runtime(state=state), + runtime_mode="trees", + route=Route(command="start", mode="trees", flags={"no_resume": True}), + contexts=[context], + ) + + self.assertEqual(decision.decision_kind, "fresh_run") + self.assertEqual(decision.reason, "explicit_no_resume") + self.assertIs(decision.candidate_state, state) + def test_run_reuse_rejects_project_root_mismatch_even_when_names_match(self) -> None: context = ProjectContext(name="feature-a-1", root=Path("/repo/trees/feature-a/1"), ports={}) route = Route(command="plan", mode="trees", raw_args=[], passthrough_args=[], projects=["feature-a"], flags={}) @@ -571,6 +624,182 @@ def test_discover_projects_and_reserve_project_ports(self) -> None: reserve_project_ports(runtime, context) self.assertEqual(events[-1][0], "port.reserved") + def test_no_infra_reserves_no_application_or_dependency_ports(self) -> None: + reserve_calls: list[tuple[int, str]] = [] + events: list[tuple[str, dict[str, object]]] = [] + plans = { + name: PortPlan(project="Main", requested=port, assigned=port, final=port, source="fixed") + for name, port in { + "backend": 8000, + "frontend": 9000, + "db": 5432, + "redis": 6379, + "n8n": 5678, + "supabase_api": 54321, + "worker": 9100, + }.items() + } + runtime = SimpleNamespace( + port_planner=SimpleNamespace( + reserve_next=lambda assigned, owner: reserve_calls.append((assigned, owner)) or assigned, + max_port=65000, + availability_mode="lock", + update_final_port=lambda plan, reserved, source: None, + ), + _emit=lambda event, **payload: events.append((event, payload)), + _lock_inventory=lambda: [], + _requirement_enabled=lambda *_args, **_kwargs: self.fail("disabled dependencies need no lookup"), + ) + route = Route( + command="start", + mode="main", + flags={"launch_backend": False, "launch_frontend": False, "launch_dependencies": False}, + ) + + reserve_project_ports( + runtime, + ProjectContext(name="Main", root=Path("/repo"), ports=plans), + route=route, + ) + + self.assertEqual(reserve_calls, []) + self.assertEqual( + [payload["service"] for event, payload in events if event == "port.reservation.skipped"], + list(plans), + ) + + def test_disabled_requirements_do_not_reserve_dependency_ports(self) -> None: + reserve_calls: list[tuple[int, str]] = [] + plans = { + "backend": PortPlan("Main", 8000, 8000, 8000, "fixed"), + "db": PortPlan("Main", 5432, 5432, 5432, "fixed"), + "redis": PortPlan("Main", 6379, 6379, 6379, "fixed"), + "n8n": PortPlan("Main", 5678, 5678, 5678, "fixed"), + "supabase_api": PortPlan("Main", 54321, 54321, 54321, "fixed"), + } + runtime = SimpleNamespace( + port_planner=SimpleNamespace( + reserve_next=lambda assigned, owner: reserve_calls.append((assigned, owner)) or assigned, + max_port=65000, + availability_mode="lock", + update_final_port=lambda plan, reserved, source: None, + ), + _emit=lambda *_args, **_kwargs: None, + _lock_inventory=lambda: [], + _requirement_enabled=lambda *_args, **_kwargs: False, + ) + + reserve_project_ports( + runtime, + ProjectContext(name="Main", root=Path("/repo"), ports=plans), + route=Route(command="start", mode="main", flags={"runtime_scope": "entire-system"}), + ) + + self.assertEqual(reserve_calls, [(8000, "Main:backend")]) + + def test_setup_worktree_reserves_dependencies_for_effective_trees_mode(self) -> None: + reserve_calls: list[tuple[int, str]] = [] + enabled_modes: list[str] = [] + runtime = SimpleNamespace( + port_planner=SimpleNamespace( + reserve_next=lambda assigned, owner: reserve_calls.append( + (assigned, owner) + ) + or assigned, + max_port=65000, + availability_mode="lock", + update_final_port=lambda plan, reserved, source: None, + ), + _emit=lambda *_args, **_kwargs: None, + _lock_inventory=lambda: [], + _setup_worktree_requested=lambda _route: True, + _requirement_enabled=lambda _dependency, *, mode, route: ( + enabled_modes.append(mode) or mode == "trees" + ), + ) + route = Route( + command="start", + mode="main", + flags={"setup_worktree": True, "runtime_scope": "entire-system"}, + ) + + reserve_project_ports( + runtime, + ProjectContext( + name="FeatureA", + root=Path("/repo/trees/FeatureA"), + ports={"db": PortPlan("FeatureA", 5432, 5432, 5432, "fixed")}, + ), + route=route, + ) + + self.assertEqual(enabled_modes, ["trees"]) + self.assertEqual(reserve_calls, [(5432, "FeatureA:db")]) + + def test_port_reservations_match_the_selected_runtime_surface(self) -> None: + voice = SimpleNamespace( + name="voice", + depends_on=(), + enabled_for_project_root=lambda _mode, _root: True, + ) + cases = ( + ("frontend only", {"runtime_scope": "frontend"}, {"frontend", "db"}), + ("backend only", {"runtime_scope": "backend"}, {"backend", "db"}), + ("dependencies only", {"runtime_scope": "dependencies"}, {"db"}), + ( + "no infra", + {"launch_backend": False, "launch_frontend": False, "launch_dependencies": False}, + set(), + ), + ("selected additional service", {"services": ["Main Voice"]}, {"voice", "db"}), + ) + + for label, flags, expected_services in cases: + with self.subTest(label): + reserve_calls: list[tuple[int, str]] = [] + plans = { + name: PortPlan("Main", port, port, port, "fixed") + for name, port in { + "backend": 8000, + "frontend": 9000, + "voice": 9100, + "db": 5432, + "redis": 6379, + "n8n": 5678, + "supabase_api": 54321, + }.items() + } + runtime = SimpleNamespace( + config=SimpleNamespace( + additional_services=(voice,), + all_app_service_names_for_mode=lambda _mode, project_root=None: ( + "backend", + "frontend", + "voice", + ), + ), + port_planner=SimpleNamespace( + reserve_next=lambda assigned, owner: reserve_calls.append((assigned, owner)) or assigned, + max_port=65000, + availability_mode="lock", + update_final_port=lambda plan, reserved, source: None, + ), + _emit=lambda *_args, **_kwargs: None, + _lock_inventory=lambda: [], + _requirement_enabled=lambda requirement, **_kwargs: requirement == "postgres", + ) + + reserve_project_ports( + runtime, + ProjectContext(name="Main", root=Path("/repo"), ports=plans), + route=Route(command="start", mode="main", flags=flags), + ) + + self.assertEqual( + {owner.removeprefix("Main:") for _port, owner in reserve_calls}, + expected_services, + ) + def test_reserve_project_ports_releases_matching_restart_lock_before_reuse(self) -> None: events: list[tuple[str, dict[str, object]]] = [] released: list[tuple[int, str | None]] = [] @@ -627,7 +856,9 @@ def update_final_port(plan: PortPlan, reserved: int, source: str) -> None: ) context = ProjectContext(name="Main", root=Path("/repo"), ports={"backend": port_plan}) - reserve_project_ports(runtime, context, route=Route(command="start", mode="main", flags={"interactive_command": True})) + reserve_project_ports( + runtime, context, route=Route(command="start", mode="main", flags={"interactive_command": True}) + ) rebound_payloads = [payload for event, payload in events if event == "port.rebound"] self.assertEqual(len(rebound_payloads), 1) diff --git a/tests/python/runtime/test_engine_runtime_state_lookup.py b/tests/python/runtime/test_engine_runtime_state_lookup.py index 4e3eba43..3f9b5c2a 100644 --- a/tests/python/runtime/test_engine_runtime_state_lookup.py +++ b/tests/python/runtime/test_engine_runtime_state_lookup.py @@ -18,19 +18,29 @@ class EngineRuntimeStateLookupTests(unittest.TestCase): def test_try_load_existing_state_emits_fingerprint_after_reload(self) -> None: state = RunState(run_id="run-1", mode="main") events: list[tuple[str, dict[str, object]]] = [] + load_calls: list[dict[str, object]] = [] runtime = SimpleNamespace( - state_repository=SimpleNamespace(load_latest=lambda **kwargs: state), + state_repository=SimpleNamespace(load_latest=lambda **kwargs: load_calls.append(kwargs) or state), _emit=lambda event, **payload: events.append((event, payload)), _state_fingerprint=lambda loaded: f"fp:{loaded.run_id}", ) - loaded = try_load_existing_state(runtime, mode="main", strict_mode_match=True) + loaded = try_load_existing_state( + runtime, + mode="main", + strict_mode_match=True, + project_names=["Main"], + ) self.assertIs(loaded, state) self.assertEqual( events, [("state.fingerprint.after_reload", {"run_id": "run-1", "state_fingerprint": "fp:run-1"})], ) + self.assertEqual( + load_calls, + [{"mode": "main", "strict_mode_match": True, "project_names": ["Main"]}], + ) def test_state_matches_scope_accepts_missing_scope_and_matches_exact_scope(self) -> None: runtime = SimpleNamespace(config=SimpleNamespace(runtime_scope_id="repo-123")) diff --git a/tests/python/runtime/test_help_topics.py b/tests/python/runtime/test_help_topics.py index b84c8eba..97dc17a9 100644 --- a/tests/python/runtime/test_help_topics.py +++ b/tests/python/runtime/test_help_topics.py @@ -29,6 +29,15 @@ def test_pr_preview_controller_help_topic_is_available(self) -> None: self.assertIn("envctl pr-preview-controller - run the GitHub PR-label preview controller", text) self.assertIn("--command ", text) + def test_stop_help_distinguishes_detach_from_managed_docker_removal(self) -> None: + stop_text = render_command_help(COMMAND_HELP_TOPICS["stop"]) + stop_all_text = render_command_help(COMMAND_HELP_TOPICS["stop-all"]) + + self.assertIn("managed Docker stacks and storage remain reusable", stop_text) + self.assertIn("stop-all --remove-volumes", stop_text) + self.assertIn("releases dependency records and port locks", stop_all_text) + self.assertIn("remove supported managed containers and volumes", stop_all_text) + def test_help_text_for_route_returns_none_for_general_help(self) -> None: self.assertIsNone(help_text_for_route(parse_route(["--help"], env={}))) diff --git a/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py b/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py index d28f7f30..5e840ce4 100644 --- a/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py +++ b/tests/python/runtime/test_lifecycle_cleanup_spinner_integration.py @@ -1,17 +1,26 @@ from __future__ import annotations +import json +import tempfile +import unittest from contextlib import contextmanager, redirect_stdout from io import StringIO from pathlib import Path from types import SimpleNamespace -import unittest from unittest.mock import patch REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.lifecycle_cleanup_orchestrator import LifecycleCleanupOrchestrator +from envctl_engine.runtime.lifecycle_service_termination import ( + terminate_service_record, + terminate_services_from_state, +) +from envctl_engine.shared.ports import PortPlanner from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord +from envctl_engine.state.repository import RuntimeStateRepository +from envctl_engine.state.run_index import StateSelector from envctl_engine.ui.spinner_service import SpinnerPolicy from envctl_engine.ui.target_selector import TargetSelection @@ -19,16 +28,43 @@ class _StateRepoStub: def __init__(self) -> None: self.saved_states: list[RunState] = [] + self.saved_authoritative_project_names: list[list[str] | None] = [] self.purge_calls: list[bool] = [] + self.active_states: list[RunState] = [] + self.deactivated_run_ids: list[str] = [] - def save_selected_stop_state(self, *, state, emit, runtime_map_builder): # noqa: ANN001 + def save_selected_stop_state( # noqa: ANN001 + self, + *, + state, + emit, + runtime_map_builder, + authoritative_project_names=None, + ): _ = emit, runtime_map_builder self.saved_states.append(state) + self.saved_authoritative_project_names.append(authoritative_project_names) + self.active_states = [ + state if active_state.run_id == state.run_id else active_state for active_state in self.active_states + ] return {} def purge(self, *, aggressive: bool = False) -> None: self.purge_calls.append(aggressive) + def load_all(self, *, mode: str | None = None) -> list[RunState]: + return [state for state in self.active_states if mode is None or state.mode == mode] + + def deactivate_runs(self, run_ids: list[str]) -> bool: + selected = set(run_ids) + removed = [state for state in self.active_states if state.run_id in selected] + self.active_states = [state for state in self.active_states if state.run_id not in selected] + self.deactivated_run_ids.extend(state.run_id for state in removed) + return bool(removed) + + def has_active_runs(self) -> bool: + return bool(self.active_states) + class _RuntimeStub: def __init__(self) -> None: @@ -165,6 +201,571 @@ def test_stop_all_tolerates_runtime_without_port_session_hook(self) -> None: self.assertEqual(runtime.state_repository.purge_calls, [False]) self.assertTrue(any(item.get("event") == "cleanup.stop_all" for item in runtime.events)) + def test_stop_all_releases_every_scoped_port_lock_after_last_run_is_removed(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime = _RuntimeStub() + scoped_planner = PortPlanner( + lock_dir=str(Path(tmpdir) / "scope-a"), + availability_checker=lambda _port: True, + ) + foreign_planner = PortPlanner( + lock_dir=str(Path(tmpdir) / "scope-b"), + availability_checker=lambda _port: True, + ) + scoped_planner.reserve_next(8100, owner="old-startup") + foreign_planner.reserve_next(8200, owner="other-repository") + runtime.port_planner = scoped_planner + + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(code, 0) + self.assertEqual(list(scoped_planner.lock_dir.glob("*.lock")), []) + self.assertEqual( + [path.name for path in foreign_planner.lock_dir.glob("*.lock")], + ["8200.lock"], + ) + self.assertEqual(runtime.state_repository.purge_calls, [False]) + + def test_stop_all_purge_failure_preserves_scoped_port_locks(self) -> None: + runtime = _RuntimeStub() + releases: list[str] = [] + runtime.port_planner = SimpleNamespace(release_all=lambda: releases.append("all")) + + with ( + patch.object(runtime.state_repository, "purge", side_effect=OSError("registry unavailable")), + self.assertRaisesRegex(OSError, "registry unavailable"), + ): + LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(releases, []) + + def test_stop_all_terminates_every_independent_active_run_once(self) -> None: + runtime = _RuntimeStub() + runtime.state_repository.active_states = [ + RunState(run_id="run-a", mode="trees"), + RunState(run_id="run-b", mode="trees"), + RunState(run_id="run-a", mode="trees"), + ] + terminated: list[str] = [] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: terminated.append(state.run_id) + ) + + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="trees")) + + self.assertEqual(code, 0) + self.assertEqual(terminated, ["run-a", "run-b"]) + self.assertEqual(runtime.state_repository.purge_calls, [False]) + + def test_plain_stop_all_terminates_active_runs_across_modes(self) -> None: + runtime = _RuntimeStub() + runtime.state_repository.active_states = [ + RunState(run_id="run-main", mode="main"), + RunState(run_id="run-tree", mode="trees"), + ] + terminated: list[str] = [] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: terminated.append(state.run_id) or set() + ) + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route(command="stop-all", mode="main") + ) + + self.assertEqual(code, 0) + self.assertEqual(terminated, ["run-main", "run-tree"]) + self.assertEqual(runtime.state_repository.active_states, []) + + def test_stop_all_trees_preserves_main_registry_alias_and_port_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime_dir = root / "runtime" + repository = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + for run_id, mode, project in ( + ("run-main", "main", "Main"), + ("run-alpha", "trees", "FeatureA"), + ("run-beta", "trees", "FeatureB"), + ): + repository.save_resume_state( + state=RunState( + run_id=run_id, + mode=mode, + services={ + f"{project} Backend": ServiceRecord( + name=f"{project} Backend", + type="backend", + cwd=str(root / project / "backend"), + project=project, + pid=100 + len(project), + ) + }, + metadata={ + "project_names": [project], + "project_roots": {project: str(root / project)}, + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, + ) + + planner = PortPlanner( + lock_dir=str(repository.runtime_root / "locks"), + session_id="main-owner", + availability_checker=lambda _port: True, + ) + main_port = planner.reserve_next(8111, owner="Main:backend") + runtime = _RuntimeStub() + runtime.state_repository = repository # type: ignore[assignment] + runtime._state_lookup_strict_mode_match = lambda _route: True # type: ignore[method-assign] + runtime.port_planner = planner + terminated: list[str] = [] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: terminated.append(state.run_id) or set() + ) + + with patch.object(repository, "purge", wraps=repository.purge) as purge: + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="trees")) + + self.assertEqual(code, 0) + self.assertEqual(terminated, ["run-beta", "run-alpha"]) + purge.assert_not_called() + remaining = repository.load_all() + self.assertEqual([state.run_id for state in remaining], ["run-main"]) + self.assertEqual(repository.load_latest(mode="main", strict_mode_match=True).run_id, "run-main") + current = json.loads(repository.run_state_path().read_text(encoding="utf-8")) + self.assertEqual((current["run_id"], current["mode"]), ("run-main", "main")) + self.assertTrue((planner.lock_dir / f"{main_port}.lock").exists()) + + def test_mode_scoped_stop_all_ignores_tampered_other_mode_and_preserves_it(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repository = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + for run_id, mode, project, pid in ( + ("run-valid", "main", "Main", 101), + ("run-tampered", "trees", "FeatureA", 202), + ): + service_name = f"{project} Backend" + repository.save_resume_state( + state=RunState( + run_id=run_id, + mode=mode, + services={ + service_name: ServiceRecord( + name=service_name, + type="backend", + cwd=f"/tmp/{project}/backend", + project=project, + pid=pid, + ) + }, + metadata={"project_names": [project]}, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, + ) + + candidates = repository.run_index.candidates(StateSelector(mode=None, project_names=())) + tampered = next(candidate for candidate in candidates if candidate.run_id == "run-tampered") + tampered.state_path.write_text('{"broken":', encoding="utf-8") + + runtime = _RuntimeStub() + runtime.state_repository = repository # type: ignore[assignment] + runtime._state_lookup_strict_mode_match = lambda _route: True # type: ignore[method-assign] + termination_calls: list[str] = [] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: termination_calls.append(state.run_id) or set() + ) + + with patch.object(repository, "purge", wraps=repository.purge) as purge: + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(code, 0) + self.assertEqual(termination_calls, ["run-valid"]) + purge.assert_not_called() + remaining = repository.run_index.candidates(StateSelector(mode=None, project_names=())) + self.assertEqual({candidate.run_id for candidate in remaining}, {"run-tampered"}) + self.assertTrue(all(candidate.state_path.exists() for candidate in remaining)) + + def test_targeted_stop_deactivates_no_infra_state_with_only_disabled_requirements(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-no-infra", + mode="trees", + services={}, + requirements={"FeatureA": RequirementsResult(project="FeatureA")}, + metadata={ + "project_names": ["FeatureA"], + "project_roots": {"FeatureA": "/tmp/FeatureA"}, + }, + ) + runtime.state_repository.active_states = [state] + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"batch": True, "runtime_scope": "entire-system"}, + ) + ) + + self.assertEqual(code, 0) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-no-infra"]) + self.assertEqual(runtime.state_repository.active_states, []) + + def test_targeted_stop_removes_only_selected_project_from_shared_no_infra_state(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-no-infra-ab", + mode="trees", + services={}, + requirements={project: RequirementsResult(project=project) for project in ("FeatureA", "FeatureB")}, + metadata={ + "project_names": ["FeatureA", "FeatureB"], + "project_roots": { + "FeatureA": "/tmp/FeatureA", + "FeatureB": "/tmp/FeatureB", + }, + }, + ) + runtime.state_repository.active_states = [state] + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"batch": True, "runtime_scope": "entire-system"}, + ) + ) + + self.assertEqual(code, 0) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(len(runtime.state_repository.saved_states), 1) + saved = runtime.state_repository.saved_states[0] + self.assertEqual(set(saved.requirements), {"FeatureB"}) + self.assertEqual(saved.metadata["project_names"], ["FeatureB"]) + self.assertEqual(saved.metadata["project_roots"], {"FeatureB": "/tmp/FeatureB"}) + self.assertEqual(runtime.state_repository.saved_authoritative_project_names, [["FeatureB"]]) + self.assertEqual(runtime.state_repository.active_states, [saved]) + + def test_targeted_service_stop_preserves_unselected_owner_and_shared_main_requirements(self) -> None: + runtime = _RuntimeStub() + state = RunState( + run_id="run-services-ab", + mode="trees", + services={ + f"{project} Backend": ServiceRecord( + name=f"{project} Backend", + type="backend", + cwd=f"/tmp/{project}", + project=project, + pid=pid, + ) + for project, pid in (("FeatureA", 101), ("FeatureB", 202)) + }, + requirements={ + "Main": RequirementsResult( + project="Main", + components={ + "postgres": { + "enabled": True, + "success": True, + "final": 5432, + } + }, + ) + }, + metadata={ + "project_names": ["FeatureA", "FeatureB"], + "project_roots": { + "FeatureA": "/tmp/FeatureA", + "FeatureB": "/tmp/FeatureB", + }, + "shared_dependencies": True, + }, + ) + runtime.state_repository.active_states = [state] + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"batch": True, "runtime_scope": "backend"}, + ) + ) + + self.assertEqual(code, 0) + saved = runtime.state_repository.saved_states[0] + self.assertEqual(set(saved.services), {"FeatureB Backend"}) + self.assertEqual(set(saved.requirements), {"Main"}) + self.assertTrue(saved.metadata["shared_dependencies"]) + self.assertEqual(saved.metadata["project_names"], ["FeatureB"]) + self.assertEqual(saved.metadata["project_roots"], {"FeatureB": "/tmp/FeatureB"}) + self.assertEqual(runtime.state_repository.saved_authoritative_project_names, [["FeatureB"]]) + + def test_targeted_stop_terminates_and_deactivates_every_shadowed_run_for_project(self) -> None: + runtime = _RuntimeStub() + release_snapshots: list[list[str]] = [] + runtime.port_planner = SimpleNamespace( + release_all=lambda: release_snapshots.append( + [state.run_id for state in runtime.state_repository.active_states] + ) + ) + runtime.state_repository.active_states = [ + RunState( + run_id=run_id, + mode="trees", + services={ + f"FeatureA Backend {run_id}": ServiceRecord( + name=f"FeatureA Backend {run_id}", + type="backend", + cwd=f"/tmp/{run_id}/backend", + project="FeatureA", + pid=pid, + ) + }, + metadata={"project_names": ["FeatureA"]}, + ) + for run_id, pid in (("run-new", 202), ("run-old", 101)) + ] + terminated: list[str] = [] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: terminated.append(state.run_id) + ) + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"batch": True}, + ) + ) + + self.assertEqual(code, 0) + self.assertEqual(terminated, ["run-new", "run-old"]) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-new", "run-old"]) + self.assertEqual(runtime.state_repository.active_states, []) + self.assertEqual(runtime.state_repository.purge_calls, []) + self.assertEqual(release_snapshots, [[]]) + + def test_targeted_stop_failure_preserves_service_and_run_tracking(self) -> None: + runtime = _RuntimeStub() + runtime.port_planner = SimpleNamespace( + release_all=lambda: self.fail("incomplete termination must retain scoped locks") + ) + state = RunState( + run_id="run-a", + mode="trees", + services={ + "FeatureA Backend": ServiceRecord( + name="FeatureA Backend", + type="backend", + cwd="/tmp/FeatureA/backend", + project="FeatureA", + pid=101, + ) + }, + metadata={"project_names": ["FeatureA"]}, + ) + runtime.state_repository.active_states = [state] + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda _state, *, selected_services, **_kwargs: set(selected_services or ()) + ) + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"batch": True}, + ) + ) + + self.assertEqual(code, 1) + self.assertEqual(set(state.services), {"FeatureA Backend"}) + self.assertEqual(runtime.state_repository.saved_states, [state]) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(runtime.state_repository.purge_calls, []) + + def test_implicit_mode_targeted_stop_cleans_every_tree_shadow_with_duplicate_pid(self) -> None: + runtime = _RuntimeStub() + main_state = RunState( + run_id="run-main", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + pid=303, + ) + }, + metadata={"project_names": ["Main"]}, + ) + shadow_states = [ + RunState( + run_id=run_id, + mode="trees", + services={ + f"FeatureA Backend {run_id}": ServiceRecord( + name=f"FeatureA Backend {run_id}", + type="backend", + cwd=f"/tmp/{run_id}/backend", + project="FeatureA", + pid=404, + actual_port=8404, + ) + }, + metadata={"project_names": ["FeatureA"]}, + ) + for run_id in ("run-tree-new", "run-tree-old") + ] + runtime.state_repository.active_states = [main_state, *shadow_states] + runtime._try_load_existing_state = ( # type: ignore[method-assign] + lambda *, mode=None, strict_mode_match=False, project_names=None: ( + shadow_states[0] if mode == "main" and not strict_mode_match and project_names == ["FeatureA"] else None + ) + ) + live_pids = {404} + termination_calls: list[int] = [] + released_ports: list[int] = [] + runtime.port_planner = SimpleNamespace(release=lambda port: released_ports.append(port)) + runtime.process_runner = SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + pid_owns_port=lambda pid, port: pid in live_pids and pid == 404 and port == 8404, + terminate_process_group=lambda pid, **_kwargs: termination_calls.append(pid) or not live_pids.discard(pid), + ) + runtime._terminate_service_record = ( # type: ignore[attr-defined] + lambda service, *, aggressive, verify_ownership: terminate_service_record( + runtime, + service, + aggressive=aggressive, + verify_ownership=verify_ownership, + ) + ) + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, *, selected_services, aggressive, verify_ownership: terminate_services_from_state( + runtime, + state, + selected_services=selected_services, + aggressive=aggressive, + verify_ownership=verify_ownership, + ) + ) + + code = LifecycleCleanupOrchestrator(runtime).execute( + Route( + command="stop", + mode="main", + projects=["FeatureA"], + flags={"batch": True}, + ) + ) + + self.assertEqual(code, 0) + self.assertEqual(termination_calls, [404]) + self.assertEqual(released_ports, [8404, 8404]) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-tree-new", "run-tree-old"]) + self.assertEqual(runtime.state_repository.active_states, [main_state]) + + def test_stop_all_failure_preserves_failed_run_and_never_purges_registry(self) -> None: + runtime = _RuntimeStub() + failed_state = RunState( + run_id="run-failed", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/backend", + project="Main", + pid=101, + ), + "FeatureC Backend": ServiceRecord( + name="FeatureC Backend", + type="backend", + cwd="/tmp/FeatureC/backend", + project="FeatureC", + pid=102, + ), + }, + requirements={ + "shared-main": RequirementsResult(project="Main", db={"enabled": True, "final": 5432}), + "FeatureC": RequirementsResult(project="FeatureC", redis={"enabled": True, "final": 6379}), + }, + ) + successful_state = RunState( + run_id="run-success", + mode="trees", + services={ + "FeatureB Backend": ServiceRecord( + name="FeatureB Backend", + type="backend", + cwd="/tmp/FeatureB/backend", + project="FeatureB", + pid=202, + ) + }, + requirements={ + "FeatureB": RequirementsResult(project="FeatureB", db={"enabled": True, "final": 5433}), + }, + ) + runtime.state_repository.active_states = [failed_state, successful_state] + released_requirements: list[str] = [] + runtime._release_requirement_ports = ( # type: ignore[attr-defined] + lambda requirements: released_requirements.append(requirements.project) + ) + runtime._terminate_services_from_state = ( # type: ignore[method-assign] + lambda state, **_kwargs: {"Main Backend"} if state.run_id == "run-failed" else set() + ) + + runtime._state_lookup_strict_mode_match = lambda _route: True # type: ignore[method-assign] + code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(code, 1) + self.assertEqual(runtime.state_repository.saved_states, [failed_state]) + self.assertEqual(set(failed_state.services), {"Main Backend"}) + self.assertEqual(set(failed_state.requirements), {"shared-main"}) + self.assertEqual(set(released_requirements), {"FeatureC"}) + self.assertEqual(runtime.state_repository.deactivated_run_ids, []) + self.assertEqual(runtime.state_repository.active_states, [failed_state, successful_state]) + self.assertEqual(runtime.state_repository.purge_calls, []) + self.assertTrue(any(event.get("event") == "cleanup.incomplete" for event in runtime.events)) + + runtime._terminate_services_from_state = lambda _state, **_kwargs: set() # type: ignore[method-assign] + retry_code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="main")) + + self.assertEqual(retry_code, 0) + self.assertEqual(released_requirements.count("Main"), 1) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-failed"]) + self.assertEqual(runtime.state_repository.active_states, [successful_state]) + self.assertEqual(runtime.state_repository.purge_calls, []) + + trees_code = LifecycleCleanupOrchestrator(runtime).execute(Route(command="stop-all", mode="trees")) + + self.assertEqual(trees_code, 0) + self.assertEqual(released_requirements.count("FeatureB"), 1) + self.assertEqual(runtime.state_repository.deactivated_run_ids, ["run-failed", "run-success"]) + self.assertEqual(runtime.state_repository.active_states, []) + self.assertEqual(runtime.state_repository.purge_calls, [False]) + def test_stop_selector_miss_does_not_start_spinner(self) -> None: runtime = _RuntimeStub() runtime._try_load_existing_state = lambda *args, **kwargs: RunState( # type: ignore[method-assign] diff --git a/tests/python/runtime/test_lifecycle_dependency_stop.py b/tests/python/runtime/test_lifecycle_dependency_stop.py index 59d176d1..7d1dca6d 100644 --- a/tests/python/runtime/test_lifecycle_dependency_stop.py +++ b/tests/python/runtime/test_lifecycle_dependency_stop.py @@ -1,6 +1,9 @@ from __future__ import annotations import unittest +from pathlib import Path +import tempfile +from types import SimpleNamespace from envctl_engine.runtime.command_router import Route from envctl_engine.runtime.lifecycle_dependency_stop import ( @@ -9,6 +12,8 @@ requirements_have_enabled_components, select_dependency_components_for_stop, ) +from envctl_engine.runtime.lifecycle_requirement_ports import requirement_component_port_owners +from envctl_engine.shared.ports import PortPlanner from envctl_engine.state.models import RequirementsResult, RunState @@ -60,9 +65,11 @@ def test_release_selected_dependency_components_releases_internal_ports_and_prun release_selected_dependency_components( state, {"Main": {"postgres", "redis"}, "Aux": {"postgres"}}, - release_component_ports_fn=lambda component: release_requirement_component_ports( - component, - release_port_fn=released.append, + release_component_ports_fn=lambda _requirements, _dependency_id, component: ( + release_requirement_component_ports( + component, + port_planner=SimpleNamespace(release=released.append), + ) ), ) @@ -86,9 +93,11 @@ def test_release_selected_dependency_components_preserves_projects_with_enabled_ release_selected_dependency_components( state, {"Main": {"redis"}}, - release_component_ports_fn=lambda component: release_requirement_component_ports( - component, - release_port_fn=released.append, + release_component_ports_fn=lambda _requirements, _dependency_id, component: ( + release_requirement_component_ports( + component, + port_planner=SimpleNamespace(release=released.append), + ) ), ) @@ -103,11 +112,53 @@ def test_release_requirement_component_ports_dedupes_positive_final_and_resource release_requirement_component_ports( {"final": 5432, "resources": {"shadow": 15432, "duplicate": 5432, "ignored": 0}}, - release_port_fn=released.append, + port_planner=SimpleNamespace(release=released.append), ) self.assertEqual(released, [5432, 15432]) + def test_selected_dependency_stop_releases_exact_prior_session_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(5432, owner="Main:db") + requirements = RequirementsResult( + project="Main", + db={ + "enabled": True, + "final": 5432, + "port_lock_session": "prior-session", + }, + redis={"enabled": True, "final": 6379}, + ) + state = RunState( + run_id="run-prior", + mode="main", + requirements={"Main": requirements}, + ) + + release_selected_dependency_components( + state, + {"Main": {"postgres"}}, + release_component_ports_fn=lambda owner, dependency_id, component: release_requirement_component_ports( + component, + port_planner=current, + owner_candidates=requirement_component_port_owners(owner, dependency_id), + ), + ) + + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + self.assertFalse(requirements.db.get("enabled", False)) + self.assertTrue(requirements.redis.get("enabled", False)) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/runtime/test_lifecycle_requirement_ports.py b/tests/python/runtime/test_lifecycle_requirement_ports.py index afa5fb5c..34a67a0a 100644 --- a/tests/python/runtime/test_lifecycle_requirement_ports.py +++ b/tests/python/runtime/test_lifecycle_requirement_ports.py @@ -1,6 +1,8 @@ from __future__ import annotations import unittest +import tempfile +from pathlib import Path from types import SimpleNamespace from envctl_engine.runtime.lifecycle_requirement_ports import ( @@ -10,6 +12,7 @@ requirement_port_values, ) from envctl_engine.state.models import RequirementsResult, RunState +from envctl_engine.shared.ports import PortPlanner class LifecycleRequirementPortsTests(unittest.TestCase): @@ -20,16 +23,21 @@ def test_requirement_key_for_project_matches_case_insensitive_key(self) -> None: self.assertIsNone(requirement_key_for_project(state, " ")) self.assertIsNone(requirement_key_for_project(state, "missing")) - def test_requirement_port_values_returns_enabled_internal_final_ports_only(self) -> None: + def test_requirement_port_values_returns_all_enabled_internal_resource_ports(self) -> None: requirements = RequirementsResult( project="Main", db={"enabled": True, "final": 5432}, redis={"enabled": False, "final": 6379}, - supabase={"enabled": True, "external": True, "final": 54321}, - n8n={"enabled": True, "final": 5678, "resources": {"ignored": 15678}}, + supabase={ + "enabled": True, + "external": True, + "final": 54321, + "resources": {"external_api": 55432}, + }, + n8n={"enabled": True, "final": 5678, "resources": {"secondary": 15678}}, ) - self.assertEqual(requirement_port_values(requirements), {5432, 5678}) + self.assertEqual(requirement_port_values(requirements), {5432, 5678, 15678}) def test_component_port_values_collects_final_and_resource_ports(self) -> None: self.assertEqual( @@ -43,12 +51,115 @@ def test_release_requirement_ports_uses_runtime_port_planner_in_sorted_order(sel requirements = RequirementsResult( project="Main", db={"enabled": True, "final": 5432}, + supabase={ + "enabled": True, + "final": 15432, + "resources": {"db": 15432, "api": 15433}, + }, n8n={"enabled": True, "final": 5678}, ) release_requirement_ports(runtime, requirements) - self.assertEqual(released, [5432, 5678]) + self.assertEqual(released, [5432, 5678, 15432, 15433]) + + def test_release_requirement_ports_can_release_verified_prior_session_owners(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(5432, owner="Main:db") + prior.reserve_next(5433, owner="Main:requirements") + requirements = RequirementsResult( + project="Main", + db={ + "enabled": True, + "final": 5432, + "resources": {"primary": 5432, "retry": 5433}, + "port_lock_session": "prior-session", + }, + ) + + release_requirement_ports(SimpleNamespace(port_planner=current), requirements) + + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + + def test_stale_requirement_state_cannot_release_newer_live_same_owner_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + newer = PortPlanner( + lock_dir=tmpdir, + session_id="newer-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + newer.reserve_next(5432, owner="Main:db") + stale_requirements = RequirementsResult( + project="Main", + db={ + "enabled": True, + "final": 5432, + "port_lock_session": "stale-session", + }, + ) + + release_requirement_ports( + SimpleNamespace(port_planner=cleanup), + stale_requirements, + ) + + self.assertTrue((Path(tmpdir) / "5432.lock").exists()) + + def test_legacy_requirement_state_preserves_live_foreign_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + live = PortPlanner( + lock_dir=tmpdir, + session_id="live-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + live.reserve_next(5432, owner="Main:db") + + release_requirement_ports( + SimpleNamespace(port_planner=cleanup), + RequirementsResult(project="Main", db={"enabled": True, "final": 5432}), + ) + + self.assertTrue((Path(tmpdir) / "5432.lock").exists()) + + def test_legacy_requirement_state_reaps_only_proven_dead_foreign_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "5432.lock" + lock_path.write_text( + '{"created_at": 0, "owner": "Main:db", "pid": 99999999, "session": "dead-session"}', + encoding="utf-8", + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + + release_requirement_ports( + SimpleNamespace(port_planner=cleanup), + RequirementsResult(project="Main", db={"enabled": True, "final": 5432}), + ) + + self.assertFalse(lock_path.exists()) if __name__ == "__main__": diff --git a/tests/python/runtime/test_lifecycle_service_stop_selection.py b/tests/python/runtime/test_lifecycle_service_stop_selection.py index 2f4bb830..2e015438 100644 --- a/tests/python/runtime/test_lifecycle_service_stop_selection.py +++ b/tests/python/runtime/test_lifecycle_service_stop_selection.py @@ -34,6 +34,44 @@ def test_runtime_scope_selects_matching_services_without_interactive_prompt(self self.assertEqual(selected, {"Aux Backend", "Main Backend"}) + def test_runtime_scope_intersects_explicit_project_selector(self) -> None: + state = RunState( + run_id="run-1", + mode="trees", + services={ + "FeatureA Backend": ServiceRecord( + name="FeatureA Backend", + type="backend", + cwd=".", + pid=1, + project="FeatureA", + ), + "FeatureB Backend": ServiceRecord( + name="FeatureB Backend", + type="backend", + cwd=".", + pid=2, + project="FeatureB", + ), + }, + ) + + selected = select_services_for_stop( + state, + Route( + command="stop", + mode="trees", + projects=["FeatureA"], + flags={"runtime_scope": "backend"}, + ), + project_name_from_service_fn=lambda name: name.split(" ", 1)[0], + selectors_from_passthrough_fn=lambda _args: set(), + interactive_stop_selection_fn=lambda _route, _state: self.fail("interactive prompt should not run"), + services_from_selection_fn=lambda _selection, _state: self.fail("selection projection should not run"), + ) + + self.assertEqual(selected, {"FeatureA Backend"}) + def test_service_and_project_selectors_match_slug_display_name_and_passthrough_project(self) -> None: state = RunState( run_id="run-1", diff --git a/tests/python/runtime/test_lifecycle_service_termination.py b/tests/python/runtime/test_lifecycle_service_termination.py index eb43fa87..6438d2d2 100644 --- a/tests/python/runtime/test_lifecycle_service_termination.py +++ b/tests/python/runtime/test_lifecycle_service_termination.py @@ -1,30 +1,36 @@ from __future__ import annotations import os +import signal +import subprocess +import sys +import tempfile import unittest +from pathlib import Path from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import Mock, patch from envctl_engine.runtime.lifecycle_service_termination import ( + _wait_for_pid_exit, service_port, terminate_service_record, terminate_services_from_state, terminate_started_services, ) from envctl_engine.state.models import RunState, ServiceRecord +from envctl_engine.shared.ports import PortPlanner class LifecycleServiceTerminationTests(unittest.TestCase): def test_terminate_started_services_uses_non_aggressive_unverified_shutdown(self) -> None: calls: list[tuple[int, bool, bool]] = [] runtime = SimpleNamespace( - _terminate_service_record=lambda service, *, aggressive, verify_ownership: calls.append( - (service.pid, aggressive, verify_ownership) + _terminate_service_record=lambda service, *, aggressive, verify_ownership: ( + calls.append((service.pid, aggressive, verify_ownership)) or True ) - or True ) - terminate_started_services( + failed = terminate_started_services( runtime, { "Main Backend": ServiceRecord(name="Main Backend", type="backend", cwd=".", pid=1), @@ -33,16 +39,56 @@ def test_terminate_started_services_uses_non_aggressive_unverified_shutdown(self ) self.assertEqual(calls, [(1, False, False), (2, False, False)]) + self.assertEqual(failed, set()) + + def test_terminate_started_services_reports_every_unconfirmed_exit(self) -> None: + runtime = SimpleNamespace(_terminate_service_record=lambda service, **_kwargs: service.name != "Main Backend") + + failed = terminate_started_services( + runtime, + { + "Main Backend": ServiceRecord(name="Main Backend", type="backend", cwd=".", pid=1), + "Main Frontend": ServiceRecord(name="Main Frontend", type="frontend", cwd=".", pid=2), + }, + ) + + self.assertEqual(failed, {"Main Backend"}) + + def test_terminate_started_services_continues_after_terminator_exception(self) -> None: + calls: list[str] = [] + events: list[tuple[str, dict[str, object]]] = [] + + def terminate(service: ServiceRecord, **_kwargs: object) -> bool: + calls.append(service.name) + if service.name == "Main Backend": + raise RuntimeError("probe failed") + return True + + runtime = SimpleNamespace( + _terminate_service_record=terminate, + _emit=lambda event, **payload: events.append((event, payload)), + ) + + failed = terminate_started_services( + runtime, + { + "Main Backend": ServiceRecord(name="Main Backend", type="backend", cwd="."), + "Main Frontend": ServiceRecord(name="Main Frontend", type="frontend", cwd="."), + }, + ) + + self.assertEqual(calls, ["Main Backend", "Main Frontend"]) + self.assertEqual(failed, {"Main Backend"}) + self.assertEqual(events[0][0], "cleanup.error") def test_terminate_services_from_state_filters_selected_services_and_releases_terminated_ports(self) -> None: released: list[int] = [] terminated: list[tuple[int, bool, bool]] = [] runtime = SimpleNamespace( port_planner=SimpleNamespace(release=lambda port: released.append(port)), - _terminate_service_record=lambda service, *, aggressive, verify_ownership: terminated.append( - (service.pid, aggressive, verify_ownership) - ) - or True, + _terminate_service_record=lambda service, *, aggressive, verify_ownership: ( + terminated.append((service.pid, aggressive, verify_ownership)) or True + ), ) state = RunState( run_id="run-1", @@ -55,7 +101,7 @@ def test_terminate_services_from_state_filters_selected_services_and_releases_te }, ) - terminate_services_from_state( + failed = terminate_services_from_state( runtime, state, selected_services={"Main Frontend"}, @@ -65,12 +111,229 @@ def test_terminate_services_from_state_filters_selected_services_and_releases_te self.assertEqual(terminated, [(2, True, True)]) self.assertEqual(released, [3000]) + self.assertEqual(failed, set()) + + def test_terminated_service_releases_verified_prior_session_port_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(8000, owner="Main:backend") + runtime = SimpleNamespace( + port_planner=current, + _terminate_service_record=lambda _service, **_kwargs: True, + _project_name_from_service=lambda _name: "Main", + ) + state = RunState( + run_id="run-prior", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + project="Main", + pid=1, + actual_port=8000, + port_lock_session="prior-session", + ) + }, + ) + + failed = terminate_services_from_state( + runtime, + state, + selected_services=None, + aggressive=False, + verify_ownership=True, + ) + + self.assertEqual(failed, set()) + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + + def test_terminated_rebound_frontend_releases_launch_owner_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(5173, owner="Main:frontend") + prior.reserve_next(5174, owner="Main:services:frontend-launch") + runtime = SimpleNamespace( + port_planner=current, + _terminate_service_record=lambda _service, **_kwargs: True, + _project_name_from_service=lambda _name: "Main", + ) + state = RunState( + run_id="run-prior", + mode="main", + services={ + "Main Frontend": ServiceRecord( + name="Main Frontend", + type="frontend", + cwd=".", + project="Main", + pid=1, + requested_port=5173, + actual_port=5174, + port_lock_session="prior-session", + ) + }, + ) + + failed = terminate_services_from_state( + runtime, + state, + selected_services=None, + aggressive=False, + verify_ownership=True, + ) + + self.assertEqual(failed, set()) + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + + def test_stale_service_state_cannot_release_newer_live_same_owner_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + newer = PortPlanner( + lock_dir=tmpdir, + session_id="newer-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + newer.reserve_next(8000, owner="Main:backend") + runtime = SimpleNamespace( + port_planner=cleanup, + _terminate_service_record=lambda _service, **_kwargs: True, + ) + state = RunState( + run_id="stale-run", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + project="Main", + pid=1, + actual_port=8000, + port_lock_session="stale-session", + ) + }, + ) + + failed = terminate_services_from_state( + runtime, + state, + selected_services=None, + aggressive=False, + verify_ownership=True, + ) + + self.assertEqual(failed, set()) + self.assertTrue((Path(tmpdir) / "8000.lock").exists()) + + def test_startup_rollback_releases_confirmed_service_ports_but_preserves_failed_service_ports(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="startup-session", + availability_checker=lambda _port: True, + ) + planner.reserve_next(5173, owner="Main:frontend") + planner.reserve_next(5174, owner="Main:services:frontend-launch") + planner.reserve_next(8000, owner="Main:backend") + frontend = ServiceRecord( + name="Main Frontend", + type="frontend", + cwd=".", + project="Main", + pid=1, + requested_port=5173, + actual_port=5174, + ) + backend = ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + project="Main", + pid=2, + requested_port=8000, + actual_port=8000, + ) + runtime = SimpleNamespace( + port_planner=planner, + _terminate_service_record=lambda service, **_kwargs: service is frontend, + ) + + failed = terminate_started_services( + runtime, + {frontend.name: frontend, backend.name: backend}, + ) + + self.assertEqual(failed, {backend.name}) + remaining = list(Path(tmpdir).glob("*.lock")) + self.assertEqual([path.name for path in remaining], ["8000.lock"]) + + def test_terminate_services_reports_failures_without_releasing_their_ports(self) -> None: + released: list[int] = [] + runtime = SimpleNamespace( + port_planner=SimpleNamespace(release=lambda port: released.append(port)), + _terminate_service_record=lambda service, **_kwargs: service.name != "Main Backend", + ) + state = RunState( + run_id="run-1", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=".", + pid=1, + actual_port=8000, + ), + "Main Frontend": ServiceRecord( + name="Main Frontend", + type="frontend", + cwd=".", + pid=2, + actual_port=3000, + ), + }, + ) + + failed = terminate_services_from_state( + runtime, + state, + selected_services=None, + aggressive=False, + verify_ownership=True, + ) + + self.assertEqual(failed, {"Main Backend"}) + self.assertEqual(released, [3000]) def test_terminate_service_record_skips_self_parent_and_missing_ownership_port(self) -> None: events: list[dict[str, object]] = [] runtime = SimpleNamespace( _emit=lambda _event, **payload: events.append(payload), - process_runner=SimpleNamespace(), + process_runner=SimpleNamespace(is_pid_running=lambda _pid: True), ) with patch.object(os, "getpid", return_value=100), patch.object(os, "getppid", return_value=200): @@ -96,6 +359,7 @@ def test_terminate_service_record_skips_self_parent_and_missing_ownership_port(s def test_terminate_service_record_accepts_verified_listener_child_and_prefers_process_group(self) -> None: calls: list[tuple[str, int]] = [] + live_pids = {111, 222} def pid_owns_port(pid: int, port: int) -> bool: return pid == 222 and port == 9000 @@ -103,8 +367,13 @@ def pid_owns_port(pid: int, port: int) -> bool: runtime = SimpleNamespace( _emit=lambda *_args, **_kwargs: None, process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, pid_owns_port=pid_owns_port, - terminate_process_group=lambda pid, **_kwargs: calls.append(("group", pid)) or True, + terminate_process_group=lambda pid, **_kwargs: ( + calls.append(("group", pid)), + live_pids.discard(pid), + True, + )[-1], terminate=lambda pid, **_kwargs: calls.append(("single", pid)) or True, ), ) @@ -117,7 +386,428 @@ def pid_owns_port(pid: int, port: int) -> bool: ) self.assertTrue(terminated) - self.assertEqual(calls, [("group", 111)]) + self.assertEqual(calls, [("group", 111), ("group", 222)]) + + def test_terminate_service_record_stops_non_listener_with_matching_recorded_cwd(self) -> None: + calls: list[int] = [] + live_pids = {333} + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + process_cwd=lambda _pid: "/repo/worker", + terminate_process_group=lambda pid, **_kwargs: (calls.append(pid), live_pids.discard(pid), True)[-1], + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Worker", + type="worker", + cwd="/repo/worker", + pid=333, + listener_expected=False, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertTrue(terminated) + self.assertEqual(calls, [333]) + + def test_terminate_service_record_keeps_non_listener_when_cwd_cannot_be_verified(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + runtime = SimpleNamespace( + _emit=lambda event, **payload: events.append((event, payload)), + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + process_cwd=lambda _pid: "/foreign/worker", + terminate_process_group=lambda *_args, **_kwargs: self.fail("foreign PID must not be signalled"), + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Worker", + type="worker", + cwd="/repo/worker", + pid=334, + listener_expected=False, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertFalse(terminated) + self.assertEqual(events[0][1]["reason"], "missing_port_for_ownership") + + def test_parent_dead_live_listener_child_is_still_terminated(self) -> None: + calls: list[int] = [] + live_pids = {402} + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + pid_owns_port=lambda pid, port: pid == 402 and port == 8000, + terminate_process_group=lambda pid, **_kwargs: (calls.append(pid), live_pids.discard(pid), True)[-1], + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=401, + listener_pids=[402], + actual_port=8000, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertTrue(terminated) + self.assertEqual(calls, [402]) + + def test_missing_primary_pid_does_not_hide_live_listener_child(self) -> None: + calls: list[int] = [] + live_pids = {403} + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + pid_owns_port=lambda pid, port: pid == 403 and port == 8001, + terminate_process_group=lambda pid, **_kwargs: (calls.append(pid), live_pids.discard(pid), True)[-1], + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=None, + listener_pids=[403], + actual_port=8001, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertTrue(terminated) + self.assertEqual(calls, [403]) + + def test_dead_parent_with_unrecorded_live_listener_keeps_state_without_signalling_foreign_pid(self) -> None: + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _listener_pids_for_port=lambda _port: [406], + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid == 406, + pid_owns_port=lambda *_args, **_kwargs: self.fail("no recorded live PID should be probed"), + terminate_process_group=lambda *_args, **_kwargs: self.fail("foreign PID must not be signalled"), + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=404, + listener_pids=None, + actual_port=8002, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertFalse(terminated) + + def test_pidless_unconfirmed_launch_remains_tracked(self) -> None: + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace(), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Worker", + type="worker", + cwd="/repo/worker", + pid=None, + listener_expected=False, + status="termination_failed", + ), + aggressive=False, + verify_ownership=False, + ) + + self.assertFalse(terminated) + + def test_live_port_listener_after_termination_keeps_service_tracked(self) -> None: + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _listener_pids_for_port=lambda _port: [405], + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + pid_owns_port=lambda pid, port: pid == 404 and port == 8002, + terminate_process_group=lambda *_args, **_kwargs: True, + ), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=404, + actual_port=8002, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertFalse(terminated) + + def test_malformed_listener_probe_result_fails_closed(self) -> None: + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _listener_pids_for_port=lambda _port: None, + process_runner=SimpleNamespace(is_pid_running=lambda _pid: False), + ) + + terminated = terminate_service_record( + runtime, + ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=999, + actual_port=8003, + ), + aggressive=False, + verify_ownership=True, + ) + + self.assertFalse(terminated) + + def test_terminate_service_record_treats_already_dead_pid_as_success_before_ownership_check(self) -> None: + ownership_checks: list[tuple[int, int]] = [] + terminate_calls: list[int] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: False, + pid_owns_port=lambda pid, port: ownership_checks.append((pid, port)) or False, + terminate_process_group=lambda pid, **_kwargs: terminate_calls.append(pid) or True, + ), + ) + + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Shadow Backend", pid=777, actual_port=8000), + aggressive=False, + verify_ownership=True, + ) + + self.assertTrue(terminated) + self.assertEqual(ownership_checks, []) + self.assertEqual(terminate_calls, []) + + def test_terminate_service_record_keeps_live_non_owner_fail_closed(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + runtime = SimpleNamespace( + _emit=lambda event, **payload: events.append((event, payload)), + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + pid_owns_port=lambda _pid, _port: False, + terminate_process_group=lambda *_args, **_kwargs: self.fail("non-owner must not be signalled"), + ), + ) + + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Foreign Backend", pid=778, actual_port=8001), + aggressive=False, + verify_ownership=True, + ) + + self.assertFalse(terminated) + self.assertEqual( + events, + [ + ( + "cleanup.skip", + { + "service": "Foreign Backend", + "pid": 778, + "port": 8001, + "reason": "ownership_mismatch", + }, + ) + ], + ) + + def test_raw_signal_fallback_escalates_and_fails_when_pid_survives(self) -> None: + sent_signals: list[tuple[int, signal.Signals]] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + terminate=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + ), + ) + + with ( + patch( + "envctl_engine.shared.process_termination.os.kill", + side_effect=lambda pid, requested_signal: sent_signals.append((pid, requested_signal)), + ), + patch( + "envctl_engine.shared.process_termination.wait_for_pid_exit", + side_effect=[False, False], + ) as wait_for_exit, + ): + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Main Backend", pid=779, actual_port=8002), + aggressive=False, + verify_ownership=False, + ) + + self.assertFalse(terminated) + self.assertEqual(sent_signals, [(779, signal.SIGTERM), (779, signal.SIGKILL)]) + self.assertEqual( + [call.kwargs["timeout"] for call in wait_for_exit.call_args_list], + [2.0, 1.0], + ) + + def test_raw_signal_fallback_does_not_escalate_after_confirmed_sigterm_exit(self) -> None: + sent_signals: list[tuple[int, signal.Signals]] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + terminate=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + ), + ) + + with ( + patch( + "envctl_engine.shared.process_termination.os.kill", + side_effect=lambda pid, requested_signal: sent_signals.append((pid, requested_signal)), + ), + patch( + "envctl_engine.shared.process_termination.wait_for_pid_exit", + return_value=True, + ), + ): + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Main Backend", pid=780, actual_port=8003), + aggressive=True, + verify_ownership=False, + ) + + self.assertTrue(terminated) + self.assertEqual(sent_signals, [(780, signal.SIGTERM)]) + + def test_raw_signal_fallback_accepts_process_lookup_race_as_clean_exit(self) -> None: + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + terminate=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + ), + ) + + with patch( + "envctl_engine.shared.process_termination.os.kill", + side_effect=ProcessLookupError, + ): + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Main Backend", pid=781, actual_port=8004), + aggressive=False, + verify_ownership=False, + ) + + self.assertTrue(terminated) + + def test_raw_signal_exit_wait_polls_until_pid_is_confirmed_dead(self) -> None: + runtime = SimpleNamespace( + process_runner=SimpleNamespace(is_pid_running=Mock(side_effect=[True, False])), + ) + + with ( + patch( + "envctl_engine.shared.process_termination.time.monotonic", + side_effect=[10.0, 10.0], + ), + patch("envctl_engine.shared.process_termination.time.sleep") as sleep_mock, + ): + exited = _wait_for_pid_exit( + runtime, + 782, + timeout=1.0, + initial_identity=None, + ) + + self.assertTrue(exited) + sleep_mock.assert_called_once_with(0.05) + + def test_raw_signal_fallback_really_escalates_sigterm_resistant_process(self) -> None: + process = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "import signal,time; " + "signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "print('ready', flush=True); " + "time.sleep(30)" + ), + ], + stdout=subprocess.PIPE, + text=True, + ) + try: + assert process.stdout is not None + self.assertEqual(process.stdout.readline().strip(), "ready") + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: process.poll() is None, + terminate_process_group=lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("unsupported") + ), + terminate=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("unsupported")), + ), + ) + + terminated = terminate_service_record( + runtime, + SimpleNamespace(name="Main Backend", pid=process.pid, actual_port=8005), + aggressive=True, + verify_ownership=False, + ) + + self.assertTrue(terminated) + self.assertEqual(process.wait(timeout=2.0), -signal.SIGKILL) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=2.0) def test_service_port_prefers_actual_then_requested(self) -> None: self.assertEqual(service_port(SimpleNamespace(actual_port=9000, requested_port=8000)), 9000) diff --git a/tests/python/runtime/test_lifecycle_worktree_cleanup.py b/tests/python/runtime/test_lifecycle_worktree_cleanup.py index 79773b82..8f43617d 100644 --- a/tests/python/runtime/test_lifecycle_worktree_cleanup.py +++ b/tests/python/runtime/test_lifecycle_worktree_cleanup.py @@ -12,6 +12,92 @@ class LifecycleWorktreeCleanupTests(unittest.TestCase): + def test_worktree_cleanup_persists_siblings_from_the_full_multi_project_run(self) -> None: + loader_calls: list[dict[str, object]] = [] + saved_states: list[RunState] = [] + target_service = ServiceRecord( + name="Feature Backend", + type="backend", + cwd="/repo/feature/backend", + actual_port=8000, + project="Feature", + ) + sibling_service = ServiceRecord( + name="Sibling Backend", + type="backend", + cwd="/repo/sibling/backend", + actual_port=8020, + project="Sibling", + ) + full_state = RunState( + run_id="run-multi", + mode="trees", + services={target_service.name: target_service, sibling_service.name: sibling_service}, + requirements={ + "Feature": RequirementsResult(project="Feature", db={"enabled": True, "final": 5432}), + "Sibling": RequirementsResult(project="Sibling", db={"enabled": True, "final": 5452}), + "SiblingInfra": RequirementsResult( + project="SiblingInfra", + redis={"enabled": True, "final": 6399}, + ), + }, + metadata={ + "project_names": ["Feature", "Sibling", "SiblingInfra"], + "project_roots": { + "Feature": "/repo/feature", + "Sibling": "/repo/sibling", + "SiblingInfra": "/repo/sibling-infra", + }, + }, + ) + + def load_state(**kwargs: object) -> RunState | None: + loader_calls.append(dict(kwargs)) + if kwargs.get("mode") != "trees": + return None + if kwargs.get("project_names"): + return RunState( + run_id=full_state.run_id, + mode=full_state.mode, + services={target_service.name: target_service}, + requirements={"Feature": full_state.requirements["Feature"]}, + metadata={ + "project_names": ["Feature"], + "project_roots": {"Feature": "/repo/feature"}, + }, + ) + return full_state + + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _try_load_existing_state=load_state, + _project_name_from_service=lambda name: str(name).removesuffix(" Backend"), + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + port_planner=SimpleNamespace(release=lambda _port: None), + state_repository=SimpleNamespace( + save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) + ), + process_runner=SimpleNamespace(), + env={}, + ) + + blast_worktree_before_delete( + runtime, + project_name="Feature", + project_root=Path("/repo/feature"), + source_command="delete-worktree", + ) + + self.assertTrue(loader_calls) + self.assertTrue(all("project_names" not in call for call in loader_calls)) + self.assertEqual(set(saved_states[0].services), {"Sibling Backend"}) + self.assertEqual(set(saved_states[0].requirements), {"Sibling", "SiblingInfra"}) + self.assertEqual(saved_states[0].metadata["project_names"], ["Sibling", "SiblingInfra"]) + self.assertEqual( + saved_states[0].metadata["project_roots"], + {"Sibling": "/repo/sibling", "SiblingInfra": "/repo/sibling-infra"}, + ) + def test_blast_worktree_before_delete_updates_matching_project_state(self) -> None: events: list[tuple[str, dict[str, object]]] = [] saved_states: list[RunState] = [] @@ -86,6 +172,56 @@ def test_blast_worktree_before_delete_uses_injected_cleanup_dependencies(self) - self.assertEqual(warnings, []) self.assertEqual(calls, ["ports", "cwd", "artifacts", "containers"]) + def test_worktree_delete_aborts_and_retains_authority_when_service_exit_is_unconfirmed(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + saved_states: list[RunState] = [] + released: list[int] = [] + service = ServiceRecord( + name="Feature Backend", + type="backend", + cwd="/repo/feature/backend", + pid=10, + actual_port=8000, + ) + requirements = RequirementsResult( + project="Feature", + db={"enabled": True, "final": 5432}, + ) + state = RunState( + run_id="run-1", + mode="trees", + services={service.name: service}, + requirements={"Feature": requirements}, + metadata={"project_roots": {"Feature": "/repo/feature"}}, + ) + runtime = SimpleNamespace( + _emit=lambda event, **payload: events.append((event, payload)), + _try_load_existing_state=lambda **kwargs: state if kwargs["mode"] == "trees" else None, + _project_name_from_service=lambda name: "Feature" if "Feature" in name else "", + _terminate_services_from_state=lambda *args, **kwargs: {service.name}, + port_planner=SimpleNamespace(release=lambda port: released.append(port)), + state_repository=SimpleNamespace( + save_selected_stop_state=lambda **kwargs: saved_states.append(kwargs["state"]) + ), + process_runner=SimpleNamespace(), + env={}, + ) + + with self.assertRaisesRegex(RuntimeError, "could not confirm service exit"): + blast_worktree_before_delete( + runtime, + project_name="Feature", + project_root=Path("/repo/feature"), + source_command="delete-worktree", + ) + + self.assertEqual(len(saved_states), 1) + self.assertIs(saved_states[0].services[service.name], service) + self.assertIs(saved_states[0].requirements["Feature"], requirements) + self.assertEqual(saved_states[0].metadata["project_roots"], {"Feature": "/repo/feature"}) + self.assertEqual(released, []) + self.assertEqual(events[-1][0], "cleanup.worktree.warning") + def test_prune_project_metadata_returns_test_artifacts_and_removes_empty_keys(self) -> None: state = RunState( run_id="run-1", @@ -152,7 +288,9 @@ def test_blast_tree_listener_ports_skips_docker_processes_and_kills_owned_listen self.assertEqual(warnings, []) self.assertEqual(killed, [222]) - self.assertEqual([event for event, _payload in events], ["cleanup.worktree.port.skip", "cleanup.worktree.port.kill"]) + self.assertEqual( + [event for event, _payload in events], ["cleanup.worktree.port.skip", "cleanup.worktree.port.kill"] + ) def test_remove_tree_containers_matches_hashed_and_legacy_names_and_removes_volumes(self) -> None: calls: list[list[str]] = [] diff --git a/tests/python/runtime/test_pr_preview_controller.py b/tests/python/runtime/test_pr_preview_controller.py index deb17c54..9918d31f 100644 --- a/tests/python/runtime/test_pr_preview_controller.py +++ b/tests/python/runtime/test_pr_preview_controller.py @@ -11,6 +11,9 @@ import envctl_engine.pr_preview_controller as preview_controller +REAL_PROBE_PUBLIC_ROUTE_URL = preview_controller.probe_public_route_url + + def load_controller(): return preview_controller @@ -1308,6 +1311,113 @@ def test_manual_start_refreshes_preview_ttl_when_label_is_old(tmp_path): assert "2000-01-01" not in runner.comments[-1] +def test_manual_start_reconciles_same_head_instead_of_reusing_stale_runtime( + tmp_path, + monkeypatch, +): + controller = load_controller() + root = tmp_path / "control" / "trees" / "imported" / "feature-demo" + root.mkdir(parents=True) + runner = FakeRunner( + controller, + projects={"projects": [{"name": "feature/demo", "root": str(root), "running": True}]}, + endpoints={"frontend": {"port": 9000}, "backend": {"port": 8000}}, + root_to_branch={str(root): "feature/demo"}, + ) + instance = controller.PreviewController(make_config(controller, tmp_path), runner) + pr = controller.pr_from_event( + pr_payload( + action="labeled", + labels=["deploy-app"], + event_label="deploy-app", + ) + ) + instance.save_state( + controller.PreviewState( + pr_number=pr.number, + label="deploy-app", + project="feature/demo", + root=str(root), + head_ref=pr.head_ref, + head_sha=pr.head_sha, + status="running", + label_added_at="2026-06-14T00:00:00Z", + started_at="2026-06-14T00:00:00Z", + expires_at="2026-06-14T00:45:00Z", + updated_at="2026-06-14T00:00:00Z", + endpoints={"frontend": {"public_url": "https://preview.example"}}, + deployment_id="12345", + ) + ) + monkeypatch.setattr(instance, "get_pr", lambda _number: pr) + + exit_code = instance.run_command("start", pr.number) + + assert exit_code == 0 + sequence = [ + call["argv"][:2] + for call in runner.calls + if call["argv"][:2] in (["envctl", "stop"], ["envctl", "import"], ["envctl", "start"]) + ] + assert sequence == [ + ["envctl", "stop"], + ["envctl", "import"], + ["envctl", "stop"], + ["envctl", "start"], + ] + state = instance.load_state(pr.number) + assert state is not None + assert state.status == "running" + assert state.head_sha == pr.head_sha + assert state.started_at != "2026-06-14T00:00:00Z" + assert "already running for this head" not in runner.comments[-1] + + +@pytest.mark.parametrize( + ("payload", "expected"), + [ + ({"ready": True}, None), + ( + {"ready": False, "reason": "missing_provider_credentials"}, + "reported ready=false (reason=missing_provider_credentials)", + ), + ( + {"ready": "true", "reason": "invalid/value with spaces"}, + "returned a non-boolean readiness value (reason=invalid_value_with_spaces)", + ), + ], +) +def test_public_route_probe_honors_json_readiness_semantics( + monkeypatch, + payload, + expected, +): + controller = load_controller() + + class Response: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, limit): + body = json.dumps(payload).encode("utf-8") + assert len(body) <= limit + return body + + monkeypatch.setattr(controller.urllib.request, "urlopen", lambda *_args, **_kwargs: Response()) + + result = REAL_PROBE_PUBLIC_ROUTE_URL("https://preview.example/readyz") + + if expected is None: + assert result is None + else: + assert expected in str(result) + + def test_push_event_refreshes_preview_ttl_for_labeled_branch(tmp_path): controller = load_controller() root = tmp_path / "control" / "trees" / "imported" / "feature-demo" diff --git a/tests/python/runtime/test_runtime_scope_isolation.py b/tests/python/runtime/test_runtime_scope_isolation.py index 503af616..e2e3cc96 100644 --- a/tests/python/runtime/test_runtime_scope_isolation.py +++ b/tests/python/runtime/test_runtime_scope_isolation.py @@ -13,6 +13,29 @@ class RuntimeScopeIsolationTests(unittest.TestCase): + def test_runtime_uses_repository_canonical_root_through_symlinked_parent(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + real_runtime_dir = root / "real-runtime" + runtime_alias = root / "runtime-alias" + repo = root / "repo" + real_runtime_dir.mkdir() + runtime_alias.symlink_to(real_runtime_dir, target_is_directory=True) + (repo / ".git").mkdir(parents=True) + config = load_config( + { + "RUN_REPO_ROOT": str(repo), + "RUN_SH_RUNTIME_DIR": str(runtime_alias), + } + ) + + runtime = PythonEngineRuntime(config, env={}) + + self.assertEqual(runtime.runtime_root, runtime.state_repository.runtime_root) + self.assertEqual(runtime.runtime_legacy_root, runtime.state_repository.runtime_legacy_root) + self.assertEqual(runtime.runtime_root, config.runtime_scope_dir.resolve()) + self.assertEqual(runtime.port_planner.lock_dir, runtime.runtime_root / "locks") + def test_managed_linked_worktree_uses_main_repo_runtime_scope_and_lock_dir(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -27,9 +50,7 @@ def test_managed_linked_worktree_uses_main_repo_runtime_scope_and_lock_dir(self) (worktree / ".git").write_text(f"gitdir: {gitdir}\n", encoding="utf-8") main_config = load_config({"RUN_REPO_ROOT": str(repo), "RUN_SH_RUNTIME_DIR": str(runtime_dir)}) - worktree_config = load_config( - {"RUN_REPO_ROOT": str(worktree), "RUN_SH_RUNTIME_DIR": str(runtime_dir)} - ) + worktree_config = load_config({"RUN_REPO_ROOT": str(worktree), "RUN_SH_RUNTIME_DIR": str(runtime_dir)}) main_runtime = PythonEngineRuntime(main_config, env={}) worktree_runtime = PythonEngineRuntime(worktree_config, env={"ENVCTL_INVOCATION_CWD": str(worktree)}) @@ -205,6 +226,83 @@ def test_stale_legacy_lock_symlink_is_replaced(self) -> None: (runtime.runtime_root / "locks").resolve(strict=False), ) + def test_scoped_and_read_only_construction_preserve_legacy_artifacts_and_lock_view(self) -> None: + for compat_mode in ("scoped_only", "compat_read_only"): + with self.subTest(compat_mode=compat_mode), tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime_dir = root / "runtime" + repo = root / "repo" + (repo / ".git").mkdir(parents=True) + legacy_root = runtime_dir / "python-engine" + legacy_root.mkdir(parents=True) + sentinel = legacy_root / "run_state.json" + sentinel.write_text("legacy-sentinel", encoding="utf-8") + config = load_config( + { + "RUN_REPO_ROOT": str(repo), + "RUN_SH_RUNTIME_DIR": str(runtime_dir), + } + ) + + runtime = PythonEngineRuntime( + config, + env={"ENVCTL_STATE_COMPAT_MODE": compat_mode}, + ) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "legacy-sentinel") + self.assertFalse((legacy_root / "locks").exists()) + self.assertTrue(runtime.runtime_root.is_dir()) + + def test_construction_rejects_preexisting_scoped_root_symlink_before_external_write(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime_dir = root / "runtime" + repo = root / "repo" + (repo / ".git").mkdir(parents=True) + config = load_config( + { + "RUN_REPO_ROOT": str(repo), + "RUN_SH_RUNTIME_DIR": str(runtime_dir), + } + ) + external = root / "external" + external.mkdir() + sentinel = external / "sentinel.txt" + sentinel.write_text("untouched", encoding="utf-8") + config.runtime_scope_dir.parent.mkdir(parents=True, exist_ok=True) + config.runtime_scope_dir.symlink_to(external, target_is_directory=True) + + with self.assertRaisesRegex(ValueError, "symlink"): + PythonEngineRuntime(config, env={}) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "untouched") + self.assertEqual(list(external.iterdir()), [sentinel]) + + def test_construction_rejects_preexisting_legacy_root_symlink_before_external_write(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime_dir = root / "runtime" + repo = root / "repo" + (repo / ".git").mkdir(parents=True) + external = root / "external" + external.mkdir() + sentinel = external / "sentinel.txt" + sentinel.write_text("untouched", encoding="utf-8") + runtime_dir.mkdir() + (runtime_dir / "python-engine").symlink_to(external, target_is_directory=True) + config = load_config( + { + "RUN_REPO_ROOT": str(repo), + "RUN_SH_RUNTIME_DIR": str(runtime_dir), + } + ) + + with self.assertRaisesRegex(ValueError, "symlink"): + PythonEngineRuntime(config, env={}) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "untouched") + self.assertEqual(list(external.iterdir()), [sentinel]) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/shared/test_ports_lock_reclamation.py b/tests/python/shared/test_ports_lock_reclamation.py index c6fb9f43..af200b89 100644 --- a/tests/python/shared/test_ports_lock_reclamation.py +++ b/tests/python/shared/test_ports_lock_reclamation.py @@ -1,9 +1,13 @@ from __future__ import annotations +import os import json import tempfile +import threading import unittest +from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from unittest.mock import patch REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" @@ -177,6 +181,305 @@ def test_release_session_isolated_to_own_session(self) -> None: self.assertFalse((Path(tmpdir) / f"{a_port}.lock").exists()) self.assertTrue((Path(tmpdir) / f"{b_port}.lock").exists()) + def test_concurrent_threads_never_publish_duplicate_reservations(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + worker_count = 64 + barrier = threading.Barrier(worker_count) + + def reserve(index: int) -> int: + planner = PortPlanner( + lock_dir=tmpdir, + session_id=f"session-{index}", + availability_checker=lambda _port: True, + ) + barrier.wait() + return planner.reserve_next(12000, owner=f"worker-{index}") + + with ThreadPoolExecutor(max_workers=worker_count) as executor: + reserved = list(executor.map(reserve, range(worker_count))) + + self.assertEqual(len(set(reserved)), worker_count) + self.assertEqual(len(list(Path(tmpdir).glob("*.lock"))), worker_count) + PortPlanner(lock_dir=tmpdir, session_id="cleanup").release_all() + + def test_concurrent_stale_reclaim_is_serialized(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "13000.lock" + lock_path.write_text( + json.dumps({"owner": "dead", "session": "dead", "pid": 99999999, "created_at": 0}), + encoding="utf-8", + ) + worker_count = 32 + barrier = threading.Barrier(worker_count) + + def reserve(index: int) -> int: + planner = PortPlanner( + lock_dir=tmpdir, + session_id=f"session-{index}", + availability_checker=lambda _port: True, + ) + barrier.wait() + return planner.reserve_next(13000, owner=f"worker-{index}") + + with ThreadPoolExecutor(max_workers=worker_count) as executor: + reserved = list(executor.map(reserve, range(worker_count))) + + self.assertEqual(len(set(reserved)), worker_count) + self.assertEqual(len(list(Path(tmpdir).glob("*.lock"))), worker_count) + PortPlanner(lock_dir=tmpdir, session_id="cleanup").release_all() + + def test_lock_path_is_linked_only_after_payload_is_complete(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + observed: dict[str, object] = {} + real_link = os.link + + def inspected_link(source: str | Path, destination: str | Path) -> None: + source_path = Path(source) + destination_path = Path(destination) + observed["destination_preexisted"] = destination_path.exists() + observed["payload"] = json.loads(source_path.read_text(encoding="utf-8")) + real_link(source_path, destination_path) + + planner = PortPlanner( + lock_dir=tmpdir, + session_id="complete-publication", + availability_checker=lambda _port: True, + ) + with ( + patch("envctl_engine.shared.ports.os.link", side_effect=inspected_link), + patch("envctl_engine.shared.ports.fsync_directory") as fsync_directory, + ): + reserved = planner.reserve_next(14000, owner="worker") + + self.assertEqual(reserved, 14000) + self.assertFalse(observed["destination_preexisted"]) + self.assertEqual(observed["payload"], json.loads((Path(tmpdir) / "14000.lock").read_text())) + self.assertEqual(list((Path(tmpdir) / ".port-guards").glob("*.pending")), []) + fsync_directory.assert_called_once_with(Path(tmpdir)) + + def test_corrupt_lock_gets_a_grace_period_then_becomes_stale(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + now = [1000.0] + lock_path = Path(tmpdir) / "15000.lock" + lock_path.write_text("{}", encoding="utf-8") + os.utime(lock_path, (now[0], now[0])) + planner = PortPlanner( + lock_dir=tmpdir, + session_id="new", + availability_checker=lambda _port: True, + time_provider=lambda: now[0], + corrupt_lock_grace_seconds=2.0, + ) + + self.assertEqual(planner.reserve_next(15000, owner="new"), 15001) + planner.release_session() + now[0] += 3.0 + self.assertEqual(planner.reserve_next(15000, owner="new"), 15000) + + def test_permission_error_means_lock_owner_is_still_live(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "16000.lock" + lock_path.write_text( + json.dumps({"owner": "foreign", "session": "foreign", "pid": 424242, "created_at": 0}), + encoding="utf-8", + ) + planner = PortPlanner( + lock_dir=tmpdir, + session_id="new", + availability_checker=lambda _port: True, + ) + + with patch("envctl_engine.shared.ports.os.kill", side_effect=PermissionError("denied")): + self.assertEqual(planner.reserve_next(16000, owner="new"), 16001) + + def test_boolean_pid_is_corrupt_data_not_a_live_pid_one_reservation(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "16010.lock" + lock_path.write_text( + json.dumps({"owner": "broken", "session": "broken", "pid": True, "created_at": False}), + encoding="utf-8", + ) + os.utime(lock_path, (0, 0)) + planner = PortPlanner( + lock_dir=tmpdir, + session_id="new", + availability_checker=lambda _port: True, + pid_checker=lambda _pid: self.fail("a JSON boolean must not be treated as PID 1"), + corrupt_lock_grace_seconds=0.0, + ) + + self.assertEqual(planner.reserve_next(16010, owner="new"), 16010) + + def test_failed_pid_probe_preserves_a_foreign_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "16020.lock" + lock_path.write_text( + json.dumps({"owner": "foreign", "session": "foreign", "pid": 424242, "created_at": 0}), + encoding="utf-8", + ) + planner = PortPlanner( + lock_dir=tmpdir, + session_id="new", + availability_checker=lambda _port: True, + pid_checker=lambda _pid: (_ for _ in ()).throw(OSError("probe unavailable")), + ) + + self.assertEqual(planner.reserve_next(16020, owner="new"), 16021) + self.assertEqual(json.loads(lock_path.read_text(encoding="utf-8"))["owner"], "foreign") + + def test_non_finite_corrupt_timestamp_cannot_pin_a_port_forever(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "16030.lock" + lock_path.write_text( + '{"owner":"broken","session":"broken","pid":false,"created_at":"nan"}', + encoding="utf-8", + ) + os.utime(lock_path, (0, 0)) + planner = PortPlanner( + lock_dir=tmpdir, + session_id="new", + availability_checker=lambda _port: True, + corrupt_lock_grace_seconds=0.0, + ) + + self.assertEqual(planner.reserve_next(16030, owner="new"), 16030) + + def test_normal_release_is_session_qualified_but_release_all_is_explicitly_scoped(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + owner = PortPlanner( + lock_dir=tmpdir, + session_id="session-a", + availability_checker=lambda _port: True, + ) + foreign = PortPlanner( + lock_dir=tmpdir, + session_id="session-b", + availability_checker=lambda _port: True, + ) + port = owner.reserve_next(17000, owner="Main:backend") + + foreign.release(port, owner="Main:backend") + self.assertTrue((Path(tmpdir) / f"{port}.lock").exists()) + foreign.release_all() + self.assertFalse((Path(tmpdir) / f"{port}.lock").exists()) + + def test_reap_stale_never_removes_a_live_foreign_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "18000.lock" + lock_path.write_text( + json.dumps({"owner": "worker", "session": "old", "pid": os.getpid(), "created_at": 0}), + encoding="utf-8", + ) + planner = PortPlanner(lock_dir=tmpdir, session_id="new", availability_checker=lambda _port: True) + + self.assertFalse(planner.reap_stale(18000, owner="worker")) + self.assertTrue(lock_path.exists()) + lock_path.write_text( + json.dumps({"owner": "worker", "session": "old", "pid": 99999999, "created_at": 0}), + encoding="utf-8", + ) + self.assertTrue(planner.reap_stale(18000, owner="worker")) + self.assertFalse(lock_path.exists()) + + def test_release_owned_requires_exact_owner_and_original_session(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + original = PortPlanner( + lock_dir=tmpdir, + session_id="old-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + port = original.reserve_next(19000, owner="Main:backend") + + self.assertFalse(cleanup.release_owned(port, "Other:backend", expected_session="old-session")) + self.assertTrue((Path(tmpdir) / f"{port}.lock").exists()) + self.assertFalse(cleanup.release_owned(port, "Main:backend", expected_session="cleanup-session")) + self.assertFalse(cleanup.release_owned(port, "Main:backend", expected_session="")) + self.assertTrue((Path(tmpdir) / f"{port}.lock").exists()) + self.assertTrue(cleanup.release_owned(port, "Main:backend", expected_session="old-session")) + self.assertFalse((Path(tmpdir) / f"{port}.lock").exists()) + + def test_bulk_stale_reap_preserves_live_and_non_numeric_locks(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + (root / "19100.lock").write_text( + json.dumps({"owner": "dead", "session": "old", "pid": 99999999, "created_at": 0}), + encoding="utf-8", + ) + (root / "19101.lock").write_text( + json.dumps({"owner": "live", "session": "live", "pid": os.getpid(), "created_at": 0}), + encoding="utf-8", + ) + (root / "19102.lock").write_text("{}", encoding="utf-8") + os.utime(root / "19102.lock", (0, 0)) + (root / "not-a-port.lock").write_text("invalid", encoding="utf-8") + planner = PortPlanner(lock_dir=tmpdir, session_id="cleanup", corrupt_lock_grace_seconds=1.0) + + self.assertEqual(planner.reap_stale_locks(), 2) + self.assertFalse((root / "19100.lock").exists()) + self.assertTrue((root / "19101.lock").exists()) + self.assertFalse((root / "19102.lock").exists()) + self.assertTrue((root / "not-a-port.lock").exists()) + + def test_event_handler_failure_cannot_hide_a_successful_acquire_or_release(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="event-failure", + availability_checker=lambda _port: True, + event_handler=lambda _name, _payload: (_ for _ in ()).throw(RuntimeError("telemetry failed")), + ) + + port = planner.reserve_next(19200, owner="worker") + + self.assertEqual(port, 19200) + self.assertTrue((Path(tmpdir) / "19200.lock").exists()) + planner.release(port, owner="worker") + self.assertFalse((Path(tmpdir) / "19200.lock").exists()) + + def test_lock_and_guard_directories_reject_symlinks(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + external = root / "external" + external.mkdir() + lock_link = root / "lock-link" + lock_link.symlink_to(external, target_is_directory=True) + + with self.assertRaisesRegex(OSError, "not a real directory"): + PortPlanner(lock_dir=str(lock_link)) + + lock_dir = root / "locks" + lock_dir.mkdir() + (lock_dir / ".port-guards").symlink_to(external, target_is_directory=True) + planner = PortPlanner( + lock_dir=str(lock_dir), + session_id="symlink-guard", + availability_checker=lambda _port: True, + ) + with self.assertRaisesRegex(OSError, "not a real directory"): + planner.reserve_next(19300, owner="worker") + self.assertEqual(list(external.iterdir()), []) + + def test_guard_artifacts_are_bounded_by_the_fixed_shard_count(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="bounded", + availability_checker=lambda _port: True, + ) + for port in range(20000, 20128): + self.assertEqual(planner.reserve_next(port, owner=f"worker-{port}"), port) + planner.release(port, owner=f"worker-{port}") + + guard_dir = Path(tmpdir) / ".port-guards" + self.assertLessEqual(len(list(guard_dir.glob("*.guard"))), 64) + self.assertEqual(list(guard_dir.glob("*.pending")), []) + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/shared/test_process_cwd.py b/tests/python/shared/test_process_cwd.py new file mode 100644 index 00000000..3843c6d1 --- /dev/null +++ b/tests/python/shared/test_process_cwd.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import subprocess +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from envctl_engine.shared.process_cwd import parse_lsof_cwd, process_cwd + + +class ProcessCwdTests(unittest.TestCase): + def test_parse_lsof_cwd_ignores_pid_record_and_returns_name_record(self) -> None: + self.assertEqual(parse_lsof_cwd("p123\nfcwd\nn/repo/backend\n"), "/repo/backend") + self.assertIsNone(parse_lsof_cwd("p123\nfcwd\n")) + + def test_process_cwd_prefers_proc_symlink_without_spawning_lsof(self) -> None: + with ( + patch("envctl_engine.shared.process_cwd.os.readlink", return_value="/repo/from-proc"), + patch("envctl_engine.shared.process_cwd.subprocess.run") as run_mock, + ): + result = process_cwd(123) + + self.assertEqual(result, "/repo/from-proc") + run_mock.assert_not_called() + + def test_process_cwd_falls_back_to_bounded_noninteractive_lsof(self) -> None: + with ( + patch("envctl_engine.shared.process_cwd.os.readlink", side_effect=OSError), + patch( + "envctl_engine.shared.process_cwd.subprocess.run", + return_value=SimpleNamespace(returncode=0, stdout="p123\nfcwd\nn/repo/from-lsof\n"), + ) as run_mock, + ): + result = process_cwd(123) + + self.assertEqual(result, "/repo/from-lsof") + self.assertEqual( + run_mock.call_args.args[0], + ["lsof", "-a", "-p", "123", "-d", "cwd", "-Fn"], + ) + self.assertEqual(run_mock.call_args.kwargs["timeout"], 1.0) + self.assertIs(run_mock.call_args.kwargs["stdin"], subprocess.DEVNULL) + + def test_process_cwd_returns_none_when_lsof_times_out(self) -> None: + with ( + patch("envctl_engine.shared.process_cwd.os.readlink", side_effect=OSError), + patch( + "envctl_engine.shared.process_cwd.subprocess.run", + side_effect=subprocess.TimeoutExpired(["lsof"], 1.0), + ), + ): + self.assertIsNone(process_cwd(123)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/shared/test_process_runner_listener_detection.py b/tests/python/shared/test_process_runner_listener_detection.py index cfa9824d..45ff0d0b 100644 --- a/tests/python/shared/test_process_runner_listener_detection.py +++ b/tests/python/shared/test_process_runner_listener_detection.py @@ -1,7 +1,9 @@ from __future__ import annotations +import os import signal import subprocess +import sys import unittest from pathlib import Path from unittest.mock import patch @@ -274,7 +276,9 @@ def _fake_killpg(pid: int, sig: int) -> None: with ( patch("envctl_engine.shared.process_runner.os.killpg", side_effect=_fake_killpg), - patch.object(runner, "is_pid_running", return_value=False), + patch("envctl_engine.shared.process_runner.os.getpgid", side_effect=lambda pid: pid), + patch.object(runner, "_process_group_member_pids", return_value={4321}), + patch.object(runner, "_wait_for_process_group_exit", return_value=True), patch.object(runner, "_pid_identity", return_value="orig", create=True), ): result = runner.terminate_process_group(4321, term_timeout=0.1, kill_timeout=0.1) @@ -291,13 +295,10 @@ def _fake_killpg(pid: int, sig: int) -> None: with ( patch("envctl_engine.shared.process_runner.os.killpg", side_effect=_fake_killpg), - patch.object(runner, "is_pid_running", side_effect=[True, True, False]), + patch("envctl_engine.shared.process_runner.os.getpgid", side_effect=lambda pid: pid), + patch.object(runner, "_process_group_member_pids", return_value={5678}), + patch.object(runner, "_wait_for_process_group_exit", side_effect=[False, True]), patch.object(runner, "_pid_identity", side_effect=["orig"] * 8, create=True), - patch( - "envctl_engine.shared.process_runner.time.monotonic", - side_effect=[0.0, 0.0, 0.2, 0.2, 0.2, 0.4], - ), - patch("envctl_engine.shared.process_runner.time.sleep", return_value=None), ): result = runner.terminate_process_group(5678, term_timeout=0.1, kill_timeout=0.1) @@ -313,19 +314,120 @@ def _fake_killpg(pid: int, sig: int) -> None: with ( patch("envctl_engine.shared.process_runner.os.killpg", side_effect=_fake_killpg), - patch.object(runner, "is_pid_running", return_value=True), + patch("envctl_engine.shared.process_runner.os.getpgid", side_effect=lambda pid: pid), + patch.object(runner, "_process_group_member_pids", return_value={8765}), + patch.object(runner, "_wait_for_process_group_exit", return_value=False), patch.object(runner, "_pid_identity", side_effect=["orig", "reused"], create=True), - patch( - "envctl_engine.shared.process_runner.time.monotonic", - side_effect=[0.0, 0.0, 0.2, 0.2, 0.2, 0.4], - ), - patch("envctl_engine.shared.process_runner.time.sleep", return_value=None), ): result = runner.terminate_process_group(8765, term_timeout=0.1, kill_timeout=0.1) - self.assertTrue(result) + self.assertFalse(result) self.assertEqual(killpg_calls, [(8765, signal.SIGTERM)]) + def test_terminate_process_group_kills_term_ignoring_descendant_after_leader_exits(self) -> None: + runner = ProcessRunner() + parent = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "import subprocess,sys,time; " + "subprocess.Popen([sys.executable, '-c', " + "'import os,signal,time; signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "print(os.getpid(), flush=True); time.sleep(60)']); " + "time.sleep(60)" + ), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) + child_pid = 0 + try: + assert parent.stdout is not None + child_pid = int(parent.stdout.readline().strip()) + + self.assertTrue(runner.terminate_process_group(parent.pid, term_timeout=0.2, kill_timeout=2.0)) + parent.wait(timeout=2.0) + self.assertFalse(runner.is_pid_running(child_pid)) + finally: + try: + os.killpg(parent.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + def test_terminate_process_group_resolves_dead_leaders_group_from_live_orphan_member(self) -> None: + runner = ProcessRunner() + parent = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "import os,time; " + "child=os.fork(); " + "print(child, flush=True) if child else None; " + "os._exit(0) if child else time.sleep(60)" + ), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) + child_pid = 0 + try: + assert parent.stdout is not None + child_pid = int(parent.stdout.readline().strip()) + parent.wait(timeout=2.0) + self.assertEqual(os.getpgid(child_pid), parent.pid) + + self.assertTrue(runner.terminate_process_group(child_pid, term_timeout=0.5, kill_timeout=2.0)) + self.assertFalse(runner.is_pid_running(child_pid)) + finally: + try: + os.killpg(parent.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + def test_terminate_process_group_does_not_treat_live_eperm_as_success(self) -> None: + runner = ProcessRunner() + + with ( + patch("envctl_engine.shared.process_runner.os.killpg", side_effect=PermissionError), + patch("envctl_engine.shared.process_runner.os.getpgid", return_value=8766), + patch.object(runner, "is_pid_running", return_value=True), + patch.object(runner, "_pid_identity", return_value="orig", create=True), + ): + result = runner.terminate_process_group(8766, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + + def test_terminate_process_group_does_not_treat_missing_group_for_live_pid_as_success(self) -> None: + runner = ProcessRunner() + + with ( + patch("envctl_engine.shared.process_runner.os.killpg", side_effect=ProcessLookupError), + patch("envctl_engine.shared.process_runner.os.getpgid", return_value=8767), + patch.object(runner, "is_pid_running", return_value=True), + patch.object(runner, "_pid_identity", return_value="orig", create=True), + ): + result = runner.terminate_process_group(8767, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + + def test_terminate_does_not_treat_live_eperm_as_success(self) -> None: + runner = ProcessRunner() + + with ( + patch("envctl_engine.shared.process_runner.os.kill", side_effect=PermissionError), + patch.object(runner, "is_pid_running", return_value=True), + patch.object(runner, "_pid_identity", return_value="orig", create=True), + ): + result = runner.terminate(8768, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + def test_is_pid_running_treats_zombie_process_as_not_running(self) -> None: runner = ProcessRunner() @@ -340,6 +442,12 @@ def test_is_pid_running_treats_zombie_process_as_not_running(self) -> None: ): self.assertFalse(runner.is_pid_running(4321)) + def test_is_pid_running_treats_permission_denied_as_still_running(self) -> None: + runner = ProcessRunner() + + with patch("envctl_engine.shared.process_runner.os.kill", side_effect=PermissionError): + self.assertTrue(runner.is_pid_running(4321)) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/shared/test_process_termination.py b/tests/python/shared/test_process_termination.py new file mode 100644 index 00000000..190c4a47 --- /dev/null +++ b/tests/python/shared/test_process_termination.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import os +import signal +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from envctl_engine.shared.process_termination import terminate_pid + + +class ProcessTerminationTests(unittest.TestCase): + def test_terminate_pid_refuses_self_and_parent_without_signalling(self) -> None: + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: self.fail("protected PID must not be signalled"), + ) + + self.assertFalse(terminate_pid(os.getpid(), process_runner=runner, term_timeout=0.0, kill_timeout=0.0)) + self.assertFalse(terminate_pid(os.getppid(), process_runner=runner, term_timeout=0.0, kill_timeout=0.0)) + + def test_failed_group_termination_does_not_kill_only_the_leader(self) -> None: + runner_calls: list[str] = [] + sent: list[signal.Signals] = [] + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: runner_calls.append("group") or False, + terminate=lambda *_args, **_kwargs: runner_calls.append("single") or False, + ) + + with ( + patch( + "envctl_engine.shared.process_termination._send_signal", + side_effect=lambda _pid, requested_signal: sent.append(requested_signal) or True, + ), + patch("envctl_engine.shared.process_termination.wait_for_pid_exit", side_effect=[False, False]), + ): + result = terminate_pid(987654, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + self.assertEqual(runner_calls, ["group"]) + self.assertEqual(sent, []) + + def test_runner_without_group_support_falls_back_to_raw_signals(self) -> None: + sent: list[signal.Signals] = [] + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate=lambda *_args, **_kwargs: False, + ) + + with ( + patch( + "envctl_engine.shared.process_termination._send_signal", + side_effect=lambda _pid, requested_signal: sent.append(requested_signal) or True, + ), + patch("envctl_engine.shared.process_termination.wait_for_pid_exit", side_effect=[False, False]), + ): + result = terminate_pid(987654, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + self.assertEqual(sent, [signal.SIGTERM, signal.SIGKILL]) + + def test_runner_success_is_not_trusted_while_same_identified_pid_remains_live(self) -> None: + sent: list[signal.Signals] = [] + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + _pid_identity=lambda _pid: "original", + terminate_process_group=lambda *_args, **_kwargs: True, + ) + + with patch( + "envctl_engine.shared.process_termination._send_signal", + side_effect=lambda _pid, requested_signal: sent.append(requested_signal) or True, + ): + result = terminate_pid(987655, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + self.assertEqual(sent, []) + + def test_identity_incapable_runner_success_is_not_trusted_while_pid_remains_live(self) -> None: + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: True, + ) + + result = terminate_pid(987657, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + + def test_identity_replacement_after_failed_group_cleanup_is_not_reported_as_success(self) -> None: + identities = iter(("original", "replacement")) + sent: list[signal.Signals] = [] + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + _pid_identity=lambda _pid: next(identities), + terminate_process_group=lambda *_args, **_kwargs: False, + ) + + with patch( + "envctl_engine.shared.process_termination._send_signal", + side_effect=lambda _pid, requested_signal: sent.append(requested_signal) or True, + ): + result = terminate_pid(987656, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertFalse(result) + self.assertEqual(sent, []) + + def test_group_cleanup_exception_is_success_after_leader_replacement(self) -> None: + identities = iter(("original", "replacement")) + sent: list[signal.Signals] = [] + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + _pid_identity=lambda _pid: next(identities), + terminate_process_group=lambda *_args, **_kwargs: (_ for _ in ()).throw( + OSError("post-kill reporting failed") + ), + ) + + with patch( + "envctl_engine.shared.process_termination._send_signal", + side_effect=lambda _pid, requested_signal: sent.append(requested_signal) or True, + ): + result = terminate_pid(987658, process_runner=runner, term_timeout=0.0, kill_timeout=0.0) + + self.assertTrue(result) + self.assertEqual(sent, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/startup/test_requirements_project_startup.py b/tests/python/startup/test_requirements_project_startup.py new file mode 100644 index 00000000..5769e0f7 --- /dev/null +++ b/tests/python/startup/test_requirements_project_startup.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from envctl_engine.requirements.orchestrator import FailureClass, RequirementOutcome +from envctl_engine.shared.ports import PortPlanner +from envctl_engine.startup.requirements_component_startup import RequirementComponentStarter +from envctl_engine.startup.requirements_project_startup import RequirementProjectStarter +from envctl_engine.state.models import PortPlan + + +class RequirementProjectStarterTests(unittest.TestCase): + @staticmethod + def _plan(port: int) -> PortPlan: + return PortPlan( + project="Main", + requested=port, + assigned=port, + final=port, + source="test", + ) + + def _starter(self, *, parallel: bool) -> RequirementProjectStarter: + postgres = SimpleNamespace( + id="postgres", + resources=(SimpleNamespace(name="db", legacy_port_key="db"),), + ) + redis = SimpleNamespace( + id="redis", + resources=(SimpleNamespace(name="redis", legacy_port_key="redis"),), + ) + + def start_component(_context, component, plan, _reserve_next, **_kwargs): # noqa: ANN001, ANN202 + if component == "redis": + raise RuntimeError("redis native startup crashed") + return RequirementOutcome( + service_name=component, + success=True, + requested_port=plan.requested, + final_port=plan.final, + retries=0, + container_name="postgres-main", + ) + + runtime = SimpleNamespace( + env={"ENVCTL_REQUIREMENTS_PARALLEL": "true" if parallel else "false"}, + config=SimpleNamespace(raw={}, strict_n8n_bootstrap=False), + port_planner=SimpleNamespace(session_id="requirements-session"), + _emit=lambda *_args, **_kwargs: None, + _start_requirement_component=start_component, + ) + return RequirementProjectStarter( + orchestrator=SimpleNamespace(runtime=runtime), + context=SimpleNamespace( + name="Main", + ports={"db": self._plan(5432), "redis": self._plan(6379)}, + ), + mode="main", + definitions=[postgres, redis], + ) + + def test_sequential_component_exception_becomes_partial_failed_outcome(self) -> None: + starter = self._starter(parallel=False) + + starter._run_enabled_components(starter.definitions) + + self.assertTrue(starter.outcomes["postgres"].success) + self.assertFalse(starter.outcomes["redis"].success) + self.assertEqual(starter.outcomes["redis"].final_port, 6379) + self.assertIn("native startup crashed", str(starter.outcomes["redis"].error)) + + def test_parallel_component_exception_waits_for_and_preserves_other_outcomes(self) -> None: + starter = self._starter(parallel=True) + + with patch( + "envctl_engine.startup.requirements_project_startup.requirements_parallel_enabled", + return_value=True, + ): + starter._run_enabled_components(starter.definitions) + + self.assertEqual(set(starter.outcomes), {"postgres", "redis"}) + self.assertTrue(starter.outcomes["postgres"].success) + self.assertFalse(starter.outcomes["redis"].success) + + def test_managed_components_record_exact_port_planner_session(self) -> None: + starter = self._starter(parallel=False) + starter._run_enabled_components(starter.definitions) + + components = starter._build_components( + {"postgres": True, "redis": True}, + {"postgres": False, "redis": False}, + ) + + self.assertEqual( + components["postgres"]["port_lock_session"], + "requirements-session", + ) + self.assertEqual( + components["redis"]["port_lock_session"], + "requirements-session", + ) + + def test_multiple_requirement_rebinds_release_every_failed_port_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="startup-session", + availability_checker=lambda _port: True, + ) + planner.reserve_next(5432, owner="Main:db") + starter = object.__new__(RequirementComponentStarter) + starter.runtime = SimpleNamespace( + port_planner=planner, + requirements=SimpleNamespace(reason_code_for_failure=lambda *_args, **_kwargs: "port_in_use"), + _emit=lambda *_args, **_kwargs: None, + ) + starter.context = SimpleNamespace(name="Main") + starter.name = "postgres" + + retry_one = planner.reserve_next(5433, owner="Main:requirements") + starter._on_retry( + "postgres", + 5432, + retry_one, + 1, + FailureClass.BIND_CONFLICT_RETRYABLE, + "address already in use", + ) + retry_two = planner.reserve_next(5434, owner="Main:requirements") + starter._on_retry( + "postgres", + retry_one, + retry_two, + 2, + FailureClass.BIND_CONFLICT_RETRYABLE, + "address already in use", + ) + + locks = list(Path(tmpdir).glob("*.lock")) + self.assertEqual([path.name for path in locks], ["5434.lock"]) + payload = json.loads(locks[0].read_text(encoding="utf-8")) + self.assertEqual(payload["owner"], "Main:requirements") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/startup/test_restart_prestop_support.py b/tests/python/startup/test_restart_prestop_support.py index 49bc4699..b7c94dc3 100644 --- a/tests/python/startup/test_restart_prestop_support.py +++ b/tests/python/startup/test_restart_prestop_support.py @@ -1,9 +1,12 @@ from __future__ import annotations import unittest +import tempfile +from pathlib import Path from types import SimpleNamespace from envctl_engine.runtime.command_router import Route +from envctl_engine.shared.ports import PortPlanner from envctl_engine.startup.restart_prestop_support import ( RestartOrphanListenerScan, apply_restart_port_assignments, @@ -105,11 +108,14 @@ def test_handle_restart_prestop_stops_selected_services_and_preserves_remaining_ restart_state=None, preserved_requirements={}, preserved_services={}, + base_metadata={"new": True}, ) state = SimpleNamespace( + run_id="run-old", mode="main", services={"Main Backend": object(), "Other Backend": object()}, requirements={"Main": object(), "Other": object()}, + metadata={"state_source_run_ids": ["run-ancestor"], "old": True}, ) events: list[tuple[str, dict[str, object]]] = [] released: list[object] = [] @@ -122,8 +128,8 @@ def test_handle_restart_prestop_stops_selected_services_and_preserves_remaining_ _effective_start_mode=lambda route: "main", _try_load_existing_state=lambda *, mode: state, _project_name_from_service=lambda name: str(name).removesuffix(" Backend"), - _terminate_services_from_state=lambda state, *, selected_services, aggressive, verify_ownership: terminated.append( - (state, set(selected_services)) + _terminate_services_from_state=lambda state, *, selected_services, aggressive, verify_ownership: ( + terminated.append((state, set(selected_services))) ), _release_requirement_ports=released.append, _emit=lambda event, **payload: events.append((event, payload)), @@ -148,10 +154,112 @@ def test_handle_restart_prestop_stops_selected_services_and_preserves_remaining_ self.assertEqual(released, [state.requirements["Main"]]) self.assertEqual(session.preserved_services, {"Other Backend": state.services["Other Backend"]}) self.assertEqual(session.preserved_requirements, {"Other": state.requirements["Other"]}) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-ancestor", "run-old"]) + self.assertTrue(session.base_metadata["old"]) + self.assertTrue(session.base_metadata["new"]) self.assertEqual(session.effective_route.command, "start") self.assertEqual(session.effective_route.flags["_restart_selected_services"], ["Main Backend"]) self.assertEqual(spinner.events, ["enter", "succeed:Restart pre-stop complete", "exit"]) - self.assertIn(("restart.selection", {"include_requirements": True, "target_projects": ["Main"], "selected_services": ["Main Backend"]}), events) + self.assertIn( + ( + "restart.selection", + {"include_requirements": True, "target_projects": ["Main"], "selected_services": ["Main Backend"]}, + ), + events, + ) + + def test_handle_restart_prestop_aborts_without_mutating_session_when_termination_fails(self) -> None: + route = Route( + command="restart", + mode="main", + flags={"services": ["Main Backend"]}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="main", + restart_state=None, + preserved_requirements={}, + preserved_services={}, + base_metadata={}, + ) + state = SimpleNamespace( + run_id="run-old", + mode="main", + services={"Main Backend": object()}, + requirements={"Main": object()}, + metadata={}, + ) + runtime = SimpleNamespace( + env={}, + _effective_start_mode=lambda _route: "main", + _try_load_existing_state=lambda *, mode: state, + _project_name_from_service=lambda _name: "Main", + _terminate_services_from_state=lambda *_args, **_kwargs: {"Main Backend"}, + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(RuntimeError, "Main Backend"): + handle_restart_prestop( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: True, + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("orphans must not be reaped"), + spinner_factory=lambda *_args, **_kwargs: _RecordingSpinner(), + use_spinner_policy_fn=lambda _policy: _NullContext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + ) + + self.assertEqual(session.preserved_services, {}) + self.assertEqual(session.preserved_requirements, {}) + self.assertEqual(session.base_metadata, {}) + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertIs(session.restart_state, state) + self.assertIs(session.effective_route, route) + + def test_handle_restart_prestop_aborts_when_orphan_exit_is_unconfirmed(self) -> None: + route = Route(command="restart", mode="main", flags={"services": ["Main Backend"]}) + session = SimpleNamespace( + effective_route=route, + runtime_mode="main", + restart_state=None, + preserved_requirements={}, + preserved_services={}, + base_metadata={}, + ) + state = SimpleNamespace( + run_id="run-old", + mode="main", + services={"Main Backend": object()}, + requirements={"Main": object()}, + metadata={}, + ) + runtime = SimpleNamespace( + env={}, + _effective_start_mode=lambda _route: "main", + _try_load_existing_state=lambda *, mode: state, + _project_name_from_service=lambda _name: "Main", + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + _emit=lambda *_args, **_kwargs: None, + ) + + with self.assertRaisesRegex(RuntimeError, "orphan listeners.*4242"): + handle_restart_prestop( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: True, + terminate_restart_orphan_listeners=lambda **_kwargs: {4242}, + spinner_factory=lambda *_args, **_kwargs: _RecordingSpinner(), + use_spinner_policy_fn=lambda _policy: _NullContext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, {}) + self.assertIs(session.effective_route, route) def test_restart_fallback_start_route_preserves_request_context_and_marks_restart(self) -> None: route = Route( @@ -502,7 +610,7 @@ def test_terminate_restart_orphan_listeners_releases_successfully_terminated_por released: list[int] = [] terminated: list[tuple[int, float, float]] = [] - terminate_restart_orphan_listeners( + failed = terminate_restart_orphan_listeners( state=state, selected_services={"Main Backend"}, aggressive=True, @@ -511,15 +619,121 @@ def test_terminate_restart_orphan_listeners_releases_successfully_terminated_por port_spacing=0, listener_pids_for_port=lambda port: [42] if port == 8000 else [], process_cwd=lambda pid: "/repo/backend" if pid == 42 else None, - terminate_pid=lambda pid, *, term_timeout, kill_timeout: terminated.append( - (pid, term_timeout, kill_timeout) - ) - or True, - release_port=released.append, + terminate_pid=lambda pid, *, term_timeout, kill_timeout: ( + terminated.append((pid, term_timeout, kill_timeout)) or True + ), + port_planner=SimpleNamespace(release=released.append), ) self.assertEqual(terminated, [(42, 0.5, 1.0)]) self.assertEqual(released, [8000]) + self.assertEqual(failed, set()) + + def test_restart_orphan_cleanup_releases_verified_prior_session_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + prior = PortPlanner( + lock_dir=tmpdir, + session_id="prior-session", + availability_checker=lambda _port: True, + ) + current = PortPlanner( + lock_dir=tmpdir, + session_id="current-session", + availability_checker=lambda _port: True, + ) + prior.reserve_next(8000, owner="Main:backend") + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace( + project="Main", + type="backend", + cwd="/repo/backend", + actual_port=8000, + port_lock_session="prior-session", + ) + } + ) + + failed = terminate_restart_orphan_listeners( + state=state, + selected_services={"Main Backend"}, + aggressive=True, + backend_port_base=8000, + frontend_port_base=3000, + port_spacing=0, + listener_pids_for_port=lambda port: [42] if port == 8000 else [], + process_cwd=lambda pid: "/repo/backend" if pid == 42 else None, + terminate_pid=lambda *_args, **_kwargs: True, + port_planner=current, + ) + + self.assertEqual(failed, set()) + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + + def test_stale_restart_state_cannot_release_newer_same_owner_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + newer = PortPlanner( + lock_dir=tmpdir, + session_id="newer-session", + availability_checker=lambda _port: True, + ) + cleanup = PortPlanner( + lock_dir=tmpdir, + session_id="cleanup-session", + availability_checker=lambda _port: True, + ) + newer.reserve_next(8000, owner="Main:backend") + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace( + project="Main", + type="backend", + cwd="/repo/backend", + actual_port=8000, + port_lock_session="stale-session", + ) + } + ) + + failed = terminate_restart_orphan_listeners( + state=state, + selected_services={"Main Backend"}, + aggressive=True, + backend_port_base=8000, + frontend_port_base=3000, + port_spacing=0, + listener_pids_for_port=lambda port: [42] if port == 8000 else [], + process_cwd=lambda pid: "/repo/backend" if pid == 42 else None, + terminate_pid=lambda *_args, **_kwargs: True, + port_planner=cleanup, + ) + + self.assertEqual(failed, set()) + self.assertTrue((Path(tmpdir) / "8000.lock").exists()) + + def test_terminate_restart_orphan_listeners_reports_unconfirmed_exits_without_releasing_ports(self) -> None: + state = SimpleNamespace( + services={ + "Main Backend": SimpleNamespace(type="backend", cwd="/repo/backend", actual_port=8000), + } + ) + released: list[int] = [] + + failed = terminate_restart_orphan_listeners( + state=state, + selected_services={"Main Backend"}, + aggressive=True, + backend_port_base=8000, + frontend_port_base=3000, + port_spacing=0, + listener_pids_for_port=lambda port: [42] if port == 8000 else [], + process_cwd=lambda pid: "/repo/backend" if pid == 42 else None, + terminate_pid=lambda *_args, **_kwargs: False, + port_planner=SimpleNamespace(release=released.append), + ) + + self.assertEqual(failed, {42}) + self.assertEqual(released, []) if __name__ == "__main__": diff --git a/tests/python/startup/test_run_reuse_identity.py b/tests/python/startup/test_run_reuse_identity.py index 15cd8531..6496c577 100644 --- a/tests/python/startup/test_run_reuse_identity.py +++ b/tests/python/startup/test_run_reuse_identity.py @@ -1,5 +1,7 @@ from __future__ import annotations +import hashlib +import json import unittest from pathlib import Path from types import SimpleNamespace @@ -49,6 +51,52 @@ def test_build_startup_identity_metadata_records_roots_and_fingerprint_payload(s self.assertEqual(identity["services"], {"backend": True, "frontend": False}) self.assertIn("fingerprint", identity) + def test_build_startup_identity_metadata_preserves_unselected_roots_without_case_duplicates(self) -> None: + runtime = SimpleNamespace( + config=SimpleNamespace( + backend_dir_name="api", + frontend_dir_name="web", + additional_services=(), + startup_enabled_for_mode=lambda _mode: True, + service_enabled_for_mode=lambda _mode, service: service == "backend", + requirement_enabled_for_mode=lambda _mode, _requirement: False, + ) + ) + + metadata = build_startup_identity_metadata( + runtime, + runtime_mode="trees", + project_contexts=[SimpleNamespace(name="featurea", root=Path("/tmp/new-a"))], + base_metadata={ + "project_roots": { + "FEATUREA": "/tmp/old-a", + "FeatureB": "/tmp/feature-b", + } + }, + route=Route(command="start", mode="trees", flags={}), + ) + + self.assertEqual( + metadata["project_roots"], + { + "featurea": normalize_project_root("/tmp/new-a"), + "FeatureB": normalize_project_root("/tmp/feature-b"), + }, + ) + identity = metadata["startup_identity"] + self.assertEqual( + identity["projects"], + [ + {"name": "featurea", "root": normalize_project_root("/tmp/new-a")}, + {"name": "FeatureB", "root": normalize_project_root("/tmp/feature-b")}, + ], + ) + fingerprint_payload = {key: value for key, value in identity.items() if key != "fingerprint"} + self.assertEqual( + identity["fingerprint"], + hashlib.sha256(json.dumps(fingerprint_payload, sort_keys=True).encode("utf-8")).hexdigest(), + ) + def test_project_identities_from_state_uses_metadata_requirements_and_services(self) -> None: state = RunState( run_id="run-1", @@ -83,7 +131,9 @@ def test_context_identity_sorting_and_weak_keys_are_stable(self) -> None: def test_startup_identity_mismatch_ignores_matching_fingerprints_and_reports_changed_fields(self) -> None: self.assertEqual( - _startup_identity_mismatch({"fingerprint": "same", "mode": "main"}, {"fingerprint": "same", "mode": "trees"}), + _startup_identity_mismatch( + {"fingerprint": "same", "mode": "main"}, {"fingerprint": "same", "mode": "trees"} + ), {}, ) diff --git a/tests/python/startup/test_run_reuse_support.py b/tests/python/startup/test_run_reuse_support.py index 566c0b45..d6ae837c 100644 --- a/tests/python/startup/test_run_reuse_support.py +++ b/tests/python/startup/test_run_reuse_support.py @@ -44,7 +44,9 @@ def test_dashboard_stopped_service_entries_normalizes_valid_backend_frontend_ent def test_dashboard_stopped_service_entries_ignores_missing_or_invalid_metadata(self) -> None: self.assertEqual(dashboard_stopped_service_entries(SimpleNamespace(metadata={})), []) - self.assertEqual(dashboard_stopped_service_entries(SimpleNamespace(metadata={"dashboard_stopped_services": {}})), []) + self.assertEqual( + dashboard_stopped_service_entries(SimpleNamespace(metadata={"dashboard_stopped_services": {}})), [] + ) def test_metadata_without_dashboard_stopped_services_removes_restored_entries_only(self) -> None: metadata = { @@ -172,8 +174,7 @@ def test_prepare_dashboard_stopped_service_restore_ignores_active_or_unselected_ ) ) - def test_fresh_start_replacement_services_selects_configured_service_types_for_target_projects(self) -> None: - route = Route(command="start", mode="main", raw_args=["start"], flags={}) + def test_fresh_start_replacement_services_selects_every_service_for_target_projects(self) -> None: candidate_state = SimpleNamespace( services={ "Main Backend": SimpleNamespace(name="Main Backend", project="Main", type="backend"), @@ -184,20 +185,36 @@ def test_fresh_start_replacement_services_selects_configured_service_types_for_t self.assertEqual( fresh_start_replacement_services( - route=route, selected_contexts=[SimpleNamespace(name="Main")], candidate_state=candidate_state, - configured_service_types={"backend"}, - additional_services=(), project_name_from_service=lambda name: str(name).removesuffix(" Backend").removesuffix(" Frontend"), ), - {"Main Backend"}, + {"Main Backend", "Main Frontend"}, ) def test_replace_existing_project_services_for_fresh_start_terminates_selected_services_and_orphans(self) -> None: route = Route(command="start", mode="trees", raw_args=["start"], flags={}) - session = SimpleNamespace(effective_route=route, runtime_mode="trees") - candidate_state = SimpleNamespace(run_id="run-existing") + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="Main")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + candidate_state = SimpleNamespace( + run_id="run-existing", + services={ + "Main Backend": SimpleNamespace(project="Main"), + "Main Frontend": SimpleNamespace(project="Main"), + "Other Backend": SimpleNamespace(project="Other"), + }, + requirements={ + "Main": SimpleNamespace(project="Main"), + "Other": SimpleNamespace(project="Other"), + }, + metadata={"state_source_run_ids": ["run-ancestor"], "keep": True}, + ) events: list[tuple[str, dict[str, object]]] = [] announced: list[object] = [] progress: list[tuple[Route, str]] = [] @@ -205,17 +222,19 @@ def test_replace_existing_project_services_for_fresh_start_terminates_selected_s orphaned: list[tuple[object, set[str], bool]] = [] runtime = SimpleNamespace( _emit=lambda event, **payload: events.append((event, payload)), - _terminate_services_from_state=lambda state, *, selected_services, aggressive, verify_ownership: terminated.append( - (state, set(selected_services), aggressive, verify_ownership) + _terminate_services_from_state=lambda state, *, selected_services, aggressive, verify_ownership: ( + terminated.append((state, set(selected_services), aggressive, verify_ownership)) ), + _project_name_from_service=lambda name: str(name).removesuffix(" Backend").removesuffix(" Frontend"), + _release_requirement_ports=lambda requirements: released.append(requirements), ) + released: list[object] = [] replace_existing_project_services_for_fresh_start( runtime=runtime, session=session, candidate_state=candidate_state, reason="startup_fingerprint_mismatch", - fresh_start_replacement_services=lambda session, *, candidate_state: {"Main Backend", "Main Frontend"}, announce_session_identifiers=announced.append, report_progress=lambda route, message: progress.append((route, message)), terminate_restart_orphan_listeners=lambda *, state, selected_services, aggressive: orphaned.append( @@ -241,35 +260,306 @@ def test_replace_existing_project_services_for_fresh_start_terminates_selected_s ) self.assertEqual(terminated, [(candidate_state, {"Main Backend", "Main Frontend"}, False, True)]) self.assertEqual(orphaned, [(candidate_state, {"Main Backend", "Main Frontend"}, True)]) + self.assertEqual(released, [candidate_state.requirements["Main"]]) + self.assertEqual(session.preserved_services, {"Other Backend": candidate_state.services["Other Backend"]}) + self.assertEqual(session.preserved_requirements, {"Other": candidate_state.requirements["Other"]}) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-ancestor", "run-existing"]) + self.assertTrue(session.base_metadata["keep"]) + + def test_explicit_no_resume_replaces_matching_services_instead_of_leaking_a_second_copy(self) -> None: + route = Route(command="start", mode="trees", flags={"no_resume": True}) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": SimpleNamespace(project="feature-a")}, + requirements={}, + metadata={}, + ) + terminated: list[set[str]] = [] + progress: list[str] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _terminate_services_from_state=lambda _state, *, selected_services, **_kwargs: terminated.append( + set(selected_services) + ), + _project_name_from_service=lambda name: str(name).removesuffix(" Backend"), + _release_requirement_ports=lambda _requirements: None, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda _route, message: progress.append(message), + terminate_restart_orphan_listeners=lambda **_kwargs: None, + ) + + self.assertEqual(terminated, [{"feature-a Backend"}]) + self.assertEqual(progress, ["Auto-resume disabled; replacing 1 existing service(s)..."]) - def test_replace_existing_project_services_for_fresh_start_skips_non_mismatch_or_dependencies_scope(self) -> None: - runtime = SimpleNamespace(_emit=lambda *args, **kwargs: self.fail("replace should not emit")) - candidate_state = SimpleNamespace(run_id="run-existing") + def test_backend_scoped_fresh_start_preserves_frontend_and_existing_requirements(self) -> None: + route = Route( + command="start", + mode="trees", + flags={"no_resume": True, "runtime_scope": "backend"}, + ) + backend = SimpleNamespace(project="feature-a", type="backend") + frontend = SimpleNamespace(project="feature-a", type="frontend") + requirements = SimpleNamespace(project="feature-a") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": backend, "feature-a Frontend": frontend}, + requirements={"feature-a": requirements}, + metadata={}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + terminated: list[set[str]] = [] + released: list[object] = [] + runtime = SimpleNamespace( + config=SimpleNamespace(additional_services=()), + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda name: str(name).rsplit(" ", 1)[0], + _terminate_services_from_state=lambda _state, *, selected_services, **_kwargs: ( + terminated.append(set(selected_services)) or set() + ), + _release_requirement_ports=released.append, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda *_args: None, + terminate_restart_orphan_listeners=lambda **_kwargs: set(), + ) + + self.assertEqual(terminated, [{"feature-a Backend"}]) + self.assertEqual(session.preserved_services, {"feature-a Frontend": frontend}) + self.assertEqual(session.preserved_requirements, {"feature-a": requirements}) + self.assertEqual(released, []) - for reason, flags in ( - ("no_matching_state", {}), - ("startup_fingerprint_mismatch", {"runtime_scope": "dependencies"}), - ): - route = Route(command="start", mode="trees", raw_args=["start"], flags=flags) - session = SimpleNamespace(effective_route=route, runtime_mode="trees") + def test_explicit_no_resume_failure_keeps_previous_state_authoritative(self) -> None: + route = Route(command="start", mode="trees", flags={"no_resume": True}) + original_service = SimpleNamespace(project="feature-a") + original_requirements = SimpleNamespace(project="feature-a") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": original_service}, + requirements={"feature-a": original_requirements}, + metadata={"keep": True}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={"sentinel": object()}, + preserved_requirements={"sentinel": object()}, + base_metadata={"sentinel": True}, + ) + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "feature-a", + _terminate_services_from_state=lambda *_args, **_kwargs: {"feature-a Backend"}, + _release_requirement_ports=lambda _requirements: self.fail("failed replacement must not release ports"), + ) + + with self.assertRaisesRegex(RuntimeError, "feature-a Backend"): replace_existing_project_services_for_fresh_start( runtime=runtime, session=session, candidate_state=candidate_state, - reason=reason, - fresh_start_replacement_services=lambda session, *, candidate_state: self.fail("selection should skip"), - announce_session_identifiers=lambda session: self.fail("announce should skip"), - report_progress=lambda route, message: self.fail("progress should skip"), - terminate_restart_orphan_listeners=lambda **kwargs: self.fail("orphan cleanup should skip"), + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda _route, _message: None, + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("must not reap after failure"), ) - def test_fresh_start_replacement_services_honors_restart_service_type_filters(self) -> None: + self.assertEqual(set(session.preserved_services), {"sentinel"}) + self.assertEqual(set(session.preserved_requirements), {"sentinel"}) + self.assertEqual(session.base_metadata, {"sentinel": True}) + self.assertTrue(session.preserve_existing_state_on_failure) + + def test_fresh_start_aborts_when_orphan_exit_is_unconfirmed(self) -> None: + route = Route(command="start", mode="trees", flags={"no_resume": True}) + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": SimpleNamespace(project="feature-a")}, + requirements={"feature-a": SimpleNamespace(project="feature-a")}, + metadata={}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "feature-a", + _terminate_services_from_state=lambda *_args, **_kwargs: set(), + _release_requirement_ports=lambda _requirements: self.fail("ports must remain reserved"), + ) + + with self.assertRaisesRegex(RuntimeError, "orphan listeners.*4242"): + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: None, + report_progress=lambda *_args: None, + terminate_restart_orphan_listeners=lambda **_kwargs: {4242}, + ) + + self.assertTrue(session.preserve_existing_state_on_failure) + self.assertEqual(session.preserved_services, {}) + self.assertEqual(session.preserved_requirements, {}) + + def test_explicit_no_resume_supersedes_requirement_only_state(self) -> None: route = Route( - command="restart", - mode="main", - raw_args=["restart"], - flags={"_restart_request": True, "restart_service_types": ["frontend"]}, + command="start", + mode="trees", + projects=["feature-a"], + flags={"no_resume": True}, + ) + target_requirements = SimpleNamespace(project="feature-a") + other_requirements = SimpleNamespace(project="feature-b") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={}, + requirements={"feature-a": target_requirements, "feature-b": other_requirements}, + metadata={}, + ) + session = SimpleNamespace( + effective_route=route, + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + released: list[object] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "", + _terminate_services_from_state=lambda *_args, **_kwargs: self.fail("no service termination expected"), + _release_requirement_ports=released.append, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="explicit_no_resume", + announce_session_identifiers=lambda _session: self.fail("empty replacement needs no announcement"), + report_progress=lambda _route, _message: self.fail("empty replacement needs no progress"), + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("no orphan scan expected"), + ) + + self.assertEqual(released, [target_requirements]) + self.assertEqual(session.preserved_requirements, {"feature-b": other_requirements}) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-existing"]) + + def test_no_reusable_runtime_still_supersedes_requirement_only_state(self) -> None: + target_requirements = SimpleNamespace(project="feature-a") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={}, + requirements={"feature-a": target_requirements}, + metadata={"existing": True}, ) + session = SimpleNamespace( + effective_route=Route(command="start", mode="trees", flags={}), + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + released: list[object] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "", + _terminate_services_from_state=lambda *_args, **_kwargs: self.fail("no services should be stopped"), + _release_requirement_ports=released.append, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="no_reusable_runtime", + announce_session_identifiers=lambda _session: self.fail("no services need an announcement"), + report_progress=lambda _route, _message: self.fail("no services need progress"), + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("no orphan scan expected"), + ) + + self.assertEqual(released, [target_requirements]) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-existing"]) + + def test_dependencies_only_fresh_start_preserves_services_but_replaces_selected_requirements(self) -> None: + target_service = SimpleNamespace(project="feature-a") + other_service = SimpleNamespace(project="feature-b") + target_requirements = SimpleNamespace(project="feature-a") + other_requirements = SimpleNamespace(project="feature-b") + candidate_state = SimpleNamespace( + run_id="run-existing", + services={"feature-a Backend": target_service, "feature-b Backend": other_service}, + requirements={"feature-a": target_requirements, "feature-b": other_requirements}, + metadata={}, + ) + session = SimpleNamespace( + effective_route=Route(command="start", mode="trees", flags={"runtime_scope": "dependencies"}), + runtime_mode="trees", + selected_contexts=[SimpleNamespace(name="feature-a")], + preserved_services={}, + preserved_requirements={}, + base_metadata={}, + ) + released: list[object] = [] + runtime = SimpleNamespace( + _emit=lambda *_args, **_kwargs: None, + _project_name_from_service=lambda _name: "feature-a", + _terminate_services_from_state=lambda *_args, **_kwargs: self.fail("app services must be preserved"), + _release_requirement_ports=released.append, + ) + + replace_existing_project_services_for_fresh_start( + runtime=runtime, + session=session, + candidate_state=candidate_state, + reason="startup_fingerprint_mismatch", + announce_session_identifiers=lambda _session: self.fail("app services must be preserved"), + report_progress=lambda _route, _message: self.fail("app services must be preserved"), + terminate_restart_orphan_listeners=lambda **_kwargs: self.fail("no orphan scan expected"), + ) + + self.assertEqual(session.preserved_services, candidate_state.services) + self.assertEqual(session.preserved_requirements, {"feature-b": other_requirements}) + self.assertEqual(released, [target_requirements]) + self.assertEqual(session.base_metadata["state_source_run_ids"], ["run-existing"]) + + def test_fresh_start_replacement_services_does_not_leave_disabled_service_types_running(self) -> None: candidate_state = SimpleNamespace( services={ "Main Backend": SimpleNamespace(name="Main Backend", project="Main", type="backend"), @@ -279,14 +569,11 @@ def test_fresh_start_replacement_services_honors_restart_service_type_filters(se self.assertEqual( fresh_start_replacement_services( - route=route, selected_contexts=[SimpleNamespace(name="Main")], candidate_state=candidate_state, - configured_service_types={"backend", "frontend"}, - additional_services=(), project_name_from_service=lambda name: str(name).removesuffix(" Backend").removesuffix(" Frontend"), ), - {"Main Frontend"}, + {"Main Backend", "Main Frontend"}, ) diff --git a/tests/python/startup/test_selected_context_startup.py b/tests/python/startup/test_selected_context_startup.py index a752e410..a3619d34 100644 --- a/tests/python/startup/test_selected_context_startup.py +++ b/tests/python/startup/test_selected_context_startup.py @@ -6,13 +6,15 @@ import unittest from envctl_engine.runtime.command_router import Route +from envctl_engine.runtime.service_manager import ServiceCleanupError +from envctl_engine.startup.startup_execution_support import ProjectStartupFailure from envctl_engine.startup.selected_context_startup import ( record_project_startup, start_selected_contexts, start_selected_contexts_with_runtime, ) -from envctl_engine.startup.session import ProjectStartupResult, StartupSession -from envctl_engine.state.models import RequirementsResult +from envctl_engine.startup.session import ProjectStartupResult, StartupSession, track_unterminated_services +from envctl_engine.state.models import RequirementsResult, ServiceRecord class _SpinnerStub: @@ -77,6 +79,109 @@ def _start_project_context(self, *, context, mode: str, route: Route, run_id: st class SelectedContextStartupTests(unittest.TestCase): + def test_parallel_renderer_error_occurs_only_after_successful_resources_are_recorded(self) -> None: + runtime = _RuntimeStub() + runtime._tree_parallel_startup_config = ( # type: ignore[method-assign] + lambda **_kwargs: (True, 2) + ) + route = Route(command="start", mode="trees", flags={}) + contexts = [self._context("Alpha"), self._context("Beta")] + session = self._session(route=route, contexts=contexts) + + with self.assertRaisesRegex(OSError, "renderer failed"): + start_selected_contexts( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: False, + resolved_run_id=lambda _session: "run-1", + record_project_startup=record_project_startup, + render_project_startup_warnings=lambda **_kwargs: (_ for _ in ()).throw(OSError("renderer failed")), + should_degrade_to_plan_agent_handoff=lambda _session, _error: False, + record_plan_agent_handoff_local_startup_failure=lambda **_kwargs: None, + spinner_factory=_SpinnerFactory(), + use_spinner_policy_fn=lambda _policy: nullcontext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False, backend="rich"), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + project_spinner_group_factory=lambda **_kwargs: SimpleNamespace(), + ) + + self.assertEqual(set(session.services_by_project), {"Alpha", "Beta"}) + self.assertEqual(set(session.requirements_by_project), {"Alpha", "Beta"}) + self.assertEqual(set(session.started_context_names), {"Alpha", "Beta"}) + + def test_parallel_failure_callback_error_does_not_hide_later_success(self) -> None: + runtime = _RuntimeStub() + runtime._tree_parallel_startup_config = lambda **_kwargs: (True, 2) # type: ignore[method-assign] + + def start_context(*, context, **_kwargs): # noqa: ANN001, ANN202 + if context.name == "Alpha": + raise RuntimeError("alpha failed") + return ProjectStartupResult( + requirements=RequirementsResult(project=context.name), + services={"Beta Backend": object()}, + ) + + runtime._start_project_context = start_context # type: ignore[method-assign] + original_emit = runtime._emit + + def emit(event: str, **payload: object) -> None: + if event == "startup.project.failed": + raise OSError("event sink failed") + original_emit(event, **payload) + + runtime._emit = emit # type: ignore[method-assign] + route = Route(command="start", mode="trees", flags={}) + session = self._session(route=route, contexts=[self._context("Alpha"), self._context("Beta")]) + + with self.assertRaisesRegex(RuntimeError, "alpha failed"): + start_selected_contexts( + runtime=runtime, + session=session, + suppress_progress_output=lambda _route: False, + resolved_run_id=lambda _session: "run-1", + record_project_startup=record_project_startup, + render_project_startup_warnings=lambda **_kwargs: None, + should_degrade_to_plan_agent_handoff=lambda _session, _error: False, + record_plan_agent_handoff_local_startup_failure=lambda **_kwargs: None, + spinner_factory=_SpinnerFactory(), + use_spinner_policy_fn=lambda _policy: nullcontext(), + resolve_spinner_policy_fn=lambda _env: SimpleNamespace(enabled=False, backend="rich"), + emit_spinner_policy_fn=lambda *_args, **_kwargs: None, + project_spinner_group_factory=lambda **_kwargs: SimpleNamespace(), + ) + + self.assertIn("Beta", session.services_by_project) + self.assertIn("Beta", session.requirements_by_project) + + def test_unterminated_service_error_is_carried_into_session_state(self) -> None: + route = Route(command="start", mode="main", flags={}) + session = self._session(route=route, contexts=[]) + service = ServiceRecord(name="Main Backend", type="backend", cwd="/repo", pid=63001) + failure = ServiceCleanupError("cleanup failed", {service.name: service}) + + track_unterminated_services(session, failure) + + self.assertIs(session.unterminated_services[service.name], service) + self.assertIs(session.merged_services[service.name], service) + + def test_project_startup_failure_carries_requirements_into_failure_state(self) -> None: + route = Route(command="start", mode="main", flags={}) + session = self._session(route=route, contexts=[]) + requirements = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6379}, + ) + failure = ProjectStartupFailure( + "backend failed", + project="Main", + requirements=requirements, + ) + + track_unterminated_services(session, failure) + + self.assertIs(session.requirements_by_project["Main"], requirements) + self.assertIs(session.merged_requirements["Main"], requirements) + @staticmethod def _context(name: str) -> SimpleNamespace: return SimpleNamespace(name=name, root=Path("/tmp") / name, ports={}) @@ -110,7 +215,9 @@ def test_start_selected_contexts_builds_execution_route_and_records_sequential_r session.services_by_project.__setitem__(context.name, result.services), session.started_context_names.append(context.name), ), - render_project_startup_warnings=lambda *, context, warnings, route, project_spinner_group: warnings and None, + render_project_startup_warnings=lambda *, context, warnings, route, project_spinner_group: ( + warnings and None + ), should_degrade_to_plan_agent_handoff=lambda session, error: False, record_plan_agent_handoff_local_startup_failure=lambda session, project_name, error: None, spinner_factory=_SpinnerFactory(), @@ -133,7 +240,9 @@ def test_start_selected_contexts_builds_execution_route_and_records_sequential_r self.assertIsNone(execution_route.flags["_spinner_update_project"]) self.assertTrue(execution_route.flags["debug_suppress_progress_output"]) self.assertEqual(runtime.events[0][0], "spinner.policy") - self.assertIn(("startup.execution", {"mode": "sequential", "workers": 1, "projects": ["Alpha"]}), runtime.events) + self.assertIn( + ("startup.execution", {"mode": "sequential", "workers": 1, "projects": ["Alpha"]}), runtime.events + ) def test_record_project_startup_updates_session_result_maps_and_started_names(self) -> None: route = Route(command="start", mode="trees", raw_args=["start"], passthrough_args=[], projects=[], flags={}) @@ -194,13 +303,12 @@ def test_runtime_bound_selected_context_startup_records_results_and_uses_owner_d runtime, session, resolved_run_id=lambda session: "run-1", - render_project_startup_warnings=lambda *, context, warnings, route, project_spinner_group: rendered_warnings.append( - (context.name, list(warnings)) + render_project_startup_warnings=lambda *, context, warnings, route, project_spinner_group: ( + rendered_warnings.append((context.name, list(warnings))) + ), + should_degrade_to_plan_agent_handoff=lambda session, error: ( + degradation_checks.append((session.requested_command, error)) or False ), - should_degrade_to_plan_agent_handoff=lambda session, error: degradation_checks.append( - (session.requested_command, error) - ) - or False, record_plan_agent_handoff_local_startup_failure=lambda session, project_name, error: local_failures.append( (project_name, error) ), diff --git a/tests/python/startup/test_service_attach_execution.py b/tests/python/startup/test_service_attach_execution.py index 0c61b31f..c737c976 100644 --- a/tests/python/startup/test_service_attach_execution.py +++ b/tests/python/startup/test_service_attach_execution.py @@ -1,12 +1,15 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Callable, Iterable +import json from pathlib import Path +import tempfile from types import SimpleNamespace from typing import cast import unittest -from envctl_engine.runtime.service_manager import ServiceStartDescriptor +from envctl_engine.runtime.service_manager import ServiceCleanupError, ServiceManager, ServiceStartDescriptor +from envctl_engine.shared.ports import PortPlanner from envctl_engine.startup.service_attach_execution import ServiceAttachRunner from envctl_engine.startup.service_execution_records import PreparedServiceLaunch from envctl_engine.state.models import PortPlan, ServiceRecord @@ -17,6 +20,254 @@ def port_plan(port: int) -> PortPlan: class ServiceAttachExecutionTests(unittest.TestCase): + def test_later_layer_failure_rolls_back_and_tracks_prior_layer_records(self) -> None: + project_root = Path("/tmp/envctl-project") + voice = SimpleNamespace( + name="voice-runtime", + start_cmd="python voice.py", + depends_on=("backend",), + start_order=25, + expect_listener=True, + critical=True, + env_suffix="VOICE_RUNTIME", + ) + worker = SimpleNamespace( + name="worker", + start_cmd="python worker.py", + depends_on=("voice-runtime",), + start_order=50, + expect_listener=False, + critical=True, + env_suffix="WORKER", + ) + attach_layers: list[tuple[str, ...]] = [] + backend_record = ServiceRecord( + name="Main Backend", + type="backend", + cwd=str(project_root / "backend"), + pid=61001, + actual_port=8000, + ) + + def generic_attach(**kwargs: object) -> dict[str, ServiceRecord]: + descriptors = tuple(cast(Iterable[ServiceStartDescriptor], kwargs["descriptors"])) + layer = tuple(descriptor.service_type for descriptor in descriptors) + attach_layers.append(layer) + if "worker" not in layer: + return {backend_record.name: backend_record} + raise RuntimeError("worker startup failed") + + runtime = SimpleNamespace( + services=SimpleNamespace(start_services_with_attach=generic_attach), + _terminate_started_services=lambda _services: (_ for _ in ()).throw(OSError("layer terminator crashed")), + config=SimpleNamespace( + app_service_by_name=lambda name: {"voice-runtime": voice, "worker": worker}.get(name) + ), + _conflict_remaining={}, + _emit=lambda *_args, **_kwargs: None, + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(), + port_allocator=SimpleNamespace( + reserve_next=lambda port, owner: port, + session_id="attach-session", + ), + project_name="Main", + project_root=project_root, + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=project_root / "backend", + frontend_cwd=project_root / "frontend", + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=project_root / "backend", + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ), + "worker": PreparedServiceLaunch( + service_name="worker", + cwd=project_root / "worker", + log_path="/logs/worker.txt", + requested_port=0, + env={}, + command_source="configured", + listener_expected=False, + ), + "voice-runtime": PreparedServiceLaunch( + service_name="voice-runtime", + cwd=project_root / "voice-runtime", + log_path="/logs/voice.txt", + requested_port=8010, + env={}, + command_source="configured", + ), + }, + selected_service_types={"backend", "voice-runtime", "worker"}, + additional_services=(voice, worker), + backend_listener_expected=True, + rebound_delta=0, + ) + + with self.assertRaises(ServiceCleanupError) as raised: + runner.start(attach_parallel=False, on_service_retry=lambda *_args: None) + + self.assertEqual(attach_layers, [("backend", "voice-runtime"), ("worker",)]) + self.assertIs(raised.exception.unterminated_services["Main Backend"], backend_record) + self.assertEqual(backend_record.port_lock_session, "attach-session") + self.assertIn("cleanup error: layer terminator crashed", str(raised.exception)) + + def test_process_launch_without_pid_never_invents_envctl_or_adjacent_pid(self) -> None: + project_root = Path("/tmp/envctl-project") + runtime = SimpleNamespace( + _conflict_remaining={}, + _emit=lambda *_args, **_kwargs: None, + _service_start_command_resolved=lambda **_kwargs: (["backend"], "configured"), + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(start_background=lambda *_args, **_kwargs: object()), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="Main", + project_root=project_root, + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=project_root / "backend", + frontend_cwd=project_root / "frontend", + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=project_root / "backend", + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + success, error, pid = runner.start_backend(8000) + + self.assertTrue(success) + self.assertIsNone(error) + self.assertIsNone(pid) + + def test_post_spawn_telemetry_failure_still_returns_real_pid_to_manager(self) -> None: + project_root = Path("/tmp/envctl-project") + + def emit(event: str, **payload: object) -> None: + if event == "service.start" or payload.get("phase") == "process_launch": + raise OSError("event sink failed") + + runtime = SimpleNamespace( + _conflict_remaining={}, + _emit=emit, + _service_start_command_resolved=lambda **_kwargs: (["backend"], "configured"), + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(start_background=lambda *_args, **_kwargs: SimpleNamespace(pid=62001)), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="Main", + project_root=project_root, + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=project_root / "backend", + frontend_cwd=project_root / "frontend", + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=project_root / "backend", + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + success, error, pid = runner.start_backend(8000) + + self.assertTrue(success) + self.assertIsNone(error) + self.assertEqual(pid, 62001) + + def test_process_spawn_error_is_structured_without_listener_wait(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + runtime = SimpleNamespace( + _conflict_remaining={}, + _emit=lambda event, **payload: events.append((event, payload)), + _service_start_command_resolved=lambda **_kwargs: (["missing-backend"], "configured"), + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace( + start_background=lambda *_args, **_kwargs: (_ for _ in ()).throw( + FileNotFoundError("missing interpreter") + ) + ), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="Main", + project_root=Path("/repo"), + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=Path("/repo/backend"), + frontend_cwd=Path("/repo/frontend"), + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=Path("/repo/backend"), + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + success, error, pid = runner.start_backend(8000) + + self.assertFalse(success) + self.assertIn("process spawn failed for missing-backend", error) + self.assertIsNone(pid) + failure_events = [payload for event, payload in events if event == "service.failure"] + self.assertEqual(failure_events[0]["failure_class"], "process_spawn_failed") + def test_runner_builds_layered_descriptors_and_preserves_additional_urls(self) -> None: project_root = Path("/tmp/envctl-project") events: list[tuple[str, dict[str, object]]] = [] @@ -92,14 +343,16 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: ) process_runtime = SimpleNamespace( start_background=lambda command, cwd, env, stdout_path, stderr_path: ( - process_commands.append((list(command), str(cwd), int(env["PORT"]))) - or SimpleNamespace(pid=4321) + process_commands.append((list(command), str(cwd), int(env["PORT"]))) or SimpleNamespace(pid=4321) ) ) runner = ServiceAttachRunner( runtime=runtime, process_runtime=process_runtime, - port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + port_allocator=SimpleNamespace( + reserve_next=lambda port, owner: port, + session_id="layered-session", + ), project_name="Main", project_root=project_root, backend_plan=port_plan(8000), @@ -152,12 +405,26 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: self.assertEqual(layers, [("backend", "voice-runtime"), ("worker",)]) self.assertEqual(records["Main Backend"].actual_port, 8101) + self.assertEqual( + {record.port_lock_session for record in records.values()}, + {"layered-session"}, + ) self.assertEqual(records["Main Voice Runtime"].actual_port, 8022) self.assertIsNone(records["Main Worker"].actual_port) self.assertEqual(records["Main Voice Runtime"].public_url, "http://localhost:8010") self.assertIn((["backend", "8000"], str(project_root / "backend"), 8000), process_commands) - self.assertIn((["voice.py", "8010", "voice-runtime", str(project_root / "voice-runtime")], str(project_root / "voice-runtime"), 8010), process_commands) - self.assertIn(("service.bind.skipped", {"project": "Main", "service": "worker", "reason": "listener_not_expected"}), events) + self.assertIn( + ( + ["voice.py", "8010", "voice-runtime", str(project_root / "voice-runtime")], + str(project_root / "voice-runtime"), + 8010, + ), + process_commands, + ) + self.assertIn( + ("service.bind.skipped", {"project": "Main", "service": "worker", "reason": "listener_not_expected"}), + events, + ) self.assertTrue( any( event == "service.attach.phase" @@ -182,7 +449,10 @@ def test_additional_listener_truth_failure_emits_failure_event(self) -> None: services=SimpleNamespace(), _conflict_remaining={}, _emit=lambda event, **payload: events.append((event, payload)), - _service_start_command_resolved=lambda service_name, project_root, port: ([service_name, str(port)], "configured"), + _service_start_command_resolved=lambda service_name, project_root, port: ( + [service_name, str(port)], + "configured", + ), _detect_service_actual_port=lambda **kwargs: None, _listener_truth_enforced=lambda: True, _service_listener_failure_detail=lambda **kwargs: "process exited", @@ -220,7 +490,7 @@ def test_additional_listener_truth_failure_emits_failure_event(self) -> None: with self.assertRaisesRegex( RuntimeError, - r"voice-runtime listener not detected for Main on port 8010 \(process exited\)", + r"voice-runtime process exited before opening its listener for Main on port 8010 \(process exited\)", ): runner.detect_additional_actual("voice-runtime", pid=4321, requested=8010) @@ -230,7 +500,7 @@ def test_additional_listener_truth_failure_emits_failure_event(self) -> None: { "project": "Main", "service": "voice-runtime", - "failure_class": "listener_not_detected", + "failure_class": "process_exited", "requested_port": 8010, "detail": "process exited", }, @@ -238,16 +508,76 @@ def test_additional_listener_truth_failure_emits_failure_event(self) -> None: events, ) + def test_missing_nested_executable_is_reported_as_dependency_failure(self) -> None: + events: list[tuple[str, dict[str, object]]] = [] + detail = ( + "process 2304354 exited; log_path: /tmp/voice-runtime.txt; " + "log: scripts/envctl/start-voice-runtime.sh: line 20: exec: python: not found" + ) + runtime = SimpleNamespace( + services=SimpleNamespace(), + _conflict_remaining={}, + _emit=lambda event, **payload: events.append((event, payload)), + _detect_service_actual_port=lambda **_kwargs: None, + _listener_truth_enforced=lambda: True, + _service_listener_failure_detail=lambda **_kwargs: detail, + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(), + port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + project_name="refactoring-elevenlabs-agent-runtime-automation-and-pipecat-strangler-3", + project_root=Path("/repo"), + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=Path("/repo/backend"), + frontend_cwd=Path("/repo/frontend"), + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "voice-runtime": PreparedServiceLaunch( + service_name="voice-runtime", + cwd=Path("/repo/voice-runtime"), + log_path="/tmp/voice-runtime.txt", + requested_port=8251, + env={"PORT": "8251"}, + command_source="configured", + ) + }, + selected_service_types={"voice-runtime"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + with self.assertRaisesRegex( + RuntimeError, + "voice-runtime is missing a required executable or module", + ): + runner.detect_additional_actual("voice-runtime", pid=2304354, requested=8251) + + failure_events = [payload for event, payload in events if event == "service.failure"] + self.assertEqual(failure_events[0]["failure_class"], "dependency_missing") + self.assertEqual(failure_events[0]["detail"], detail) + def test_core_only_runner_uses_legacy_service_manager_when_generic_attach_is_missing(self) -> None: attach_kwargs: dict[str, object] = {} runtime = SimpleNamespace( services=SimpleNamespace( - start_project_with_attach=lambda **kwargs: attach_kwargs.update(kwargs) - or {"Main Backend": ServiceRecord(name="Main Backend", type="backend", cwd="/repo")} + start_project_with_attach=lambda **kwargs: ( + attach_kwargs.update(kwargs) + or {"Main Backend": ServiceRecord(name="Main Backend", type="backend", cwd="/repo")} + ) ), _conflict_remaining={}, _emit=lambda event, **payload: None, - _service_start_command_resolved=lambda service_name, project_root, port: ([service_name, str(port)], "configured"), + _service_start_command_resolved=lambda service_name, project_root, port: ( + [service_name, str(port)], + "configured", + ), _detect_service_actual_port=lambda **kwargs: kwargs["requested_port"], _listener_truth_enforced=lambda: True, _service_listener_failure_detail=lambda **kwargs: "", @@ -255,7 +585,10 @@ def test_core_only_runner_uses_legacy_service_manager_when_generic_attach_is_mis runner = ServiceAttachRunner( runtime=runtime, process_runtime=SimpleNamespace(start=lambda *args, **kwargs: SimpleNamespace(pid=123)), - port_allocator=SimpleNamespace(reserve_next=lambda port, owner: port), + port_allocator=SimpleNamespace( + reserve_next=lambda port, owner: port, + session_id="legacy-attach-session", + ), project_name="Main", project_root=Path("/repo"), backend_plan=port_plan(8000), @@ -286,9 +619,135 @@ def test_core_only_runner_uses_legacy_service_manager_when_generic_attach_is_mis records = runner.start(attach_parallel=False, on_service_retry=lambda *args: None) self.assertIn("Main Backend", records) + self.assertEqual( + records["Main Backend"].port_lock_session, + "legacy-attach-session", + ) self.assertTrue(callable(attach_kwargs["start_backend"])) self.assertFalse(attach_kwargs["parallel_start"]) + def test_multiple_service_retries_release_every_failed_port_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="startup-session", + availability_checker=lambda _port: True, + ) + planner.reserve_next(8000, owner="Main:backend") + + def generic_attach(**kwargs: object) -> dict[str, ServiceRecord]: + reserve_next = cast(Callable[[int], int], kwargs["reserve_next"]) + on_retry = cast(Callable[[str, int, int, int, str | None], None], kwargs["on_retry"]) + retry_one = reserve_next(8001) + on_retry("backend", 8000, retry_one, 1, "address already in use") + retry_two = reserve_next(retry_one + 1) + on_retry("backend", retry_one, retry_two, 2, "address already in use") + return {} + + runtime = SimpleNamespace( + services=SimpleNamespace(start_services_with_attach=generic_attach), + _conflict_remaining={}, + _emit=lambda *_args, **_kwargs: None, + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=SimpleNamespace(), + port_allocator=planner, + project_name="Main", + project_root=Path("/repo"), + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=Path("/repo/backend"), + frontend_cwd=Path("/repo/frontend"), + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "backend": PreparedServiceLaunch( + service_name="backend", + cwd=Path("/repo/backend"), + log_path="/logs/backend.txt", + requested_port=8000, + env={}, + command_source="configured", + ) + }, + selected_service_types={"backend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=0, + ) + + runner.start(attach_parallel=False, on_service_retry=lambda *_args: None) + + locks = list(Path(tmpdir).glob("*.lock")) + self.assertEqual([path.name for path in locks], ["8002.lock"]) + payload = json.loads(locks[0].read_text(encoding="utf-8")) + self.assertEqual(payload["owner"], "Main:services") + + def test_failed_rebound_frontend_releases_attempt_locks_without_bulk_session_release(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + planner = PortPlanner( + lock_dir=tmpdir, + session_id="startup-session", + availability_checker=lambda _port: True, + ) + planner.reserve_next(5173, owner="Main:frontend") + live_pids = {123} + process_runtime = SimpleNamespace( + start_background=lambda *_args, **_kwargs: SimpleNamespace(pid=123), + is_pid_running=lambda pid: pid in live_pids, + terminate_process_group=lambda pid, **_kwargs: live_pids.discard(pid) or True, + ) + runtime = SimpleNamespace( + services=ServiceManager(), + _conflict_remaining={}, + _emit=lambda *_args, **_kwargs: None, + _service_start_command_resolved=lambda **_kwargs: (["frontend"], "configured"), + _detect_service_actual_port=lambda **_kwargs: None, + _listener_truth_enforced=lambda: True, + _service_listener_failure_detail=lambda **_kwargs: "listener missing", + ) + runner = ServiceAttachRunner( + runtime=runtime, + process_runtime=process_runtime, + port_allocator=planner, + project_name="Main", + project_root=Path("/repo"), + backend_plan=port_plan(8000), + frontend_plan=port_plan(5173), + backend_cwd=Path("/repo/backend"), + frontend_cwd=Path("/repo/frontend"), + backend_log_path="/logs/backend.txt", + frontend_log_path="/logs/frontend.txt", + backend_env_extra={}, + frontend_env_extra={}, + command_env_builder=lambda port, extra: dict(extra), + prepared_launches={ + "frontend": PreparedServiceLaunch( + service_name="frontend", + cwd=Path("/repo/frontend"), + log_path="/logs/frontend.txt", + requested_port=5173, + env={}, + command_source="configured", + ) + }, + selected_service_types={"frontend"}, + additional_services=(), + backend_listener_expected=True, + rebound_delta=2, + ) + + with self.assertRaisesRegex(RuntimeError, "listener not detected"): + runner.start(attach_parallel=False, on_service_retry=lambda *_args: None) + + # Failure finalization keeps the whole session when managed + # requirements remain, so per-attempt cleanup must be sufficient. + self.assertEqual(list(Path(tmpdir).glob("*.lock")), []) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/startup/test_service_manager.py b/tests/python/startup/test_service_manager.py index 06186f09..eaafd718 100644 --- a/tests/python/startup/test_service_manager.py +++ b/tests/python/startup/test_service_manager.py @@ -3,10 +3,17 @@ import time import unittest from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" -from envctl_engine.runtime.service_manager import ServiceManager, ServiceStartDescriptor +from envctl_engine.runtime.service_manager import ( + ServiceCleanupError, + ServiceManager, + ServiceStartDescriptor, + ServiceStartError, +) class ServiceManagerTests(unittest.TestCase): @@ -149,11 +156,14 @@ def fail_backend(_pid: int | None, _requested: int) -> int: ) def test_listener_detection_failure_is_not_retried_as_port_conflict(self) -> None: - manager = ServiceManager() attempts: list[int] = [] terminated: list[int] = [] - - manager.terminate_process_group = lambda pid, **kwargs: terminated.append(pid) or True # type: ignore[attr-defined] + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda pid, **_kwargs: terminated.append(pid) or True, + ) + ) def start_backend(port: int) -> tuple[bool, str | None, int | None]: attempts.append(port) @@ -226,7 +236,6 @@ def test_non_listener_service_can_start_without_persisted_ports(self) -> None: self.assertIsNone(service.requested_port) self.assertIsNone(service.actual_port) - def test_start_services_with_attach_returns_degraded_record_for_noncritical_failure(self) -> None: manager = ServiceManager() descriptors = ( @@ -330,9 +339,13 @@ def test_start_services_with_attach_starts_arbitrary_services_and_non_listeners( self.assertIsNone(records["Main Worker"].actual_port) def test_start_services_with_attach_cleans_partial_records_on_failure(self) -> None: - manager = ServiceManager() terminated: list[int] = [] - manager.terminate_process_group = lambda pid, **kwargs: terminated.append(pid) or True # type: ignore[attr-defined] + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda pid, **_kwargs: terminated.append(pid) or True, + ) + ) descriptors = ( ServiceStartDescriptor( @@ -361,6 +374,186 @@ def test_start_services_with_attach_cleans_partial_records_on_failure(self) -> N self.assertEqual(terminated, [41001]) + def test_listener_failure_fails_closed_when_group_and_raw_cleanup_cannot_prove_exit(self) -> None: + runner = SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: False, + terminate=lambda *_args, **_kwargs: False, + ) + manager = ServiceManager(process_runner=runner) + + with ( + patch("envctl_engine.shared.process_termination._send_signal", return_value=True), + patch("envctl_engine.shared.process_termination.wait_for_pid_exit", side_effect=[False, False]), + self.assertRaises(ServiceCleanupError) as raised, + ): + manager.start_service_with_retry( + project="Main", + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=lambda _port: (True, None, 51001), + reserve_next=lambda port: port, + detect_actual=lambda *_args: (_ for _ in ()).throw(RuntimeError("backend listener not detected")), + ) + + record = raised.exception.unterminated_services["Main Backend"] + self.assertEqual(record.pid, 51001) + self.assertEqual(record.status, "termination_failed") + + def test_non_runtime_detection_exception_still_terminates_started_pid(self) -> None: + live_pids = {51501} + terminated: list[int] = [] + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + terminate_process_group=lambda pid, **_kwargs: (terminated.append(pid), live_pids.discard(pid), True)[ + -1 + ], + ) + ) + + with self.assertRaisesRegex(ServiceStartError, "listener probe crashed"): + manager.start_service_with_retry( + project="Main", + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=lambda _port: (True, None, 51501), + reserve_next=lambda port: port, + detect_actual=lambda *_args: (_ for _ in ()).throw(OSError("listener probe crashed")), + ) + + self.assertEqual(terminated, [51501]) + + def test_failed_launcher_pid_is_terminated_before_bind_retry(self) -> None: + attempts: list[int] = [] + terminated: list[int] = [] + live_pids = {52001} + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda pid: pid in live_pids, + terminate_process_group=lambda pid, **_kwargs: (terminated.append(pid), live_pids.discard(pid), True)[ + -1 + ], + ) + ) + + def start(port: int) -> tuple[bool, str | None, int | None]: + attempts.append(port) + if len(attempts) == 1: + return False, "address already in use", 52001 + return True, None, 52002 + + record = manager.start_service_with_retry( + project="Main", + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=start, + reserve_next=lambda port: port, + detect_actual=lambda _pid, requested: requested, + ) + + self.assertEqual(attempts, [8000, 8001]) + self.assertEqual(terminated, [52001]) + self.assertEqual(record.pid, 52002) + + def test_parallel_rollback_reports_every_service_whose_exit_is_unconfirmed(self) -> None: + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: False, + terminate=lambda *_args, **_kwargs: False, + ) + ) + descriptors = ( + ServiceStartDescriptor( + service_type="backend", + cwd="/repo/backend", + requested_port=8000, + start=lambda _port: (True, None, 53001), + detect_actual=lambda _pid, requested: requested, + ), + ServiceStartDescriptor( + service_type="frontend", + cwd="/repo/frontend", + requested_port=3000, + start=lambda _port: (False, "boot failed", None), + max_retries=0, + ), + ) + + with ( + patch("envctl_engine.shared.process_termination._send_signal", return_value=True), + patch("envctl_engine.shared.process_termination.wait_for_pid_exit", side_effect=[False, False]), + self.assertRaises(ServiceCleanupError) as raised, + ): + manager.start_services_with_attach( + project="Main", + descriptors=descriptors, + reserve_next=lambda port: port, + parallel_start=True, + ) + + self.assertEqual(set(raised.exception.unterminated_services), {"Main Backend"}) + + def test_noncritical_listener_cleanup_failure_is_never_downgraded_to_pidless_record(self) -> None: + manager = ServiceManager( + process_runner=SimpleNamespace( + is_pid_running=lambda _pid: True, + terminate_process_group=lambda *_args, **_kwargs: False, + terminate=lambda *_args, **_kwargs: False, + ) + ) + descriptor = ServiceStartDescriptor( + service_type="worker", + cwd="/repo/worker", + requested_port=0, + start=lambda _port: (True, None, 54001), + detect_actual=lambda *_args: (_ for _ in ()).throw(RuntimeError("worker attach failed")), + listener_expected=False, + critical=False, + ) + + with ( + patch("envctl_engine.shared.process_termination._send_signal", return_value=True), + patch("envctl_engine.shared.process_termination.wait_for_pid_exit", side_effect=[False, False]), + self.assertRaises(ServiceCleanupError) as raised, + ): + manager.start_services_with_attach( + project="Main", + descriptors=(descriptor,), + reserve_next=lambda port: port, + ) + + record = raised.exception.unterminated_services["Main Worker"] + self.assertEqual(record.pid, 54001) + self.assertEqual(record.status, "termination_failed") + + def test_success_without_valid_pid_is_tracked_as_unconfirmed_launch_even_for_non_listener(self) -> None: + manager = ServiceManager() + descriptor = ServiceStartDescriptor( + service_type="worker", + cwd="/repo/worker", + requested_port=0, + start=lambda _port: (True, None, None), + listener_expected=False, + critical=False, + ) + + with self.assertRaises(ServiceCleanupError) as raised: + manager.start_services_with_attach( + project="Main", + descriptors=(descriptor,), + reserve_next=lambda port: port, + ) + + record = raised.exception.unterminated_services["Main Worker"] + self.assertIsNone(record.pid) + self.assertEqual(record.status, "termination_failed") + self.assertFalse(record.listener_expected) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/startup/test_startup_dashboard_stopped_restore.py b/tests/python/startup/test_startup_dashboard_stopped_restore.py index af1dea6a..ad6401b8 100644 --- a/tests/python/startup/test_startup_dashboard_stopped_restore.py +++ b/tests/python/startup/test_startup_dashboard_stopped_restore.py @@ -77,7 +77,9 @@ def test_startup_restores_dashboard_stopped_services_instead_of_auto_resuming_pa ) captured_state: dict[str, RunState] = {} - engine._try_load_existing_state = lambda mode=None, strict_mode_match=False: previous_state # type: ignore[method-assign] + engine._try_load_existing_state = ( # type: ignore[method-assign] + lambda mode=None, strict_mode_match=False, project_names=None: previous_state + ) engine._discover_projects = lambda mode: [context] # type: ignore[method-assign] engine._reserve_project_ports = lambda _context, route=None: None # type: ignore[method-assign] engine._resume = lambda route: self.fail("startup should start stopped services, not auto-resume") # type: ignore[method-assign] diff --git a/tests/python/startup/test_startup_finalization.py b/tests/python/startup/test_startup_finalization.py index 72cc316b..db90245f 100644 --- a/tests/python/startup/test_startup_finalization.py +++ b/tests/python/startup/test_startup_finalization.py @@ -34,7 +34,7 @@ from envctl_engine.planning.plan_agent.models import CreatedPlanWorktree, PlanAgentLaunchOutcome, PlanAgentLaunchResult from envctl_engine.startup.finalization_failure import StartupFailureFinalizer from envctl_engine.startup.session import LocalStartupFailure, StartupSession -from envctl_engine.state.models import RunState +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord def _session(*, contexts: list[object], contexts_to_start: list[object] | None = None) -> StartupSession: @@ -72,6 +72,233 @@ def _runtime_stub(**overrides: object) -> SimpleNamespace: class StartupFinalizationTests(unittest.TestCase): + def test_failure_finalization_persists_service_when_exit_remains_unconfirmed(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=64001, + actual_port=8000, + ) + session.unterminated_services[service.name] = service + writes: list[RunState] = [] + released = {"value": False} + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("event sink failed")), + _terminate_started_services=lambda _services: {service.name}, + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + result = finalize_failed_startup( + runtime=runtime, + session=session, + error="listener truth failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace( + release_session=lambda: released.__setitem__("value", True) + ), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual(result, 1) + self.assertEqual(writes[0].metadata["termination_failed_services"], [service.name]) + self.assertIs(writes[0].services[service.name], service) + self.assertEqual(service.status, "termination_failed") + self.assertFalse(released["value"]) + + def test_failure_finalization_preserves_port_session_for_started_requirements(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + requirements = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6379}, + ) + session.requirements_by_project["Main"] = requirements + writes: list[RunState] = [] + released = {"value": False} + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: set(), + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="backend launch failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace( + release_session=lambda: released.__setitem__("value", True) + ), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertIs(writes[0].requirements["Main"], requirements) + self.assertFalse(released["value"]) + + def test_successful_rollback_is_removed_from_failure_state_when_requirements_remain(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=64002, + requested_port=8000, + actual_port=8000, + ) + session.started_context_names.append("Main") + session.services_by_project["Main"] = {service.name: service} + session.requirements_by_project["Main"] = RequirementsResult( + project="Main", + redis={"enabled": True, "success": True, "final": 6379}, + ) + writes: list[RunState] = [] + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: set(), + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="frontend launch failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual(writes[0].services, {}) + self.assertEqual(session.services_by_project, {}) + self.assertEqual(session.unterminated_services, {}) + + def test_missing_termination_result_retains_service_and_port_session(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=64003, + requested_port=8000, + actual_port=8000, + ) + session.started_context_names.append("Main") + session.services_by_project["Main"] = {service.name: service} + writes: list[RunState] = [] + released = {"value": False} + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: None, + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="frontend launch failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace( + release_session=lambda: released.__setitem__("value", True) + ), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertIs(writes[0].services[service.name], service) + self.assertEqual(writes[0].metadata["termination_failed_services"], [service.name]) + self.assertEqual(service.status, "termination_failed") + self.assertFalse(released["value"]) + + def test_failure_finalization_releases_port_session_for_external_only_requirements(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + requirements = RequirementsResult( + project="Main", + redis={ + "enabled": True, + "success": True, + "external": True, + "final": 6379, + }, + ) + session.requirements_by_project["Main"] = requirements + released = {"value": False} + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: set(), + _write_artifacts=lambda *_args, **_kwargs: None, + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="backend launch failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace( + release_session=lambda: released.__setitem__("value", True) + ), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertTrue(released["value"]) + + def test_failure_finalization_releases_port_session_for_clean_internal_start_failure(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + session.requirements_by_project["Main"] = RequirementsResult( + project="Main", + redis={"enabled": True, "success": False, "final": 6379, "error": "command missing"}, + ) + released = {"value": False} + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: None, + _terminate_started_services=lambda _services: set(), + _write_artifacts=lambda *_args, **_kwargs: None, + ) + + finalize_failed_startup( + runtime=runtime, + session=session, + error="requirements failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace( + release_session=lambda: released.__setitem__("value", True) + ), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertTrue(released["value"]) + + def test_failure_event_exception_does_not_skip_cleanup_or_failure_state_write(self) -> None: + session = _session(contexts=[SimpleNamespace(name="Main")]) + service = ServiceRecord(name="Main Backend", type="backend", cwd="/repo", pid=65001) + session.started_context_names.append("Main") + session.services_by_project["Main"] = {service.name: service} + terminated: list[dict[str, ServiceRecord]] = [] + writes: list[RunState] = [] + runtime = _runtime_stub( + _emit=lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("event sink failed")), + _terminate_started_services=lambda services: terminated.append(dict(services)) or set(), + _write_artifacts=lambda state, *_args, **_kwargs: writes.append(state), + ) + + result = finalize_failed_startup( + runtime=runtime, + session=session, + error="backend failed", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda *_args, **_kwargs: None, + render_final_failure_status=lambda _runtime, _session, error, **_kwargs: error, + ) + + self.assertEqual(result, 1) + self.assertEqual(terminated, [{service.name: service}]) + self.assertEqual(len(writes), 1) + def test_failure_finalization_uses_named_owner(self) -> None: self.assertTrue(callable(StartupFailureFinalizer.finalize)) @@ -91,7 +318,7 @@ def test_finalize_failed_startup_writes_error_artifacts_and_releases_ports(self) env={}, _emit=lambda event, **payload: events.append((event, payload)), _write_artifacts=lambda state, contexts, *, errors: writes.append((state, list(contexts), list(errors))), - _terminate_started_services=lambda services: terminated.append(dict(services)), + _terminate_started_services=lambda services: terminated.append(dict(services)) or set(), ) result = finalize_failed_startup( @@ -101,8 +328,9 @@ def test_finalize_failed_startup_writes_error_artifacts_and_releases_ports(self) ensure_run_id=lambda session: None, port_allocator=lambda runtime: SimpleNamespace(release_session=lambda: released.__setitem__("value", True)), emit_phase=lambda session, phase, started_at, **extra: events.append((f"phase:{phase}", dict(extra))), - render_final_failure_status=lambda runtime, session, error, *, interactive_tty: rendered_statuses.append(error) - or error, + render_final_failure_status=lambda runtime, session, error, *, interactive_tty: ( + rendered_statuses.append(error) or error + ), ) self.assertEqual(result, 1) @@ -170,6 +398,39 @@ def test_finalize_failed_startup_preserves_existing_startup_failure_prefix_and_s events, ) + def test_finalize_failed_startup_does_not_shadow_existing_state_after_failed_replacement(self) -> None: + session = _session(contexts=[SimpleNamespace(name="feature-a")]) + session.preserve_existing_state_on_failure = True + events: list[tuple[str, dict[str, object]]] = [] + runtime = _runtime_stub( + _emit=lambda event, **payload: events.append((event, payload)), + _write_artifacts=lambda *_args, **_kwargs: self.fail("existing state must remain authoritative"), + _terminate_started_services=lambda _services: None, + ) + + result = finalize_failed_startup( + runtime=runtime, + session=session, + error="existing service did not terminate", + ensure_run_id=lambda _session: None, + port_allocator=lambda _runtime: SimpleNamespace(release_session=lambda: None), + emit_phase=lambda _session, phase, _started_at, **extra: events.append((f"phase:{phase}", extra)), + render_final_failure_status=lambda _runtime, _session, error, *, interactive_tty: error, + ) + + self.assertEqual(result, 1) + self.assertIn( + ( + "state.failure_write.skipped", + { + "reason": "existing_runtime_state_remains_authoritative", + "run_id": "run-finalization", + }, + ), + events, + ) + self.assertIn(("phase:artifacts_write", {"status": "skipped"}), events) + def test_finalize_successful_startup_writes_artifacts_summary_snapshot_and_returns_zero(self) -> None: context = SimpleNamespace(name="Main") session = _session(contexts=[context]) @@ -275,7 +536,10 @@ def test_runtime_bound_success_finalization_writes_artifacts_and_snapshot(self) self.assertEqual(writes[0][1], [context]) self.assertEqual(summaries[0][1], [context]) self.assertIn(("validate", {"phase": "success_finalization"}), events) - self.assertIn("before_dashboard_entry", [payload.get("checkpoint") for event, payload in events if event == "ui.plan_handoff.snapshot"]) + self.assertIn( + "before_dashboard_entry", + [payload.get("checkpoint") for event, payload in events if event == "ui.plan_handoff.snapshot"], + ) def test_finalize_plan_agent_degraded_handoff_writes_artifacts_and_attach_result(self) -> None: session = _session(contexts=[]) @@ -319,9 +583,7 @@ def test_finalize_plan_agent_degraded_handoff_artifacts_returns_run_state(self) result = finalize_plan_agent_degraded_handoff_artifacts( runtime=SimpleNamespace( - _write_artifacts=lambda state, contexts, *, errors: writes.append( - (state, list(contexts), list(errors)) - ) + _write_artifacts=lambda state, contexts, *, errors: writes.append((state, list(contexts), list(errors))) ), session=session, ensure_run_id=lambda session: events.append(("ensure", {})), @@ -583,7 +845,9 @@ def test_print_headless_plan_session_summary_uses_attach_target_override_without print_headless_plan_session_summary( session, attach_target=attach_target, - validate_plan_agent_handoff=lambda session, *, phase: self.fail("override attach target should skip validation"), + validate_plan_agent_handoff=lambda session, *, phase: self.fail( + "override attach target should skip validation" + ), print_fn=lines.append, ) @@ -771,7 +1035,13 @@ def test_restart_port_rebound_summary_lines_deduplicate_port_changes(self) -> No startup_event_index=1, ) events = [ - {"event": "port.rebound", "project": "ignored", "service": "backend", "restart_preferred_port": 1, "port": 2}, + { + "event": "port.rebound", + "project": "ignored", + "service": "backend", + "restart_preferred_port": 1, + "port": 2, + }, { "event": "port.rebound", "project": "feature-a-1", diff --git a/tests/python/startup/test_startup_fingerprint_replacement.py b/tests/python/startup/test_startup_fingerprint_replacement.py index d5d9f6df..86ab9780 100644 --- a/tests/python/startup/test_startup_fingerprint_replacement.py +++ b/tests/python/startup/test_startup_fingerprint_replacement.py @@ -73,7 +73,9 @@ def test_start_fingerprint_mismatch_replaces_existing_project_services_before_ne terminate_calls: list[tuple[set[str] | None, bool]] = [] call_order: list[str] = [] - engine._try_load_existing_state = lambda mode=None, strict_mode_match=False: previous_state # type: ignore[method-assign] + engine._try_load_existing_state = ( # type: ignore[method-assign] + lambda mode=None, strict_mode_match=False, project_names=None: previous_state + ) engine._discover_projects = lambda mode: [context] # type: ignore[method-assign] engine._terminate_services_from_state = ( # type: ignore[method-assign] lambda state, selected_services, aggressive, verify_ownership: ( @@ -109,7 +111,7 @@ def test_start_fingerprint_mismatch_replaces_existing_project_services_before_ne warnings=[], ), )[1] - ) + ) code = engine.dispatch(parse_route(["--main", "--managed-deps", "--batch"], env={})) diff --git a/tests/python/startup/test_startup_orchestrator_flow_reuse.py b/tests/python/startup/test_startup_orchestrator_flow_reuse.py index 86c8e7ce..63d80557 100644 --- a/tests/python/startup/test_startup_orchestrator_flow_reuse.py +++ b/tests/python/startup/test_startup_orchestrator_flow_reuse.py @@ -1,10 +1,108 @@ # ruff: noqa: F403,F405 from __future__ import annotations +import json +import threading + +from envctl_engine.runtime.lifecycle_operation_lease import ( + release_lifecycle_operation, + run_exclusive_lifecycle_operation, +) +from envctl_engine.state.run_index import StateSelector + from tests.python.startup.startup_orchestrator_flow_test_support import * class StartupOrchestratorFlowReuseTests(StartupOrchestratorFlowTestCase): + def test_start_heals_multiple_historical_active_owners_for_main(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repo(root) + runtime = root / "runtime" + historical_candidates = [] + + for _index in range(2): + engine = self._engine(repo, runtime) + with ( + patch.object(engine, "_should_enter_post_start_interactive", return_value=False), + redirect_stdout(StringIO()), + ): + code = engine.dispatch( + parse_route(["start", "--batch", "--no-infra"], env={"ENVCTL_DEFAULT_MODE": "main"}) + ) + self.assertEqual(code, 0) + historical_candidates.append( + engine.state_repository.run_index.candidates(StateSelector("main", ("main",)))[0] + ) + + engine.state_repository.run_index.replace_all(historical_candidates) + for offset, candidate in enumerate(historical_candidates): + stale_lock = engine.port_planner.lock_dir / f"{44000 + offset}.lock" + stale_lock.write_text( + json.dumps( + { + "owner": "Main:db", + "session": f"historical-{candidate.run_id}", + "pid": 99999999, + "created_at": 0, + } + ), + encoding="utf-8", + ) + third_engine = self._engine(repo, runtime) + with ( + patch.object(third_engine, "_should_enter_post_start_interactive", return_value=False), + redirect_stdout(StringIO()), + ): + third_code = third_engine.dispatch( + parse_route(["start", "--batch", "--no-infra"], env={"ENVCTL_DEFAULT_MODE": "main"}) + ) + + self.assertEqual(third_code, 0) + index_payload = json.loads((third_engine.runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(len(index_payload["entries"]), 1) + self.assertEqual( + set(index_payload["retired_run_ids"]), + {candidate.run_id for candidate in historical_candidates}, + ) + self.assertEqual(list(third_engine.port_planner.lock_dir.glob("*.lock")), []) + + def test_repeated_no_infra_start_supersedes_old_authority_without_port_locks(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repo(root) + runtime = root / "runtime" + first_engine = self._engine(repo, runtime) + + with ( + patch.object(first_engine, "_should_enter_post_start_interactive", return_value=False), + redirect_stdout(StringIO()), + ): + first_code = first_engine.dispatch( + parse_route(["start", "--batch", "--no-infra"], env={"ENVCTL_DEFAULT_MODE": "main"}) + ) + + self.assertEqual(first_code, 0) + self.assertEqual(list(first_engine.port_planner.lock_dir.glob("*.lock")), []) + first_index = json.loads((first_engine.runtime_root / "run_index.json").read_text(encoding="utf-8")) + first_run_id = first_index["entries"][0]["run_id"] + + second_engine = self._engine(repo, runtime) + with ( + patch.object(second_engine, "_should_enter_post_start_interactive", return_value=False), + redirect_stdout(StringIO()), + ): + second_code = second_engine.dispatch( + parse_route(["start", "--batch", "--no-infra"], env={"ENVCTL_DEFAULT_MODE": "main"}) + ) + + self.assertEqual(second_code, 0) + self.assertEqual(list(second_engine.port_planner.lock_dir.glob("*.lock")), []) + second_index = json.loads((second_engine.runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(len(second_index["entries"]), 1) + self.assertNotEqual(second_index["entries"][0]["run_id"], first_run_id) + self.assertIn(first_run_id, second_index["retired_run_ids"]) + def test_disabled_startup_reopens_existing_dashboard_run_when_identity_matches(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -33,7 +131,11 @@ def test_disabled_startup_reopens_existing_dashboard_run_when_identity_matches(s with ( patch.object(engine, "_discover_projects", return_value=[context]), patch.object(engine, "_try_load_existing_state", return_value=existing_state), - patch.object(engine, "_write_artifacts", side_effect=AssertionError("fresh dashboard state should not be written")), + patch.object( + engine, + "_write_artifacts", + side_effect=AssertionError("fresh dashboard state should not be written"), + ), patch.object(engine, "_should_enter_post_start_interactive", return_value=False), patch.object( engine.state_repository, @@ -146,17 +248,27 @@ def _record_launch(_runtime: object, *, route: object, created_worktrees: tuple[ patch.object( engine.planning_worktree_orchestrator, "last_plan_selection_result", - return_value=PlanSelectionResult(raw_projects=[], selected_contexts=[context], created_worktrees=()), + return_value=PlanSelectionResult( + raw_projects=[], selected_contexts=[context], created_worktrees=() + ), ), patch("envctl_engine.startup.lifecycle.launch_plan_agent_terminals", side_effect=_record_launch), patch("envctl_engine.planning.plan_agent.omx_transport._tmux_session_exists", return_value=True), - patch("envctl_engine.planning.plan_agent.omx_transport._tmux_display_message_succeeds", return_value=(True, "%42")), + patch( + "envctl_engine.planning.plan_agent.omx_transport._tmux_display_message_succeeds", + return_value=(True, "%42"), + ), patch.object(engine, "_write_artifacts"), patch.object(engine, "_should_enter_post_start_interactive", return_value=False), ): out = StringIO() with redirect_stdout(out): - code = engine.dispatch(parse_route(["--plan", "feature-a", "--omx", "--codex", "--headless"], env={"ENVCTL_DEFAULT_MODE": "trees"})) + code = engine.dispatch( + parse_route( + ["--plan", "feature-a", "--omx", "--codex", "--headless"], + env={"ENVCTL_DEFAULT_MODE": "trees"}, + ) + ) self.assertEqual(code, 0) self.assertEqual(captured_created_worktrees, [["feature-a-1"]]) @@ -166,7 +278,10 @@ def _record_launch(_runtime: object, *, route: object, created_worktrees: tuple[ rendered, ) self.assertIn("attach: tmux attach -t omx-feature-session", rendered) - self.assertIn("new session: ENVCTL_USE_REPO_WRAPPER=1 /tmp/repo/bin/envctl --plan feature-a --omx --codex --new-session --headless", rendered) + self.assertIn( + "new session: ENVCTL_USE_REPO_WRAPPER=1 /tmp/repo/bin/envctl --plan feature-a --omx --codex --new-session --headless", + rendered, + ) self.assertIn("kill: tmux kill-session -t omx-feature-session", rendered) def test_explicit_cmux_plan_launch_reuses_selected_existing_worktree(self) -> None: @@ -207,13 +322,17 @@ def _record_launch(_runtime: object, *, route: object, created_worktrees: tuple[ patch.object( engine.planning_worktree_orchestrator, "last_plan_selection_result", - return_value=PlanSelectionResult(raw_projects=[], selected_contexts=[context], created_worktrees=()), + return_value=PlanSelectionResult( + raw_projects=[], selected_contexts=[context], created_worktrees=() + ), ), patch("envctl_engine.startup.lifecycle.launch_plan_agent_terminals", side_effect=_record_launch), patch.object(engine, "_write_artifacts"), patch.object(engine, "_should_enter_post_start_interactive", return_value=False), ): - code = engine.dispatch(parse_route(["--plan", "feature-a", "--cmux", "--headless"], env={"ENVCTL_DEFAULT_MODE": "trees"})) + code = engine.dispatch( + parse_route(["--plan", "feature-a", "--cmux", "--headless"], env={"ENVCTL_DEFAULT_MODE": "trees"}) + ) self.assertEqual(code, 0) self.assertEqual(captured_created_worktrees, [["feature-a-1"]]) @@ -252,7 +371,9 @@ def test_resume_dashboard_exact_headless_plan_prints_attach_command(self) -> Non patch.object( engine.planning_worktree_orchestrator, "last_plan_selection_result", - return_value=PlanSelectionResult(raw_projects=[], selected_contexts=[context], created_worktrees=()), + return_value=PlanSelectionResult( + raw_projects=[], selected_contexts=[context], created_worktrees=() + ), ), patch( "envctl_engine.startup.lifecycle.launch_plan_agent_terminals", @@ -273,7 +394,10 @@ def test_resume_dashboard_exact_headless_plan_prints_attach_command(self) -> Non out = StringIO() with redirect_stdout(out): code = engine.dispatch( - parse_route(["--plan", "feature-a", "--tmux", "--opencode", "--headless"], env={"ENVCTL_DEFAULT_MODE": "trees"}) + parse_route( + ["--plan", "feature-a", "--tmux", "--opencode", "--headless"], + env={"ENVCTL_DEFAULT_MODE": "trees"}, + ) ) self.assertEqual(code, 0) @@ -317,6 +441,7 @@ def test_resume_reuse_failure_falls_back_to_fresh_run_id_before_failure_write(se patch.object(engine, "_discover_projects", return_value=[context]), patch.object(engine, "_try_load_existing_state", return_value=existing_state), patch.object(engine, "_resume", return_value=1), + patch.object(engine, "_terminate_services_from_state", return_value=set()) as terminate_mock, patch.object(engine, "_new_run_id", return_value="run-fresh-after-resume-failure"), patch.object(engine, "_start_project_context", side_effect=RuntimeError("fresh startup failed")), patch.object( @@ -331,10 +456,78 @@ def test_resume_reuse_failure_falls_back_to_fresh_run_id_before_failure_write(se code = engine.dispatch(parse_route(["start", "--batch"], env={"ENVCTL_DEFAULT_MODE": "main"})) self.assertEqual(code, 1) + self.assertEqual(terminate_mock.call_count, 1) written_state = cast(RunState, captured["state"]) self.assertEqual(written_state.run_id, "run-fresh-after-resume-failure") self.assertNotEqual(written_state.run_id, existing_state.run_id) + def test_resume_failure_after_interactive_release_is_terminal_instead_of_racing_fresh_start(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repo(root) + runtime = root / "runtime" + engine = self._engine(repo, runtime) + context = self._main_context(repo) + metadata = startup_support.build_startup_identity_metadata( + engine, + runtime_mode="main", + project_contexts=[context], + ) + existing_state = RunState( + run_id="run-existing", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd=str(repo / "backend"), + pid=1111, + requested_port=8000, + actual_port=8000, + status="running", + ) + }, + metadata=metadata, + ) + competing_results: list[int] = [] + + def released_resume_failure(_route: object) -> int: + self.assertTrue(release_lifecycle_operation(engine)) + competitor = threading.Thread( + target=lambda: competing_results.append( + run_exclusive_lifecycle_operation( + engine, + command="stop", + operation=lambda: 0, + ) + ) + ) + competitor.start() + competitor.join(timeout=2) + self.assertFalse(competitor.is_alive()) + return 1 + + with ( + patch.object(engine, "_discover_projects", return_value=[context]), + patch.object(engine, "_try_load_existing_state", return_value=existing_state), + patch.object(engine, "_resume", side_effect=released_resume_failure), + patch.object( + engine, + "_start_project_context", + side_effect=AssertionError("fresh fallback must not mutate after releasing its lease"), + ) as start_project, + patch.object( + engine, + "_write_artifacts", + side_effect=AssertionError("terminal resume failure must not write fresh state"), + ), + ): + code = engine.dispatch(parse_route(["start", "--batch"], env={"ENVCTL_DEFAULT_MODE": "main"})) + + self.assertEqual(code, 1) + self.assertEqual(competing_results, [0]) + start_project.assert_not_called() + def test_reuse_expand_failure_writes_failed_state_to_fresh_run_id(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -481,4 +674,5 @@ def _start_context(context: object, **_kwargs: object) -> ProjectStartupResult: orphan_mock.assert_not_called() written_state = cast(RunState, captured["state"]) self.assertEqual(set(written_state.services), {"feature-a-1 Backend", "feature-b-1 Backend"}) + self.assertEqual(written_state.metadata["state_source_run_ids"], ["run-existing-expand"]) self.assertEqual(captured["errors"], []) diff --git a/tests/python/startup/test_startup_selection_support.py b/tests/python/startup/test_startup_selection_support.py index 2c2a40cc..d99dcfac 100644 --- a/tests/python/startup/test_startup_selection_support.py +++ b/tests/python/startup/test_startup_selection_support.py @@ -7,6 +7,7 @@ from envctl_engine.runtime.command_router import Route, parse_route from envctl_engine.startup.startup_selection_support import ( + _restart_service_types_for_project, restart_include_requirements, select_start_tree_projects, trees_start_selection_required, @@ -68,7 +69,9 @@ def test_trees_start_selection_required_for_default_or_explicit_trees_start(self self.assertTrue(trees_start_selection_required(route=self._trees_route(raw_args=[]), runtime_mode="trees")) self.assertFalse( trees_start_selection_required( - route=Route(command="plan", mode="trees", raw_args=["--plan"], passthrough_args=[], projects=[], flags={}), + route=Route( + command="plan", mode="trees", raw_args=["--plan"], passthrough_args=[], projects=[], flags={} + ), runtime_mode="trees", ) ) @@ -152,6 +155,22 @@ def test_runtime_scope_flags_control_restart_requirements(self) -> None: self.assertTrue(restart_include_requirements(parse_route(["restart"], env={}))) self.assertFalse(restart_include_requirements(parse_route(["restart", "--service", "Main Frontend"], env={}))) + def test_no_infra_selects_no_additional_application_services(self) -> None: + route = Route( + command="start", + mode="main", + flags={"launch_backend": False, "launch_frontend": False, "launch_dependencies": False}, + ) + + self.assertEqual( + _restart_service_types_for_project( + route=route, + project_name="Main", + default_service_types={"backend", "frontend", "worker"}, + ), + set(), + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/startup/test_support_module_decoupling.py b/tests/python/startup/test_support_module_decoupling.py index f58d1e23..c2caae7d 100644 --- a/tests/python/startup/test_support_module_decoupling.py +++ b/tests/python/startup/test_support_module_decoupling.py @@ -14,6 +14,7 @@ REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" from envctl_engine.startup.resume_restore_support import restore_missing # noqa: E402 +from envctl_engine.startup.resume_restore_project import ResumeProjectRestoreRunner # noqa: E402 import envctl_engine.startup.requirements_project_startup as requirements_project_startup_module # noqa: E402 from envctl_engine.startup.protocols import StartupOrchestratorLike # noqa: E402 from envctl_engine.startup.requirements_execution import ( # noqa: E402 @@ -26,6 +27,7 @@ _maybe_prewarm_docker, _requirements_failure_message, _shared_main_requirements, + ProjectStartupFailure, start_requirements_for_project, start_project_services, start_project_context, @@ -33,7 +35,7 @@ from envctl_engine.startup.startup_orchestrator import StartupOrchestrator # noqa: E402 from envctl_engine.startup.startup_selection_support import _tree_preselected_projects_from_state # noqa: E402 from envctl_engine.config import AppServiceConfig # noqa: E402 -from envctl_engine.runtime.command_router import parse_route # noqa: E402 +from envctl_engine.runtime.command_router import Route, parse_route # noqa: E402 from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord # noqa: E402 @@ -67,6 +69,183 @@ def release(self, port: int) -> None: class StartupSupportModuleDecouplingTests(unittest.TestCase): + def test_resume_stale_service_releases_recorded_foreign_session_lock(self) -> None: + released: list[tuple[int, str, str]] = [] + + class RecordingPortAllocator: + session_id = "current-session" + + def release_owned(self, port: int, owner: str, *, expected_session: str) -> bool: + released.append((port, owner, expected_session)) + return True + + def release(self, _port: int, *, owner: str | None = None) -> None: + raise AssertionError(f"current-session release must not be used: {owner}") + + allocator = RecordingPortAllocator() + service = ServiceRecord( + name="Main Frontend", + type="frontend", + cwd="/tmp/repo/frontend", + actual_port=9000, + port_lock_session="prior-session", + project="Main", + ) + runtime = SimpleNamespace( + port_planner=allocator, + env={}, + config=SimpleNamespace(raw={}), + _terminate_service_record=lambda *_args, **_kwargs: True, + _project_name_from_service=lambda _name: "Main", + ) + runner = ResumeProjectRestoreRunner( + orchestrator=SimpleNamespace(), + rt=runtime, + state=RunState(run_id="run-1", mode="main"), + missing_services=[], + route_for_startup=Route(command="resume", mode="main"), + total_projects=1, + use_project_spinner_group=False, + project_spinner_group=SimpleNamespace(), + use_single_spinner=False, + active_spinner=SimpleNamespace(), + port_allocator=allocator, + ) + + terminated_count = runner._stop_stale_services( + service_names=[service.name], + original_services={service.name: service}, + ) + + self.assertEqual(terminated_count, 1) + self.assertEqual(released, [(9000, "Main:frontend", "prior-session")]) + + def test_post_start_truth_failure_propagates_unterminated_service_records(self) -> None: + requirements = RequirementsResult(project="Main", health="healthy") + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=62001, + actual_port=8000, + ) + runtime = SimpleNamespace( + config=SimpleNamespace( + supabase_auth_users=(), + supabase_auth_user_errors=(), + supabase_auth_users_strict=True, + ), + runtime_root=Path("/tmp/runtime"), + _reserve_project_ports=lambda _context, route=None: None, + _requirements_ready=lambda _value: True, + _consume_project_startup_warnings=lambda _project: [], + _emit=lambda *_args, **_kwargs: None, + _assert_project_services_post_start_truth=lambda **_kwargs: (_ for _ in ()).throw( + RuntimeError("backend truth failed") + ), + _terminate_started_services=lambda _services: None, + ) + orchestrator = SimpleNamespace( + runtime=runtime, + _report_progress=lambda *_args, **_kwargs: None, + start_requirements_for_project=lambda *_args, **_kwargs: requirements, + start_project_services=lambda *_args, **_kwargs: {"Main Backend": service}, + ) + + with self.assertRaises(ProjectStartupFailure) as raised: + start_project_context( + orchestrator, + context=SimpleNamespace(name="Main", root=Path("/repo"), ports={}), + mode="main", + route=parse_route(["start"], env={}), + run_id="run-1", + ) + + self.assertIs(raised.exception.unterminated_services["Main Backend"], service) + self.assertIs(raised.exception.requirements, requirements) + + def test_service_attach_failure_preserves_already_started_requirements(self) -> None: + requirements = RequirementsResult( + project="Main", + health="healthy", + redis={"enabled": True, "success": True, "final": 6379}, + ) + runtime = SimpleNamespace( + config=SimpleNamespace( + supabase_auth_users=(), + supabase_auth_user_errors=(), + supabase_auth_users_strict=True, + ), + runtime_root=Path("/tmp/runtime"), + _reserve_project_ports=lambda _context, route=None: None, + _requirements_ready=lambda _value: True, + _emit=lambda *_args, **_kwargs: None, + ) + orchestrator = SimpleNamespace( + runtime=runtime, + _report_progress=lambda *_args, **_kwargs: None, + start_requirements_for_project=lambda *_args, **_kwargs: requirements, + start_project_services=lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("backend launch failed") + ), + ) + + with self.assertRaises(ProjectStartupFailure) as raised: + start_project_context( + orchestrator, + context=SimpleNamespace(name="Main", root=Path("/repo"), ports={}), + mode="main", + route=parse_route(["start"], env={}), + run_id="run-1", + ) + + self.assertEqual(raised.exception.project, "Main") + self.assertIs(raised.exception.requirements, requirements) + + def test_cleanup_exception_still_transports_all_started_resources(self) -> None: + requirements = RequirementsResult(project="Main", health="healthy") + service = ServiceRecord( + name="Main Backend", + type="backend", + cwd="/repo/backend", + pid=62002, + actual_port=8000, + ) + runtime = SimpleNamespace( + config=SimpleNamespace( + supabase_auth_users=(), + supabase_auth_user_errors=(), + supabase_auth_users_strict=True, + ), + runtime_root=Path("/tmp/runtime"), + _reserve_project_ports=lambda _context, route=None: None, + _requirements_ready=lambda _value: True, + _consume_project_startup_warnings=lambda _project: [], + _emit=lambda *_args, **_kwargs: None, + _assert_project_services_post_start_truth=lambda **_kwargs: (_ for _ in ()).throw( + RuntimeError("truth failed") + ), + _terminate_started_services=lambda _services: (_ for _ in ()).throw(OSError("terminator crashed")), + ) + orchestrator = SimpleNamespace( + runtime=runtime, + _report_progress=lambda *_args, **_kwargs: None, + start_requirements_for_project=lambda *_args, **_kwargs: requirements, + start_project_services=lambda *_args, **_kwargs: {service.name: service}, + ) + + with self.assertRaisesRegex(ProjectStartupFailure, "cleanup error: terminator crashed") as raised: + start_project_context( + orchestrator, + context=SimpleNamespace(name="Main", root=Path("/repo"), ports={}), + mode="main", + route=parse_route(["start"], env={}), + run_id="run-1", + ) + + self.assertIs(raised.exception.requirements, requirements) + self.assertIs(raised.exception.unterminated_services[service.name], service) + def test_startup_execution_support_reexports_new_owner_modules(self) -> None: self.assertIs(start_requirements_for_project, requirements_start_impl) self.assertIs(start_project_services, service_start_impl) @@ -325,7 +504,9 @@ def reconcile(project, req, *, project_root=None): # noqa: ANN001, ANN202 self.assertEqual(project_root, Path("/tmp/repo")) for dependency_id in ("redis", "supabase", "n8n"): component = req.component(dependency_id) - component["runtime_status"] = "healthy" if component.get("final") in {6557, 5610, 5856} else "unreachable" + component["runtime_status"] = ( + "healthy" if component.get("final") in {6557, 5610, 5856} else "unreachable" + ) return [ {"project": project, "component": dependency_id, "status": "unreachable"} for dependency_id in ("redis", "supabase", "n8n") @@ -352,7 +533,7 @@ def reconcile(project, req, *, project_root=None): # noqa: ANN001, ANN202 _reserve_project_ports=lambda context: reserved.append(context.name), _emit=lambda event, **payload: events.append((event, payload)), ) - ) + ), ) with mock.patch( @@ -366,7 +547,9 @@ def reconcile(project, req, *, project_root=None): # noqa: ANN001, ANN202 self.assertEqual(reused.component("n8n")["final"], 5856) self.assertEqual(reserved, ["Main"]) self.assertEqual(start_requirements.call_count, 1) - self.assertIn(("requirements.shared.reuse_stale", {"project": "Main", "source_run_id": "run-stale-main"}), events) + self.assertIn( + ("requirements.shared.reuse_stale", {"project": "Main", "source_run_id": "run-stale-main"}), events + ) def test_start_project_context_rejects_invalid_supabase_auth_user_config_before_services(self) -> None: order: list[str] = [] @@ -403,7 +586,7 @@ def test_start_project_context_rejects_invalid_supabase_auth_user_config_before_ start_requirements_for_project=lambda context, mode, route: requirements, ) - with self.assertRaisesRegex(RuntimeError, "Invalid Supabase Auth user config"): + with self.assertRaisesRegex(ProjectStartupFailure, "Invalid Supabase Auth user config") as raised: start_project_context( orchestrator, context=SimpleNamespace(name="Main", root=Path("/tmp/repo"), ports={}), @@ -413,6 +596,7 @@ def test_start_project_context_rejects_invalid_supabase_auth_user_config_before_ ) self.assertNotIn("services", order) + self.assertIs(raised.exception.requirements, requirements) def test_start_project_context_syncs_supabase_auth_users_before_services(self) -> None: order: list[str] = [] @@ -444,8 +628,10 @@ def project_env(**kwargs): # noqa: ANN003, ANN201 _record_project_startup_warning=lambda project, warning: None, _consume_project_startup_warnings=lambda project: [], _emit=lambda event, **payload: ( - order.append(event), events.append((event, payload)) - ) if event.startswith("supabase.auth_users") else None, + (order.append(event), events.append((event, payload))) + if event.startswith("supabase.auth_users") + else None + ), _assert_project_services_post_start_truth=lambda **kwargs: None, _terminate_started_services=lambda services: None, ) @@ -685,7 +871,9 @@ def start_project_services(_context, *, requirements, run_id, route): # noqa: A _terminate_service_record=lambda service, **kwargs: terminated_services.append(service.name) or True, _service_port=lambda service: service.actual_port, _release_requirement_ports=lambda req: self.fail("healthy reused requirements should not be released"), - _reserve_project_ports=lambda ctx: self.fail("partial service restore should reserve only selected services"), + _reserve_project_ports=lambda ctx: self.fail( + "partial service restore should reserve only selected services" + ), _start_requirements_for_project=lambda *args, **kwargs: self.fail("healthy requirements should be reused"), _start_project_services=start_project_services, ) @@ -884,7 +1072,9 @@ def start_project_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: _project_service_env=lambda context, requirements, route=None: {}, _resolve_backend_env_file=lambda context, backend_cwd: (None, False), _resolve_frontend_env_file=lambda context, frontend_cwd: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service: True, process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: None), _prepare_backend_runtime=prepare_backend_runtime, @@ -963,7 +1153,9 @@ def test_start_project_services_skips_default_backend_frontend_when_no_local_sys _project_service_env=lambda context, requirements, route=None, service_name=None: {}, _resolve_backend_env_file=lambda context, backend_cwd: (None, False), _resolve_frontend_env_file=lambda context, frontend_cwd: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service: True, process_runner=SimpleNamespace( start_background=lambda *args, **kwargs: background_calls.append((args, kwargs)) @@ -1039,7 +1231,9 @@ def test_start_project_services_keeps_missing_command_failure_for_explicit_enabl _project_service_env=lambda context, requirements, route=None, service_name=None: {}, _resolve_backend_env_file=lambda context, backend_cwd: (None, False), _resolve_frontend_env_file=lambda context, frontend_cwd: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service: True, process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: None), _prepare_backend_runtime=lambda **kwargs: None, @@ -1101,7 +1295,9 @@ def test_start_project_services_keeps_missing_command_failure_for_explicit_addit _project_service_env=lambda context, requirements, route=None, service_name=None: {}, _resolve_backend_env_file=lambda context, backend_cwd: (None, False), _resolve_frontend_env_file=lambda context, frontend_cwd: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service: True, process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: None), _prepare_backend_runtime=lambda **kwargs: None, @@ -1182,12 +1378,16 @@ def test_start_project_services_keeps_autodetectable_default_services_startable( _project_service_env=lambda context, requirements, route=None, service_name=None: {}, _resolve_backend_env_file=lambda context, backend_cwd: (None, False), _resolve_frontend_env_file=lambda context, frontend_cwd: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service: True, process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: None), _prepare_backend_runtime=lambda **kwargs: None, _prepare_frontend_runtime=lambda **kwargs: None, - _service_command_source=lambda service_name, **kwargs: source_calls.append(service_name) or "autodetected", + _service_command_source=lambda service_name, **kwargs: ( + source_calls.append(service_name) or "autodetected" + ), _service_start_command_resolved=lambda service_name, **kwargs: (["npm", "run", "dev"], "autodetected"), _command_exists=lambda command: command in {"npm", "npx"}, _detect_service_actual_port=lambda pid, requested_port, service_name, log_path: requested_port, @@ -1332,15 +1532,17 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: _resolve_frontend_env_file=lambda context, frontend_cwd: None, _prepare_backend_runtime=lambda **kwargs: None, _prepare_frontend_runtime=lambda **kwargs: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=service_enabled_for_mode, _service_command_source=lambda **kwargs: "configured", _service_start_command_resolved=lambda service_name, **kwargs: ("python -c pass", "configured"), _split_command=lambda command, **kwargs: ["python", "-c", "pass"], process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: SimpleNamespace(pid=1234)), - _detect_service_actual_port=lambda service_name, pid, requested_port, **kwargs: 8022 - if service_name == "voice-runtime" - else requested_port, + _detect_service_actual_port=lambda service_name, pid, requested_port, **kwargs: ( + 8022 if service_name == "voice-runtime" else requested_port + ), _listener_truth_enforced=lambda: True, _service_listener_failure_detail=lambda **kwargs: "", _conflict_remaining={}, @@ -1348,7 +1550,9 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: ) orchestrator = SimpleNamespace( runtime=runtime, - _restart_service_types_for_project=lambda route, project_name, default_service_types: set(default_service_types), + _restart_service_types_for_project=lambda route, project_name, default_service_types: set( + default_service_types + ), _suppress_timing_output=lambda route: True, ) context = SimpleNamespace( @@ -1444,13 +1648,18 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: _resolve_frontend_env_file=lambda context, frontend_cwd: None, _prepare_backend_runtime=lambda **kwargs: None, _prepare_frontend_runtime=lambda **kwargs: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service_name: service_name == "voice-runtime", _service_command_source=lambda **kwargs: "configured", - _split_command=lambda command, **kwargs: split_calls.append({"command": command, **kwargs}) or [ - "scripts/envctl/start-voice-runtime.sh", - str(kwargs["port"]), - ], + _split_command=lambda command, **kwargs: ( + split_calls.append({"command": command, **kwargs}) + or [ + "scripts/envctl/start-voice-runtime.sh", + str(kwargs["port"]), + ] + ), process_runner=SimpleNamespace(start_background=lambda *args, **kwargs: SimpleNamespace(pid=1234)), _detect_service_actual_port=lambda **kwargs: 8010, _listener_truth_enforced=lambda: True, @@ -1556,7 +1765,9 @@ def start_services_with_attach(**kwargs: object) -> dict[str, ServiceRecord]: _resolve_frontend_env_file=lambda context, frontend_cwd: None, _prepare_backend_runtime=lambda **kwargs: None, _prepare_frontend_runtime=lambda **kwargs: None, - _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: dict(base_env), + _service_env_from_file=lambda base_env, env_file, include_app_env_file, env_file_authoritative=False: ( + dict(base_env) + ), _service_enabled_for_mode=lambda mode, service_name: service_name == "voice-runtime", _service_command_source=lambda **kwargs: "configured", _split_command=lambda command, **kwargs: ["python", "-c", "pass"], diff --git a/tests/python/state/test_fingerprints.py b/tests/python/state/test_fingerprints.py new file mode 100644 index 00000000..e210b7af --- /dev/null +++ b/tests/python/state/test_fingerprints.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from pathlib import Path + +from envctl_engine.runtime.state_fingerprint_support import ( + state_fingerprint as runtime_state_fingerprint, +) +from envctl_engine.state.fingerprints import file_fingerprint, state_fingerprint, text_fingerprint +from envctl_engine.state.models import RunState + + +def test_shared_fingerprint_helpers_agree_for_text_file_and_runtime_facade(tmp_path: Path) -> None: + payload = "deterministic payload\n" + path = tmp_path / "payload.txt" + path.write_text(payload, encoding="utf-8") + + assert file_fingerprint(path) == text_fingerprint(payload) + + state = RunState(run_id="run-1", mode="trees", metadata={"project_names": ["alpha"]}) + assert runtime_state_fingerprint(state) == state_fingerprint(state) + + +def test_state_fingerprint_changes_with_serialized_state() -> None: + first = RunState(run_id="run-1", mode="main") + second = RunState(run_id="run-1", mode="main", metadata={"changed": True}) + + assert state_fingerprint(first) != state_fingerprint(second) diff --git a/tests/python/state/test_lookup.py b/tests/python/state/test_lookup.py new file mode 100644 index 00000000..5e1fc5be --- /dev/null +++ b/tests/python/state/test_lookup.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import unittest + +from envctl_engine.state.lookup import call_state_loader + + +class StateLookupTests(unittest.TestCase): + def test_passes_all_supported_selection_arguments(self) -> None: + captured: dict[str, object] = {} + + def loader( + *, + mode: str | None, + strict_mode_match: bool, + project_names: tuple[str, ...] | None, + ) -> str: + captured.update( + mode=mode, + strict_mode_match=strict_mode_match, + project_names=project_names, + ) + return "selected" + + result = call_state_loader( + loader, + mode="trees", + strict_mode_match=True, + project_names=("feature-a",), + ) + + self.assertEqual(result, "selected") + self.assertEqual( + captured, + { + "mode": "trees", + "strict_mode_match": True, + "project_names": ("feature-a",), + }, + ) + + def test_omits_unsupported_compatibility_keywords_for_untargeted_lookup(self) -> None: + calls = 0 + + def legacy_loader() -> str: + nonlocal calls + calls += 1 + return "legacy" + + self.assertEqual(call_state_loader(legacy_loader, mode="main"), "legacy") + self.assertEqual(calls, 1) + + def test_targeted_lookup_fails_closed_when_loader_cannot_select_projects(self) -> None: + called = False + + def legacy_loader(*, mode: str | None = None) -> str: + nonlocal called + called = True + return str(mode) + + with self.assertRaisesRegex(TypeError, "accepts project_names"): + call_state_loader(legacy_loader, mode="trees", project_names=("feature-a",)) + self.assertFalse(called) + + def test_var_keyword_loader_receives_selection_arguments(self) -> None: + captured: dict[str, object] = {} + + def flexible_loader(**kwargs: object) -> dict[str, object]: + captured.update(kwargs) + return captured + + self.assertIs( + call_state_loader( + flexible_loader, + mode="trees", + strict_mode_match=True, + project_names=("feature-a",), + ), + captured, + ) + self.assertEqual(captured["project_names"], ("feature-a",)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/state/test_persistence.py b/tests/python/state/test_persistence.py new file mode 100644 index 00000000..2844c6bc --- /dev/null +++ b/tests/python/state/test_persistence.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import errno +import os +import tempfile +import threading +import unittest +from pathlib import Path +from unittest.mock import patch + +from envctl_engine.state.persistence import ( + advisory_file_lock, + atomic_write_text, + durable_mkdir, + require_path_component, + scavenge_atomic_write_temps, +) + + +class StatePersistenceTests(unittest.TestCase): + def test_failed_atomic_replaces_preserve_old_value_and_leave_no_temp_files_or_descriptors(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + target = root / "state.json" + target.write_text("old", encoding="utf-8") + descriptor_root = Path("/dev/fd") + descriptors_before = len(list(descriptor_root.iterdir())) if descriptor_root.is_dir() else None + + with patch("envctl_engine.state.persistence.os.replace", side_effect=OSError("forced failure")): + for _ in range(25): + with self.assertRaisesRegex(OSError, "forced failure"): + atomic_write_text(target, "new") + + descriptors_after = len(list(descriptor_root.iterdir())) if descriptor_root.is_dir() else None + self.assertEqual(target.read_text(encoding="utf-8"), "old") + self.assertEqual(list(root.glob(".*.tmp")), []) + if descriptors_before is not None: + self.assertEqual(descriptors_after, descriptors_before) + + def test_target_local_scavenge_removes_only_exact_regular_atomic_temps(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + target = root / "state.json" + stale = root / ".state.json.abcdefgh.tmp" + unrelated_target = root / ".other.json.abcdefgh.tmp" + malformed_token = root / ".state.json.not-atomic.tmp" + generic_tmp = root / "notes.tmp" + external = root / "external" + symlink_temp = root / ".state.json.ijklmnop.tmp" + for path in (stale, unrelated_target, malformed_token, generic_tmp): + path.write_text(path.name, encoding="utf-8") + external.write_text("sentinel", encoding="utf-8") + symlink_temp.symlink_to(external) + + removed = scavenge_atomic_write_temps(target) + + self.assertEqual(removed, 1) + self.assertFalse(stale.exists()) + self.assertTrue(unrelated_target.is_file()) + self.assertTrue(malformed_token.is_file()) + self.assertTrue(generic_tmp.is_file()) + self.assertTrue(symlink_temp.is_symlink()) + self.assertEqual(external.read_text(encoding="utf-8"), "sentinel") + + def test_target_local_scavenge_rejects_symlink_parent_without_touching_target(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + external = root / "external" + external.mkdir() + stale = external / ".state.json.abcdefgh.tmp" + stale.write_text("sentinel", encoding="utf-8") + linked_parent = root / "linked-parent" + linked_parent.symlink_to(external, target_is_directory=True) + + with self.assertRaises(OSError): + scavenge_atomic_write_temps(linked_parent / "state.json") + + self.assertEqual(stale.read_text(encoding="utf-8"), "sentinel") + + def test_path_component_validation_accepts_safe_names_and_rejects_traversal(self) -> None: + self.assertEqual(require_path_component("run-2026_07.11", label="run_id"), "run-2026_07.11") + for value in ("", " ", ".", "..", "../run", "nested/run", "nested\\run", "/tmp/run", "bad\0id"): + with self.subTest(value=value): + with self.assertRaisesRegex(ValueError, "path component"): + require_path_component(value, label="run_id") + + def test_lock_descriptor_is_closed_if_stream_construction_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "state.lock" + opened_descriptors: list[int] = [] + closed_descriptors: list[int] = [] + real_open = os.open + real_close = os.close + + def tracked_open(*args: object, **kwargs: object) -> int: + descriptor = real_open(*args, **kwargs) # type: ignore[arg-type] + opened_descriptors.append(descriptor) + return descriptor + + def tracked_close(descriptor: int) -> None: + closed_descriptors.append(descriptor) + real_close(descriptor) + + with ( + patch("envctl_engine.state.persistence.os.open", side_effect=tracked_open), + patch("envctl_engine.state.persistence.os.fdopen", side_effect=OSError("fdopen failed")), + patch("envctl_engine.state.persistence.os.close", side_effect=tracked_close), + ): + with self.assertRaisesRegex(OSError, "fdopen failed"): + with advisory_file_lock(lock_path, exclusive=True): + self.fail("lock acquisition unexpectedly succeeded") + + self.assertEqual(len(opened_descriptors), 1) + self.assertEqual(closed_descriptors, opened_descriptors) + + def test_lock_rejects_symlink_without_touching_its_target(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + target = root / "external-target" + target.write_text("sentinel", encoding="utf-8") + lock_path = root / "state.lock" + lock_path.symlink_to(target) + + with self.assertRaises(OSError): + with advisory_file_lock(lock_path, exclusive=True): + self.fail("symlink lock unexpectedly succeeded") + + self.assertEqual(target.read_text(encoding="utf-8"), "sentinel") + + def test_lock_timeout_does_not_leak_or_enter_critical_section(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + lock_path = Path(tmpdir) / "state.lock" + holder_entered = threading.Event() + release_holder = threading.Event() + + def hold_lock() -> None: + with advisory_file_lock(lock_path, exclusive=True): + holder_entered.set() + release_holder.wait(timeout=2) + + holder = threading.Thread(target=hold_lock) + holder.start() + self.assertTrue(holder_entered.wait(timeout=1)) + try: + with self.assertRaisesRegex(TimeoutError, "Timed out acquiring lock"): + with advisory_file_lock(lock_path, exclusive=True, timeout_seconds=0.02): + self.fail("contended lock unexpectedly succeeded") + finally: + release_holder.set() + holder.join(timeout=1) + self.assertFalse(holder.is_alive()) + + def test_unsupported_directory_fsync_does_not_undo_committed_replace(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + target = Path(tmpdir) / "state.json" + with patch( + "envctl_engine.state.persistence.os.fsync", + side_effect=[None, OSError(errno.EINVAL, "directory fsync unsupported")], + ): + atomic_write_text(target, "committed") + + self.assertEqual(target.read_text(encoding="utf-8"), "committed") + + def test_durable_mkdir_fsyncs_each_new_directory_and_its_parent(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + target = root / "runs" / "run-a" / "revisions" + synced: list[Path] = [] + + with patch( + "envctl_engine.state.persistence.fsync_directory", + side_effect=lambda path: synced.append(path), + ): + durable_mkdir(target) + + self.assertTrue(target.is_dir()) + for directory in (root / "runs", root / "runs" / "run-a", target): + self.assertIn(directory, synced) + self.assertIn(directory.parent, synced) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/state/test_project_runtime_resolution.py b/tests/python/state/test_project_runtime_resolution.py index 0c6cc543..1502a9e0 100644 --- a/tests/python/state/test_project_runtime_resolution.py +++ b/tests/python/state/test_project_runtime_resolution.py @@ -8,6 +8,7 @@ active_project_names, cwd_runtime_warnings, dependency_mode_summary, + filter_project_scoped_metadata, infer_cwd_project, requested_project_not_running_payload, resolve_requested_project_state, @@ -55,6 +56,80 @@ def test_exact_requested_project_match_filters_state_to_canonical_project(self) self.assertEqual(set(resolution.state.requirements), {"Alpha"}) self.assertEqual(resolution.state.metadata["project_roots"], {"Alpha": "/tmp/a"}) + def test_project_selection_filters_every_project_scoped_metadata_surface(self) -> None: + state = RunState( + run_id="run-metadata", + mode="trees", + services={ + "Alpha Backend": ServiceRecord(name="Alpha Backend", type="backend", cwd="/tmp/a", project="Alpha"), + "Beta Voice": ServiceRecord(name="Beta Voice", type="voice", cwd="/tmp/b", project="Beta"), + }, + metadata={ + "project_roots": {"Alpha": "/tmp/a", "Beta": "/tmp/b"}, + "project_pr_links": {"Alpha": "pr-a", "Beta": "pr-b"}, + "project_test_summaries": {"Alpha": {"failed": 0}, "Beta": {"failed": 1}}, + "project_test_results_root": "/tmp/results", + "project_test_results_updated_at": "now", + "project_action_reports": {"Alpha": {"test": {}}, "Beta": {"test": {}}}, + "dashboard_project_configured_services": { + "Alpha": ["backend"], + "Beta": ["voice"], + }, + "dashboard_configured_service_types": ["backend", "voice"], + "dashboard_stopped_services": [ + {"name": "Alpha Backend", "project": "Alpha", "type": "backend"}, + {"name": "Beta Voice", "project": "Beta", "type": "voice"}, + {"project": "Alpha", "type": "frontend"}, + ], + "startup_identity": { + "mode": "trees", + "projects": [ + {"name": "Alpha", "root": "/tmp/a"}, + {"name": "Beta", "root": "/tmp/b"}, + ], + "fingerprint": "stale", + }, + "unrelated": "preserved", + }, + ) + + resolution = resolve_requested_project_state(state, ["Alpha"], command="health") + + self.assertTrue(resolution.ok) + assert resolution.state is not None + metadata = resolution.state.metadata + for key in ( + "project_roots", + "project_pr_links", + "project_test_summaries", + "project_action_reports", + "dashboard_project_configured_services", + ): + self.assertEqual(set(metadata[key]), {"Alpha"}) + self.assertEqual( + metadata["dashboard_stopped_services"], + [{"name": "Alpha Backend", "project": "Alpha", "type": "backend"}], + ) + self.assertEqual(metadata["dashboard_configured_service_types"], ["backend"]) + self.assertEqual(metadata["startup_identity"]["projects"], [{"name": "Alpha", "root": "/tmp/a"}]) + self.assertNotEqual(metadata["startup_identity"]["fingerprint"], "stale") + self.assertNotIn("project_test_results_root", metadata) + self.assertNotIn("project_test_results_updated_at", metadata) + self.assertEqual(metadata["unrelated"], "preserved") + + def test_filter_removes_test_companion_fields_when_no_selected_summary_remains(self) -> None: + metadata = filter_project_scoped_metadata( + { + "project_test_summaries": {"Beta": {"failed": 1}}, + "project_test_results_root": "/tmp/results", + "project_test_results_updated_at": "now", + }, + {"Alpha"}, + ) + + self.assertNotIn("project_test_summaries", metadata) + self.assertNotIn("project_test_results_root", metadata) + self.assertNotIn("project_test_results_updated_at", metadata) def test_case_mismatched_project_selector_fails_closed(self) -> None: state = RunState( @@ -148,14 +223,18 @@ def test_dependency_mode_summary_prefers_explicit_metadata_and_derives_legacy_sh mode="trees", metadata={"dependency_mode": "isolated", "shared_dependencies": False}, ) - self.assertEqual(dependency_mode_summary(explicit), {"dependency_mode": "isolated", "shared_dependencies": False}) + self.assertEqual( + dependency_mode_summary(explicit), {"dependency_mode": "isolated", "shared_dependencies": False} + ) legacy_shared = RunState( run_id="run-legacy", mode="trees", metadata={"dashboard_dependency_scope": "shared"}, ) - self.assertEqual(dependency_mode_summary(legacy_shared), {"dependency_mode": "shared", "shared_dependencies": True}) + self.assertEqual( + dependency_mode_summary(legacy_shared), {"dependency_mode": "shared", "shared_dependencies": True} + ) unknown = RunState(run_id="run-unknown", mode="trees") self.assertEqual(dependency_mode_summary(unknown), {"dependency_mode": "unknown", "shared_dependencies": None}) diff --git a/tests/python/state/test_run_index.py b/tests/python/state/test_run_index.py new file mode 100644 index 00000000..db8a6147 --- /dev/null +++ b/tests/python/state/test_run_index.py @@ -0,0 +1,437 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import patch + +from envctl_engine.state.persistence import atomic_write_text +from envctl_engine.state.run_index import RunIndex, RunIndexCandidate, StateSelector + + +class RunIndexTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.runtime_dir = Path(self.temporary_directory.name) / "runtime" + self.runtime_root = self.runtime_dir / "repo-scope" + self.index = RunIndex( + runtime_root=self.runtime_root, + runtime_dir=self.runtime_dir, + runtime_scope_id="repo-scope", + ) + self.revision_number = 0 + + def _state_path(self, run_id: str) -> Path: + self.revision_number += 1 + path = self.runtime_root / "runs" / run_id / "revisions" / f"revision-{self.revision_number}" / "run_state.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{}\n", encoding="utf-8") + return path + + def _record(self, run_id: str, *, mode: str = "trees", projects: list[str]) -> Path: + path = self._state_path(run_id) + self.index.record( + state_path=path, + run_id=run_id, + mode=mode, + project_names=projects, + ) + return path.resolve() + + def test_new_records_replace_overlapping_project_owners_without_crossing_modes(self) -> None: + _ = self._record("exact-old", projects=["Feature-A"]) + containing = self._record("containing", projects=["feature-a", "feature-b"]) + partial = self._record("partial-overlap", projects=["feature-c", "feature-d"]) + exact_new = self._record("exact-new", projects=["FEATURE-A"]) + _ = self._record("disjoint", projects=["feature-z"]) + wrong_mode = self._record("wrong-mode", mode="main", projects=["feature-a"]) + + self.assertEqual( + self.index.candidate_paths(StateSelector(mode="TREES", project_names=("feature-a",))), + [exact_new], + ) + self.assertEqual( + self.index.candidate_paths(StateSelector(mode="trees", project_names=("feature-b",))), + [containing], + ) + self.assertEqual( + self.index.candidate_paths(StateSelector(mode="main", project_names=("feature-a",))), + [wrong_mode], + ) + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-a", "feature-c"))), + [exact_new, partial], + ) + + def test_record_heals_multiple_historical_owners_for_the_same_project(self) -> None: + run_a = self._state_path("run-a") + run_b = self._state_path("run-b") + self.index.replace_all( + [ + RunIndexCandidate(run_a, "run-a", "main", ("main",), 1), + RunIndexCandidate(run_b, "run-b", "main", ("main",), 2), + ] + ) + + run_c = self._record("run-c", mode="main", projects=["Main"]) + + self.assertEqual( + self.index.candidate_paths(StateSelector("main", ("main",))), + [run_c], + ) + payload = json.loads(self.index.index_path.read_text(encoding="utf-8")) + self.assertEqual(payload["retired_run_ids"], ["run-a", "run-b"]) + + def test_record_trims_mixed_historical_owner_without_losing_unrelated_project(self) -> None: + run_a = self._state_path("run-a") + run_b = self._state_path("run-b") + self.index.replace_all( + [ + RunIndexCandidate(run_a, "run-a", "trees", ("feature-x", "feature-y"), 1), + RunIndexCandidate(run_b, "run-b", "trees", ("feature-x",), 2), + ] + ) + + run_c = self._record("run-c", projects=["feature-x"]) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-x",))), + [run_c], + ) + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-y",))), + [run_a.resolve()], + ) + payload = json.loads(self.index.index_path.read_text(encoding="utf-8")) + entries = {entry["run_id"]: entry["project_names"] for entry in payload["entries"]} + self.assertEqual(entries, {"run-a": ["feature-y"], "run-c": ["feature-x"]}) + self.assertEqual(payload["retired_run_ids"], ["run-b"]) + + def test_preserves_independent_entries_during_concurrent_records(self) -> None: + run_ids = [f"run-{index:02d}" for index in range(32)] + state_paths = {run_id: self._state_path(run_id).resolve() for run_id in run_ids} + + def record(run_id: str) -> None: + self.index.record( + state_path=state_paths[run_id], + run_id=run_id, + mode="trees", + project_names=[run_id], + ) + + with ThreadPoolExecutor(max_workers=12) as executor: + list(executor.map(record, run_ids)) + + candidates = self.index.candidate_paths(StateSelector(mode="trees", project_names=())) + self.assertEqual(len(candidates), len(run_ids)) + self.assertEqual(set(candidates), set(state_paths.values())) + payload = json.loads((self.runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(len(payload["entries"]), len(run_ids)) + + def test_corrupt_index_fails_closed_and_later_record_replaces_it(self) -> None: + self.runtime_root.mkdir(parents=True, exist_ok=True) + index_path = self.runtime_root / "run_index.json" + index_path.write_text('{"entries":', encoding="utf-8") + + self.assertEqual(self.index.candidate_paths(StateSelector(mode=None, project_names=())), []) + + recovered = self._record("recovered", projects=["feature-a"]) + self.assertEqual( + self.index.candidate_paths(StateSelector(mode="trees", project_names=("feature-a",))), + [recovered], + ) + payload = json.loads(index_path.read_text(encoding="utf-8")) + self.assertEqual(payload["version"], 4) + self.assertGreaterEqual(payload["generation"], 1) + + def test_clean_registry_write_scavenges_only_its_crash_temp_targets(self) -> None: + self.runtime_root.mkdir(parents=True, exist_ok=True) + stale_primary = self.runtime_root / ".run_index.json.abcdefgh.tmp" + stale_backup = self.runtime_root / ".run_index.backup.json.ijklmnop.tmp" + unrelated = self.runtime_root / ".run_state.json.abcdefgh.tmp" + for path in (stale_primary, stale_backup, unrelated): + path.write_text("crash debris", encoding="utf-8") + + self.index.initialize_empty() + + self.assertFalse(stale_primary.exists()) + self.assertFalse(stale_backup.exists()) + self.assertTrue(unrelated.is_file()) + + def test_rejects_record_paths_outside_runtime_dir(self) -> None: + outside = Path(self.temporary_directory.name) / "runtime-sibling" / "run_state.json" + outside.parent.mkdir(parents=True, exist_ok=True) + outside.write_text("{}\n", encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "escapes runtime_root"): + self.index.record( + state_path=outside, + run_id="outside", + mode="trees", + project_names=["feature-a"], + ) + + def test_rejects_state_path_from_another_runtime_scope(self) -> None: + foreign = self.runtime_dir / "another-scope" / "runs" / "foreign" / "run_state.json" + foreign.parent.mkdir(parents=True, exist_ok=True) + foreign.write_text("{}\n", encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "escapes runtime_root"): + self.index.record( + state_path=foreign, + run_id="foreign", + mode="trees", + project_names=["feature-a"], + ) + + def test_rejects_symlinked_or_noncanonical_revision_paths(self) -> None: + real_path = self._state_path("safe") + alias_path = real_path.parent / "alias.json" + alias_path.symlink_to(real_path) + + for path in (alias_path, real_path.parent / ".." / real_path.parent.name / real_path.name): + with self.subTest(path=path): + with self.assertRaisesRegex(ValueError, "canonical"): + self.index.record( + state_path=path, + run_id="safe", + mode="trees", + project_names=["feature-a"], + ) + + def test_runtime_root_retarget_after_construction_is_rejected_without_external_writes(self) -> None: + external = Path(self.temporary_directory.name) / "external" + external.mkdir() + sentinel = external / "sentinel.txt" + sentinel.write_text("untouched", encoding="utf-8") + self.runtime_root.parent.mkdir(parents=True, exist_ok=True) + self.runtime_root.symlink_to(external, target_is_directory=True) + + with self.assertRaisesRegex(RuntimeError, "symlink"): + self.index.candidates(StateSelector(mode=None, project_names=())) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "untouched") + self.assertEqual(list(external.iterdir()), [sentinel]) + + def test_traversal_entry_fails_closed_and_wrong_scope_is_not_loaded(self) -> None: + safe = self._record("safe", projects=["feature-a"]) + index_path = self.runtime_root / "run_index.json" + payload = json.loads(index_path.read_text(encoding="utf-8")) + payload["entries"].append( + { + "state_path": str(self.runtime_root / ".." / ".." / "escaped.json"), + "run_id": "escaped", + "mode": "trees", + "project_names": ["feature-a"], + "sequence": 2, + "runtime_scope_id": "repo-scope", + } + ) + index_path.write_text(json.dumps(payload), encoding="utf-8") + self.assertEqual(self.index.candidate_paths(StateSelector("trees", ("feature-a",))), [safe]) + + payload["entries"] = [entry for entry in payload["entries"] if entry["run_id"] == "safe"] + payload["runtime_scope_id"] = "another-scope" + index_path.write_text(json.dumps(payload), encoding="utf-8") + (self.runtime_root / "run_index.backup.json").write_text(json.dumps(payload), encoding="utf-8") + self.assertEqual(self.index.candidate_paths(StateSelector("trees", ("feature-a",))), []) + self.assertTrue(safe.is_file()) + + def test_purge_removes_index_without_preventing_future_records(self) -> None: + before = self._record("before-purge", projects=["feature-a"]) + self.index.purge() + payload = json.loads((self.runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(payload["entries"], []) + self.assertEqual(payload["retired_run_ids"], ["before-purge"]) + self.assertEqual(self.index.candidate_paths(StateSelector(None, ())), []) + + with self.assertRaisesRegex(RuntimeError, "retired"): + self.index.record( + state_path=before, + run_id="before-purge", + mode="trees", + project_names=["feature-a"], + ) + + after = self._record("after-purge", projects=["feature-b"]) + self.assertEqual(self.index.candidate_paths(StateSelector("trees", ("feature-b",))), [after]) + + def test_empty_update_preserves_existing_project_ownership(self) -> None: + state_path = self._record("run-a", projects=["feature-a"]) + + self.index.record( + state_path=state_path, + run_id="run-a", + mode="trees", + project_names=[], + ) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-a",))), + [state_path], + ) + + def test_rejects_invalid_run_id_components_before_indexing(self) -> None: + for run_id in (".", "..", "../other", "nested/run", "nested\\run", "/absolute"): + with self.subTest(run_id=run_id): + with self.assertRaisesRegex(ValueError, "path component"): + self.index.record( + state_path=self.runtime_root / "runs" / "safe" / "run_state.json", + run_id=run_id, + mode="trees", + project_names=["feature-a"], + ) + + def test_remove_deactivates_only_requested_run(self) -> None: + run_a = self._record("run-a", projects=["feature-a"]) + run_b = self._record("run-b", projects=["feature-b"]) + + self.assertTrue(self.index.remove("run-a")) + self.assertFalse(self.index.remove("run-a")) + self.assertEqual(self.index.candidate_paths(StateSelector("trees", ("feature-a",))), []) + self.assertEqual(self.index.candidate_paths(StateSelector("trees", ("feature-b",))), [run_b]) + self.assertTrue(run_a.is_file()) + + def test_superseded_run_is_durably_fenced_from_reclaiming_ownership(self) -> None: + run_a = self._record("run-a", projects=["feature-a"]) + run_b = self._state_path("run-b") + self.index.record( + state_path=run_b, + run_id="run-b", + mode="trees", + project_names=["feature-a"], + supersede_run_ids=["run-a"], + ) + + with self.assertRaisesRegex(RuntimeError, "retired"): + self.index.record( + state_path=run_a, + run_id="run-a", + mode="trees", + project_names=["feature-a"], + ) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-a",))), + [run_b.resolve()], + ) + + def test_removing_untracked_source_id_still_fences_late_writer(self) -> None: + self.index.initialize_empty() + self.assertFalse(self.index.remove("late-source")) + + with self.assertRaisesRegex(RuntimeError, "retired"): + self.index.record( + state_path=self._state_path("late-source"), + run_id="late-source", + mode="trees", + project_names=["feature-a"], + ) + + def test_single_copy_failure_keeps_newest_generation_readable_and_repairable(self) -> None: + run_a = self._record("run-a", projects=["feature-a"]) + run_b = self._state_path("run-b") + + def write_backup_only(path: Path, text: str) -> None: + if path == self.index.index_path: + raise OSError("primary unavailable") + atomic_write_text(path, text) + + with patch("envctl_engine.state.run_index.atomic_write_text", side_effect=write_backup_only): + self.index.record( + state_path=run_b, + run_id="run-b", + mode="trees", + project_names=["feature-b"], + ) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ())), + [run_b.resolve(), run_a], + ) + primary_before = json.loads(self.index.index_path.read_text(encoding="utf-8")) + backup_before = json.loads(self.index.backup_path.read_text(encoding="utf-8")) + self.assertGreater(backup_before["generation"], primary_before["generation"]) + + self.index.repair_copies() + + primary_after = json.loads(self.index.index_path.read_text(encoding="utf-8")) + backup_after = json.loads(self.index.backup_path.read_text(encoding="utf-8")) + self.assertEqual(primary_after, backup_after) + self.assertEqual( + {entry["run_id"] for entry in primary_after["entries"]}, + {"run-a", "run-b"}, + ) + + def test_equal_generation_divergence_fails_closed_without_overwriting_either_copy(self) -> None: + self._record("run-a", projects=["feature-a"]) + indexed_entry = self.index.candidates(StateSelector("trees", ()))[0] + primary_before = self.index.index_path.read_text(encoding="utf-8") + backup_payload = json.loads(self.index.backup_path.read_text(encoding="utf-8")) + backup_payload["entries"][0]["project_names"] = ["feature-b"] + backup_before = json.dumps(backup_payload, indent=2, sort_keys=True) + "\n" + self.index.backup_path.write_text(backup_before, encoding="utf-8") + + with self.assertRaisesRegex(RuntimeError, "diverged at generation"): + self.index.candidate_paths(StateSelector("trees", ())) + with self.assertRaisesRegex(RuntimeError, "diverged at generation"): + self.index.repair_copies() + with self.assertRaisesRegex(RuntimeError, "diverged at generation"): + self.index.initialize_empty() + with self.assertRaisesRegex(RuntimeError, "diverged at generation"): + self.index.replace_all([indexed_entry]) + + self.assertEqual(self.index.index_path.read_text(encoding="utf-8"), primary_before) + self.assertEqual(self.index.backup_path.read_text(encoding="utf-8"), backup_before) + + def test_total_registry_write_failure_preserves_last_committed_generation(self) -> None: + run_a = self._record("run-a", projects=["feature-a"]) + run_b = self._state_path("run-b") + + with patch( + "envctl_engine.state.run_index.atomic_write_text", + side_effect=OSError("storage unavailable"), + ): + with self.assertRaisesRegex(OSError, "storage unavailable"): + self.index.record( + state_path=run_b, + run_id="run-b", + mode="trees", + project_names=["feature-b"], + ) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ())), + [run_a], + ) + + def test_visible_registry_replace_is_commit_even_if_post_replace_fsync_reports_error(self) -> None: + state_path = self._state_path("run-visible") + + def replace_then_report_error(path: Path, text: str) -> None: + atomic_write_text(path, text) + raise OSError("directory fsync result unknown") + + with patch( + "envctl_engine.state.run_index.atomic_write_text", + side_effect=replace_then_report_error, + ): + self.index.record( + state_path=state_path, + run_id="run-visible", + mode="trees", + project_names=["feature-a"], + ) + + self.assertEqual( + self.index.candidate_paths(StateSelector("trees", ("feature-a",))), + [state_path.resolve()], + ) + self.assertTrue(state_path.is_file()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/state/test_state_aggregation.py b/tests/python/state/test_state_aggregation.py new file mode 100644 index 00000000..2274bf75 --- /dev/null +++ b/tests/python/state/test_state_aggregation.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +from pathlib import Path + +from envctl_engine.state.models import RunState, ServiceRecord +from envctl_engine.state.run_index import RunIndexCandidate +from envctl_engine.state.state_aggregation import ( + IndexedState, + select_indexed_owners, + state_from_indexed_owners, +) + + +def _candidate( + run_id: str, + *projects: str, + sequence: int, + activation_sequence: int, +) -> RunIndexCandidate: + return RunIndexCandidate( + state_path=Path("/runtime") / run_id / "run_state.json", + run_id=run_id, + mode="trees", + project_names=tuple(project.casefold() for project in projects), + sequence=sequence, + activation_sequence=activation_sequence, + ) + + +def _service_project_name(_name: str, service: ServiceRecord) -> str: + return str(service.project or "") + + +def _project_names(state: RunState) -> list[str]: + raw = state.metadata.get("project_names") + return [str(name) for name in raw] if isinstance(raw, list) else [] + + +def _source_run_ids(state: RunState) -> list[str]: + raw = state.metadata.get("state_source_run_ids") + return [str(run_id) for run_id in raw] if isinstance(raw, list) else [] + + +def test_owner_selection_is_total_for_an_empty_inventory() -> None: + assert ( + select_indexed_owners( + [], + selected_projects=(), + service_project_name=_service_project_name, + ) + == [] + ) + + +def test_owner_selection_filters_shadowed_run_to_projects_it_still_owns() -> None: + newest = IndexedState( + candidate=_candidate("new", "alpha", sequence=2, activation_sequence=3), + state=RunState( + run_id="new", + mode="trees", + services={ + "alpha backend": ServiceRecord( + name="alpha backend", + type="backend", + cwd="/alpha", + project="alpha", + ) + }, + metadata={"project_names": ["alpha"]}, + ), + ) + previous = IndexedState( + candidate=_candidate("old", "alpha", "beta", sequence=1, activation_sequence=1), + state=RunState( + run_id="old", + mode="trees", + services={ + project: ServiceRecord( + name=project, + type="backend", + cwd=f"/{project}", + project=project, + ) + for project in ("alpha", "beta") + }, + metadata={"project_names": ["alpha", "beta"]}, + ), + ) + + selected = select_indexed_owners( + [newest, previous], + selected_projects=(), + service_project_name=_service_project_name, + ) + + assert [indexed.candidate.run_id for indexed in selected] == ["new", "old"] + assert set(selected[0].state.services) == {"alpha backend"} + assert set(selected[1].state.services) == {"beta"} + assert selected[1].state.metadata["project_names"] == ["beta"] + + +def test_merged_state_uses_activation_owner_for_global_identity() -> None: + activated = IndexedState( + candidate=_candidate("activated", "alpha", sequence=1, activation_sequence=5), + state=RunState( + run_id="activated", + mode="trees", + metadata={ + "project_names": ["alpha"], + "project_roots": {"alpha": "/alpha"}, + "global_marker": "activated", + "state_source_run_ids": ["ancestor"], + }, + ), + ) + later_sequence = IndexedState( + candidate=_candidate("stable", "beta", sequence=2, activation_sequence=2), + state=RunState( + run_id="stable", + mode="trees", + metadata={ + "project_names": ["beta"], + "project_roots": {"beta": "/beta"}, + "global_marker": "stable", + }, + ), + ) + + merged = state_from_indexed_owners( + [activated, later_sequence], + project_names_from_state=_project_names, + source_run_ids=_source_run_ids, + ) + + assert merged is not None + assert merged.run_id == "activated" + assert merged.metadata["global_marker"] == "activated" + assert merged.metadata["project_names"] == ["alpha", "beta"] + assert merged.metadata["state_source_run_ids"] == ["activated", "ancestor", "stable"] diff --git a/tests/python/state/test_state_repository_contract.py b/tests/python/state/test_state_repository_contract.py index 1fc8b92d..396d70db 100644 --- a/tests/python/state/test_state_repository_contract.py +++ b/tests/python/state/test_state_repository_contract.py @@ -4,20 +4,25 @@ import json import tempfile import unittest +from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from threading import Event from types import MappingProxyType from types import SimpleNamespace +from unittest.mock import patch REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" _models = importlib.import_module("envctl_engine.state.models") _state = importlib.import_module("envctl_engine.state") _state_repository = importlib.import_module("envctl_engine.state.repository") +_run_index = importlib.import_module("envctl_engine.state.run_index") RunState = _models.RunState ServiceRecord = _models.ServiceRecord dump_state = _state.dump_state load_state = _state.load_state RuntimeStateRepository = _state_repository.RuntimeStateRepository +StateSelector = _run_index.StateSelector class StateRepositoryContractTests(unittest.TestCase): @@ -212,10 +217,10 @@ def test_save_resume_state_updates_scoped_and_legacy_runtime_files(self) -> None self.assertTrue((legacy_root / "runtime_map.json").is_file()) main_pointer = runtime_root / ".last_state.main" self.assertTrue(main_pointer.is_file()) - self.assertEqual( - main_pointer.read_text(encoding="utf-8").strip(), - str(runtime_root / "runs" / "run-1" / "run_state.json"), - ) + pointer_target = Path(main_pointer.read_text(encoding="utf-8").strip()) + self.assertEqual(pointer_target.name, "run_state.json") + self.assertEqual(pointer_target.parent.parent.name, "revisions") + self.assertEqual(pointer_target.parent.parent.parent.name, "run-1") def test_save_selected_stop_state_updates_scoped_and_legacy_runtime_files(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -230,7 +235,19 @@ def test_save_selected_stop_state_updates_scoped_and_legacy_runtime_files(self) runtime_scope_id="repo-123", compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, ) - state = RunState(run_id="run-2", mode="trees", metadata={"repo_scope_id": "repo-123"}) + state = RunState( + run_id="run-2", + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/tmp/feature-a/backend", + project="feature-a", + ) + }, + metadata={"repo_scope_id": "repo-123"}, + ) repo.save_selected_stop_state( state=state, @@ -290,10 +307,10 @@ def test_save_resume_main_pointer_survives_following_trees_runtime_update(self) ) main_pointer = runtime_root / ".last_state.main" main_pointer_target = main_pointer.read_text(encoding="utf-8").strip() - self.assertEqual( - main_pointer_target, - str(runtime_root / "runs" / "run-main-resume" / "run_state.json"), - ) + main_target = Path(main_pointer_target) + self.assertEqual(main_target.name, "run_state.json") + self.assertEqual(main_target.parent.parent.name, "revisions") + self.assertEqual(main_target.parent.parent.parent.name, "run-main-resume") tree_state = RunState( run_id="run-tree-update", @@ -368,7 +385,9 @@ def test_save_run_main_preserves_existing_tree_pointers_for_mode_switch_resume(s ) self.assertTrue(tree_pointer.is_file()) - self.assertEqual(tree_pointer.read_text(encoding="utf-8").strip(), str(tree_state_path)) + migrated_target = Path(tree_pointer.read_text(encoding="utf-8").strip()) + self.assertEqual(load_state(str(migrated_target)).run_id, "run-tree") + self.assertIn("revisions", migrated_target.parts) def test_load_latest_ignores_invalid_scoped_pointer_and_uses_next_valid_candidate(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -472,7 +491,9 @@ def test_save_run_trees_preserves_existing_main_pointer_for_mode_switch_resume(s ) self.assertTrue(main_pointer.is_file()) - self.assertEqual(main_pointer.read_text(encoding="utf-8").strip(), str(main_state_path)) + migrated_target = Path(main_pointer.read_text(encoding="utf-8").strip()) + self.assertEqual(load_state(str(migrated_target)).run_id, "run-main") + self.assertIn("revisions", migrated_target.parts) loaded_main = repo.load_latest(mode="main", strict_mode_match=True) self.assertIsNotNone(loaded_main) @@ -495,7 +516,9 @@ def test_save_run_trees_uses_safe_pointer_filename_for_project_name(self) -> Non compat_mode=RuntimeStateRepository.SCOPED_ONLY, ) - tree_context = SimpleNamespace(name="Feature/API v2", root=Path(tmpdir) / "trees" / "Feature_API_v2", ports={}) + tree_context = SimpleNamespace( + name="Feature/API v2", root=Path(tmpdir) / "trees" / "Feature_API_v2", ports={} + ) state = RunState(run_id="run-tree-safe-name", mode="trees", metadata={"repo_scope_id": "repo-123"}) repo.save_run( state=state, @@ -506,87 +529,1833 @@ def test_save_run_trees_uses_safe_pointer_filename_for_project_name(self) -> Non runtime_map_builder=lambda _state: {"projection": {}}, ) - safe_pointer = runtime_root / ".last_state.trees.Feature_API_v2" + pointer_name = repo._tree_pointer_name("Feature/API v2") + self.assertIsNotNone(pointer_name) + safe_pointer = runtime_root / str(pointer_name) unsafe_pointer_parent = runtime_root / ".last_state.trees.Feature" self.assertTrue(safe_pointer.is_file()) + self.assertIn("Feature_API_v2-", safe_pointer.name) self.assertFalse(unsafe_pointer_parent.exists()) loaded = repo.load_latest(mode="trees", strict_mode_match=True) self.assertIsNotNone(loaded) assert loaded is not None self.assertEqual(loaded.run_id, "run-tree-safe-name") - def test_resume_orchestrator_does_not_write_legacy_state_files_directly(self) -> None: - resume_path = REPO_ROOT / "python/envctl_engine/startup/resume_orchestrator.py" - raw = resume_path.read_text(encoding="utf-8") - self.assertNotIn('runtime_legacy_root / "run_state.json"', raw) - self.assertNotIn('runtime_legacy_root / "runtime_map.json"', raw) + def test_independent_tree_runs_remain_selection_addressable(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) - def test_lifecycle_cleanup_orchestrator_does_not_write_legacy_state_files_directly(self) -> None: - cleanup_path = REPO_ROOT / "python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py" - raw = cleanup_path.read_text(encoding="utf-8") - self.assertNotIn('runtime_legacy_root / "run_state.json"', raw) - self.assertNotIn('runtime_legacy_root / "runtime_map.json"', raw) + for project_name in ("feature-zeta", "feature-alpha"): + state = RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=f"/tmp/{project_name}", + pid=123, + requested_port=8000, + actual_port=8000, + status="running", + ) + }, + metadata={"repo_scope_id": "repo-123"}, + ) + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + self.assertTrue((runtime_root / ".last_state.trees.feature-zeta").is_file()) + self.assertTrue((runtime_root / ".last_state.trees.feature-alpha").is_file()) + zeta = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-zeta"], + ) + alpha = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-alpha"], + ) - def test_engine_runtime_legacy_compat_helper_is_removed(self) -> None: - runtime_path = REPO_ROOT / "python/envctl_engine/runtime/engine_runtime.py" - raw = runtime_path.read_text(encoding="utf-8") - self.assertNotIn("def _write_legacy_runtime_compat_files(", raw) + self.assertIsNotNone(zeta) + self.assertIsNotNone(alpha) + assert zeta is not None and alpha is not None + self.assertEqual(zeta.run_id, "run-feature-zeta") + self.assertEqual(alpha.run_id, "run-feature-alpha") - def test_aggressive_purge_removes_runtime_scoped_test_results_with_run(self) -> None: + def test_selection_aware_lookup_uses_new_owner_when_superset_supersedes_exact_run(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: runtime_dir = Path(tmpdir) / "runtime" runtime_root = runtime_dir / "scope" - legacy_root = runtime_dir / "python-engine" - runtime_root.mkdir(parents=True, exist_ok=True) - legacy_root.mkdir(parents=True, exist_ok=True) repo = RuntimeStateRepository( runtime_root=runtime_root, - runtime_legacy_root=legacy_root, + runtime_legacy_root=runtime_dir / "python-engine", runtime_dir=runtime_dir, runtime_scope_id="repo-123", compat_mode=RuntimeStateRepository.SCOPED_ONLY, ) - run_state = RunState(run_id="run-1", mode="main", metadata={"repo_scope_id": "repo-123"}) + exact_context = SimpleNamespace(name="feature-a", root=Path(tmpdir) / "feature-a", ports={}) + repo.save_run( + state=RunState(run_id="run-exact", mode="trees", metadata={"repo_scope_id": "repo-123"}), + contexts=[exact_context], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + superset_contexts = [ + exact_context, + SimpleNamespace(name="feature-b", root=Path(tmpdir) / "feature-b", ports={}), + ] + repo.save_run( + state=RunState(run_id="run-superset", mode="trees", metadata={"repo_scope_id": "repo-123"}), + contexts=superset_contexts, + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + loaded = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual(loaded.run_id, "run-superset") + + def test_save_run_atomically_supersedes_every_source_run(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + contexts = [ + SimpleNamespace(name="feature-a", root=Path(tmpdir) / "feature-a", ports={}), + SimpleNamespace(name="feature-b", root=Path(tmpdir) / "feature-b", ports={}), + ] + old_state = RunState( + run_id="run-old", + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/tmp/feature-a", + project="feature-a", + ), + "feature-b Backend": ServiceRecord( + name="feature-b Backend", + type="backend", + cwd="/tmp/feature-b", + project="feature-b", + ), + }, + metadata={"repo_scope_id": "repo-123"}, + ) + repo.save_run( + state=old_state, + contexts=contexts, + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + replacement = RunState( + run_id="run-new", + mode="trees", + services=dict(old_state.services), + metadata={ + "repo_scope_id": "repo-123", + "state_source_run_ids": ["run-old"], + }, + ) + + repo.save_run( + state=replacement, + contexts=[contexts[0]], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + self.assertEqual([state.run_id for state in repo.load_all(mode="trees")], ["run-new"]) + loaded_b = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-b"], + ) + self.assertIsNotNone(loaded_b) + assert loaded_b is not None + self.assertEqual(loaded_b.run_id, "run-new") + self.assertEqual(set(loaded_b.services), {"feature-a Backend", "feature-b Backend"}) + with self.assertRaisesRegex(RuntimeError, "retired"): + repo.save_run( + state=old_state, + contexts=contexts, + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + def test_merged_state_uses_most_recently_activated_run_for_global_metadata(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + def save(project: str, run_id: str, marker: str) -> None: + repo.save_resume_state( + state=RunState( + run_id=run_id, + mode="trees", + services={ + f"{project} Backend": ServiceRecord( + name=f"{project} Backend", + type="backend", + cwd=f"/tmp/{project}", + project=project, + ) + }, + metadata={"repo_scope_id": "repo-123", "activation_marker": marker}, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + save("feature-a", "run-a", "a-original") + save("feature-b", "run-b", "b-newer-sequence") + save("feature-a", "run-a", "a-reactivated") + loaded = repo.load_latest(mode="trees", strict_mode_match=True) + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual(loaded.run_id, "run-a") + self.assertEqual(loaded.metadata["activation_marker"], "a-reactivated") + self.assertEqual(set(loaded.services), {"feature-a Backend", "feature-b Backend"}) + + def test_selection_aware_lookup_does_not_return_disjoint_latest_state(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + context = SimpleNamespace(name="feature-a", root=Path(tmpdir) / "feature-a", ports={}) + repo.save_run( + state=RunState(run_id="run-a", mode="trees", metadata={"repo_scope_id": "repo-123"}), + contexts=[context], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + loaded = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-b"], + ) + + self.assertIsNone(loaded) + + def test_multi_project_lookup_consolidates_disjoint_active_runs_without_losing_history(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + for project_name in ("feature-a", "feature-b"): + repo.save_resume_state( + state=RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=f"/tmp/{project_name}/backend", + project=project_name, + pid=100 if project_name == "feature-a" else 200, + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + merged = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a", "feature-b"], + ) + self.assertIsNotNone(merged) + assert merged is not None + self.assertEqual(set(merged.services), {"feature-a Backend", "feature-b Backend"}) + self.assertEqual(set(merged.metadata["project_names"]), {"feature-a", "feature-b"}) + repo.save_resume_state( - state=run_state, + state=merged, emit=lambda *_args, **_kwargs: None, runtime_map_builder=lambda _state: {"projection": {}}, ) - test_summary = ( - repo.test_results_dir_path("run-1", "run_20260309_100000") / "Main" / "failed_tests_summary.txt" + active = repo.load_all(mode="trees") + self.assertEqual([state.run_id for state in active], ["run-feature-b"]) + self.assertTrue((runtime_root / "runs" / "run-feature-a" / "run_state.json").is_file()) + + def test_load_all_filters_projects_subtracted_from_a_mixed_historical_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + mixed_services = { + f"{project} Backend": ServiceRecord( + name=f"{project} Backend", + type="backend", + cwd=f"/tmp/{project}/backend", + project=project, + ) + for project in ("feature-a", "feature-b") + } + repo.save_resume_state( + state=RunState(run_id="run-mixed", mode="trees", services=mixed_services), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, + ) + repo.save_resume_state( + state=RunState( + run_id="run-new-a", + mode="trees", + services={"feature-a Backend": mixed_services["feature-a Backend"]}, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, ) - test_summary.parent.mkdir(parents=True, exist_ok=True) - test_summary.write_text("No failed tests.\n", encoding="utf-8") - repo.purge(aggressive=False) - self.assertTrue(test_summary.is_file()) + active = {state.run_id: set(state.services) for state in repo.load_all(mode="trees")} - repo.purge(aggressive=True) - self.assertFalse(repo.run_dir_path("run-1").exists()) - self.assertFalse(test_summary.exists()) + self.assertEqual( + active, + { + "run-mixed": {"feature-b Backend"}, + "run-new-a": {"feature-a Backend"}, + }, + ) - def test_tree_diffs_dir_path_scopes_to_run_and_runtime_root(self) -> None: + def test_aggregate_and_subset_saves_keep_project_metadata_owned_by_live_projects(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: runtime_dir = Path(tmpdir) / "runtime" runtime_root = runtime_dir / "scope" - legacy_root = runtime_dir / "python-engine" - runtime_root.mkdir(parents=True, exist_ok=True) - legacy_root.mkdir(parents=True, exist_ok=True) repo = RuntimeStateRepository( runtime_root=runtime_root, - runtime_legacy_root=legacy_root, + runtime_legacy_root=runtime_dir / "python-engine", runtime_dir=runtime_dir, runtime_scope_id="repo-123", compat_mode=RuntimeStateRepository.SCOPED_ONLY, ) - self.assertEqual(repo.tree_diffs_dir_path(), runtime_root / "tree-diffs") + for project_name, service_type in (("feature-a", "backend"), ("feature-b", "frontend")): + repo.save_resume_state( + state=RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} {service_type}": ServiceRecord( + name=f"{project_name} {service_type}", + type=service_type, + cwd=f"/tmp/{project_name}/{service_type}", + project=project_name, + ) + }, + metadata={ + "project_roots": { + project_name: f"/tmp/{project_name}", + "stale-project": "/tmp/stale", + }, + "project_pr_links": { + project_name: f"pr-{project_name}", + "stale-project": "pr-stale", + }, + "project_test_summaries": { + project_name: {"failed": 0}, + "stale-project": {"failed": 99}, + }, + "project_test_results_root": f"/tmp/results/{project_name}", + "project_test_results_updated_at": project_name, + "project_action_reports": { + project_name: {"test": {"status": "success"}}, + "stale-project": {"test": {"status": "failed"}}, + }, + "dashboard_project_configured_services": { + project_name: [service_type], + "stale-project": ["worker"], + }, + "dashboard_configured_service_types": [service_type, "worker"], + "dashboard_stopped_services": [ + { + "name": f"{project_name} stopped", + "project": project_name, + "type": service_type, + }, + { + "name": "stale stopped", + "project": "stale-project", + "type": "worker", + }, + ], + "startup_identity": { + "mode": "trees", + "projects": [ + {"name": project_name, "root": f"/tmp/{project_name}"}, + {"name": "stale-project", "root": "/tmp/stale"}, + ], + "fingerprint": project_name, + }, + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + merged = repo.load_latest(mode="trees", strict_mode_match=True) + self.assertIsNotNone(merged) + assert merged is not None + for key in ( + "project_roots", + "project_pr_links", + "project_test_summaries", + "project_action_reports", + "dashboard_project_configured_services", + ): + self.assertEqual(set(merged.metadata[key]), {"feature-a", "feature-b"}) self.assertEqual( - repo.tree_diffs_dir_path("run-1", "review"), - runtime_root / "runs" / "run-1" / "tree-diffs" / "review", + {item["project"] for item in merged.metadata["dashboard_stopped_services"]}, + {"feature-a", "feature-b"}, + ) + self.assertEqual( + merged.metadata["dashboard_configured_service_types"], + ["backend", "frontend"], + ) + self.assertEqual( + {project["name"] for project in merged.metadata["startup_identity"]["projects"]}, + {"feature-a", "feature-b"}, ) + self.assertNotIn("project_test_results_root", merged.metadata) + self.assertNotIn("project_test_results_updated_at", merged.metadata) + + selected = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + self.assertIsNotNone(selected) + assert selected is not None + for key in ( + "project_roots", + "project_pr_links", + "project_test_summaries", + "project_action_reports", + "dashboard_project_configured_services", + ): + self.assertEqual(set(selected.metadata[key]), {"feature-a"}) + self.assertEqual( + {item["project"] for item in selected.metadata["dashboard_stopped_services"]}, + {"feature-a"}, + ) + self.assertEqual(selected.metadata["dashboard_configured_service_types"], ["backend"]) + + merged.services = { + name: service for name, service in merged.services.items() if service.project == "feature-a" + } + repo.save_resume_state( + state=merged, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + saved_subset = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + self.assertIsNotNone(saved_subset) + assert saved_subset is not None + for key in ( + "project_roots", + "project_pr_links", + "project_test_summaries", + "project_action_reports", + "dashboard_project_configured_services", + ): + self.assertEqual(set(saved_subset.metadata[key]), {"feature-a"}) + self.assertEqual( + {item["project"] for item in saved_subset.metadata["dashboard_stopped_services"]}, + {"feature-a"}, + ) + self.assertIsNone( + repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-b"], + ) + ) + + def test_deactivate_run_preserves_other_active_run_and_historical_artifacts(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + for project_name in ("feature-a", "feature-b"): + repo.save_resume_state( + state=RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=f"/tmp/{project_name}/backend", + project=project_name, + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, + ) + + self.assertTrue(repo.deactivate_run("run-feature-b")) + self.assertFalse(repo.deactivate_run("run-feature-b")) + self.assertIsNone(repo.load_latest(mode="trees", strict_mode_match=True, project_names=["feature-b"])) + remaining = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + self.assertIsNotNone(remaining) + assert remaining is not None + self.assertEqual(remaining.run_id, "run-feature-a") + self.assertEqual(load_state(str(runtime_root / "run_state.json")).run_id, "run-feature-a") + self.assertTrue((runtime_root / "runs" / "run-feature-b" / "run_state.json").is_file()) + + self.assertTrue(repo.deactivate_run("run-feature-a")) + self.assertEqual(repo.load_all(mode="trees"), []) + self.assertFalse((runtime_root / "run_state.json").exists()) + self.assertTrue((runtime_root / "runs" / "run-feature-a" / "run_state.json").is_file()) + + def test_concurrent_saves_publish_coherent_current_snapshot_and_keep_both_runs(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + first_builder_entered = Event() + release_first_builder = Event() + second_builder_entered = Event() + + def save(project_name: str) -> None: + state = RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=f"/tmp/{project_name}/backend", + project=project_name, + ) + }, + ) + + def runtime_map_builder(_state: RunState) -> dict[str, object]: + if project_name == "feature-a": + first_builder_entered.set() + self.assertTrue(release_first_builder.wait(timeout=2)) + else: + second_builder_entered.set() + return {"run_id": state.run_id} + + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=runtime_map_builder, + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + first = executor.submit(save, "feature-a") + self.assertTrue(first_builder_entered.wait(timeout=2)) + second = executor.submit(save, "feature-b") + self.assertTrue(second_builder_entered.wait(timeout=0.1)) + release_first_builder.set() + first.result(timeout=2) + second.result(timeout=2) + + current_state = load_state(str(runtime_root / "run_state.json")) + current_map = json.loads((runtime_root / "runtime_map.json").read_text(encoding="utf-8")) + self.assertEqual(current_state.run_id, current_map["run_id"]) + self.assertEqual( + {state.run_id for state in repo.load_all(mode="trees")}, {"run-feature-a", "run-feature-b"} + ) + self.assertEqual(list(runtime_root.rglob("*.tmp")), []) + + def test_purge_deactivates_index_without_deleting_history_unless_aggressive(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_resume_state( + state=RunState(run_id="run-a", mode="main"), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + historical_state = runtime_root / "runs" / "run-a" / "run_state.json" + + repo.purge(aggressive=False) + + self.assertIsNone(repo.load_latest()) + index_payload = json.loads((runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(index_payload["entries"], []) + self.assertTrue(historical_state.is_file()) + + repo.purge(aggressive=True) + self.assertFalse(historical_state.exists()) + + def test_invalid_run_ids_and_artifact_components_cannot_escape_runtime_root(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + for run_id in (".", "..", "../escaped", "nested/run", "nested\\run", "/absolute"): + with self.subTest(run_id=run_id): + runtime_root = runtime_dir / f"scope-{abs(hash(run_id))}" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id=f"scope-{abs(hash(run_id))}", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + with self.assertRaisesRegex(ValueError, "path component"): + repo.save_resume_state( + state=RunState(run_id=run_id, mode="trees"), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + self.assertFalse(runtime_root.exists()) + + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "safe-scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="safe-scope", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + for invalid_component in ("..", "../escaped", "/absolute", "nested/name"): + with self.subTest(component=invalid_component): + with self.assertRaisesRegex(ValueError, "path component"): + repo.test_results_dir_path("run-a", invalid_component) + with self.assertRaisesRegex(ValueError, "path component"): + repo.tree_diffs_dir_path("run-a", invalid_component) + + def test_missing_or_corrupt_index_is_rebuilt_from_scoped_run_history(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + for project_name in ("feature-a", "feature-b"): + state_path = runtime_root / "runs" / f"run-{project_name}" / "run_state.json" + state_path.parent.mkdir(parents=True, exist_ok=True) + dump_state( + RunState( + run_id=f"run-{project_name}", + mode="trees", + services={ + f"{project_name} Backend": ServiceRecord( + name=f"{project_name} Backend", + type="backend", + cwd=f"/tmp/{project_name}/backend", + project=project_name, + ) + }, + ), + str(state_path), + ) + + recovered_a = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + self.assertIsNotNone(recovered_a) + assert recovered_a is not None + self.assertEqual(recovered_a.run_id, "run-feature-a") + self.assertEqual({state.run_id for state in repo.load_all()}, {"run-feature-a", "run-feature-b"}) + + (runtime_root / "run_index.json").write_text('{"entries":', encoding="utf-8") + recovered_b = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-b"], + ) + self.assertIsNotNone(recovered_b) + assert recovered_b is not None + self.assertEqual(recovered_b.run_id, "run-feature-b") + rebuilt_payload = json.loads((runtime_root / "run_index.json").read_text(encoding="utf-8")) + self.assertEqual(len(rebuilt_payload["entries"]), 2) + + def test_resume_orchestrator_does_not_write_legacy_state_files_directly(self) -> None: + resume_path = REPO_ROOT / "python/envctl_engine/startup/resume_orchestrator.py" + raw = resume_path.read_text(encoding="utf-8") + self.assertNotIn('runtime_legacy_root / "run_state.json"', raw) + self.assertNotIn('runtime_legacy_root / "runtime_map.json"', raw) + + def test_lifecycle_cleanup_orchestrator_does_not_write_legacy_state_files_directly(self) -> None: + cleanup_path = REPO_ROOT / "python/envctl_engine/runtime/lifecycle_cleanup_orchestrator.py" + raw = cleanup_path.read_text(encoding="utf-8") + self.assertNotIn('runtime_legacy_root / "run_state.json"', raw) + self.assertNotIn('runtime_legacy_root / "runtime_map.json"', raw) + + def test_engine_runtime_legacy_compat_helper_is_removed(self) -> None: + runtime_path = REPO_ROOT / "python/envctl_engine/runtime/engine_runtime.py" + raw = runtime_path.read_text(encoding="utf-8") + self.assertNotIn("def _write_legacy_runtime_compat_files(", raw) + + def test_aggressive_purge_removes_runtime_scoped_test_results_with_run(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + runtime_root.mkdir(parents=True, exist_ok=True) + legacy_root.mkdir(parents=True, exist_ok=True) + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + run_state = RunState(run_id="run-1", mode="main", metadata={"repo_scope_id": "repo-123"}) + repo.save_resume_state( + state=run_state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + test_summary = ( + repo.test_results_dir_path("run-1", "run_20260309_100000") / "Main" / "failed_tests_summary.txt" + ) + test_summary.parent.mkdir(parents=True, exist_ok=True) + test_summary.write_text("No failed tests.\n", encoding="utf-8") + + repo.purge(aggressive=False) + self.assertTrue(test_summary.is_file()) + + repo.purge(aggressive=True) + self.assertFalse(repo.run_dir_path("run-1").exists()) + self.assertFalse(test_summary.exists()) + + def test_tree_diffs_dir_path_scopes_to_run_and_runtime_root(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + runtime_root.mkdir(parents=True, exist_ok=True) + legacy_root.mkdir(parents=True, exist_ok=True) + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + self.assertEqual(repo.tree_diffs_dir_path(), (runtime_root / "tree-diffs").resolve()) + self.assertEqual( + repo.tree_diffs_dir_path("run-1", "review"), + (runtime_root / "runs" / "run-1" / "tree-diffs" / "review").resolve(), + ) + + def test_runtime_artifact_writer_updates_active_revision_and_derived_aliases(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + state = RunState(run_id="run-1", mode="main") + repo.save_run( + state=state, + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + accepted = repo.write_runtime_artifact( + run_id="run-1", + artifact_name="runtime_readiness_report.json", + text='{"passed": true}', + ) + + self.assertTrue(accepted) + candidate = repo.run_index.candidates(_state_repository.StateSelector(mode=None, project_names=()))[0] + artifact_paths = ( + candidate.state_path.parent / "runtime_readiness_report.json", + repo.run_dir_path("run-1") / "runtime_readiness_report.json", + runtime_root / "runtime_readiness_report.json", + legacy_root / "runtime_readiness_report.json", + ) + for path in artifact_paths: + with self.subTest(path=path): + self.assertEqual(path.read_text(encoding="utf-8"), '{"passed": true}') + + def test_retired_runtime_artifact_writer_cannot_recreate_files_after_deactivation(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + repo.save_run( + state=RunState(run_id="run-retired", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + self.assertTrue(repo.deactivate_run("run-retired")) + + accepted = repo.write_runtime_artifact( + run_id="run-retired", + artifact_name="runtime_readiness_report.json", + text='{"stale": true}', + ) + + self.assertFalse(accepted) + self.assertFalse((repo.run_dir_path("run-retired") / "runtime_readiness_report.json").exists()) + self.assertFalse((runtime_root / "runtime_readiness_report.json").exists()) + self.assertFalse((legacy_root / "runtime_readiness_report.json").exists()) + + def test_stale_runtime_artifact_writer_cannot_recreate_files_after_aggressive_purge(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + repo.save_run( + state=RunState(run_id="run-purged", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + repo.purge(aggressive=True) + + readiness_accepted = repo.write_runtime_artifact( + run_id="run-purged", + artifact_name="runtime_readiness_report.json", + text='{"stale": true}', + ) + events_accepted = repo.write_runtime_artifact( + run_id="run-purged", + artifact_name="events.jsonl", + text='{"event": "stale"}\n', + ) + + self.assertFalse(readiness_accepted) + self.assertFalse(events_accepted) + self.assertFalse(repo.run_dir_path("run-purged").exists()) + for root in (runtime_root, legacy_root): + with self.subTest(root=root): + self.assertFalse((root / "runtime_readiness_report.json").exists()) + self.assertFalse((root / "events.jsonl").exists()) + + def test_purge_waits_for_inflight_runtime_artifact_write_and_removes_its_output(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_run( + state=RunState(run_id="run-racing", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + artifact_write_started = Event() + release_artifact_write = Event() + original_write_text = repo._write_text + + def blocking_write_text(path: Path, text: str) -> None: + if ( + path.name == "runtime_readiness_report.json" + and "revisions" in path.parts + and not artifact_write_started.is_set() + ): + artifact_write_started.set() + self.assertTrue(release_artifact_write.wait(timeout=2)) + original_write_text(path, text) + + repo._write_text = blocking_write_text # type: ignore[method-assign] + with ThreadPoolExecutor(max_workers=2) as executor: + writer = executor.submit( + repo.write_runtime_artifact, + run_id="run-racing", + artifact_name="runtime_readiness_report.json", + text='{"passed": true}', + ) + self.assertTrue(artifact_write_started.wait(timeout=2)) + purge = executor.submit(repo.purge, aggressive=True) + release_artifact_write.set() + self.assertTrue(writer.result(timeout=2)) + purge.result(timeout=2) + + self.assertFalse(repo.run_dir_path("run-racing").exists()) + self.assertFalse((runtime_root / "runtime_readiness_report.json").exists()) + + def test_unbound_runtime_artifact_is_scoped_and_artifact_name_is_allowlisted(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + accepted = repo.write_runtime_artifact( + run_id=None, + artifact_name="events.jsonl", + text='{"event": "doctor"}\n', + ) + + self.assertTrue(accepted) + self.assertEqual( + (runtime_root / "events.jsonl").read_text(encoding="utf-8"), + '{"event": "doctor"}\n', + ) + self.assertFalse((legacy_root / "events.jsonl").exists()) + with self.assertRaisesRegex(ValueError, "unsupported mutable runtime artifact"): + repo.write_runtime_artifact( + run_id=None, + artifact_name="run_state.json", + text="{}", + ) + with self.assertRaisesRegex(ValueError, "path component"): + repo.write_runtime_artifact( + run_id=None, + artifact_name="../events.jsonl", + text="", + ) + + def test_unbound_runtime_artifact_never_attaches_to_an_active_run(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_run( + state=RunState(run_id="run-active", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[{"event": "run-bound"}], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + candidate = repo.run_index.candidates(_state_repository.StateSelector(mode=None, project_names=()))[0] + + accepted = repo.write_runtime_artifact( + run_id=None, + artifact_name="events.jsonl", + text='{"event": "scope-diagnostic"}\n', + ) + + self.assertTrue(accepted) + self.assertEqual( + (runtime_root / "events.jsonl").read_text(encoding="utf-8"), + json.dumps({"event": "run-bound"}, sort_keys=True) + "\n", + ) + self.assertEqual( + (runtime_root / "diagnostics" / "events.jsonl").read_text(encoding="utf-8"), + '{"event": "scope-diagnostic"}\n', + ) + expected_run_text = json.dumps({"event": "run-bound"}, sort_keys=True) + "\n" + self.assertEqual( + (repo.run_dir_path("run-active") / "events.jsonl").read_text(encoding="utf-8"), + expected_run_text, + ) + self.assertEqual( + (candidate.state_path.parent / "events.jsonl").read_text(encoding="utf-8"), + expected_run_text, + ) + + def test_retired_overlapping_run_artifact_writer_is_fenced_from_current_alias(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + for run_id in ("run-old", "run-new"): + repo.save_run( + state=RunState(run_id=run_id, mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[{"event": run_id}], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + + accepted = repo.write_runtime_artifact( + run_id="run-old", + artifact_name="events.jsonl", + text='{"event": "late-old-writer"}\n', + ) + + self.assertFalse(accepted) + self.assertEqual( + (repo.run_dir_path("run-old") / "events.jsonl").read_text(encoding="utf-8"), + json.dumps({"event": "run-old"}, sort_keys=True) + "\n", + ) + expected_current = json.dumps({"event": "run-new"}, sort_keys=True) + "\n" + self.assertEqual((runtime_root / "events.jsonl").read_text(encoding="utf-8"), expected_current) + self.assertEqual((legacy_root / "events.jsonl").read_text(encoding="utf-8"), expected_current) + + def test_corrupt_authoritative_revision_does_not_reactivate_retired_project_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + for run_id, pid in (("run-old", 101), ("run-new", 202)): + repo.save_resume_state( + state=RunState( + run_id=run_id, + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/tmp/feature-a/backend", + project="feature-a", + pid=pid, + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda state: {"run_id": state.run_id}, + ) + + candidates = repo.run_index.candidates(_state_repository.StateSelector(mode="trees", project_names=())) + new_candidate = next(candidate for candidate in candidates if candidate.run_id == "run-new") + new_candidate.state_path.write_text('{"broken":', encoding="utf-8") + + loaded = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["feature-a"], + ) + + self.assertIsNone(loaded) + self.assertFalse((runtime_root / "run_state.json").exists()) + self.assertFalse((runtime_root / ".last_state").exists()) + + def test_generic_lookup_skips_corrupt_newest_mode_and_returns_newest_valid_mode(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_resume_state( + state=RunState( + run_id="run-main", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + repo.save_resume_state( + state=RunState( + run_id="run-trees", + mode="trees", + services={ + "feature-a Backend": ServiceRecord( + name="feature-a Backend", + type="backend", + cwd="/tmp/feature-a/backend", + project="feature-a", + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + corrupt_candidate = repo.run_index.candidates( + _state_repository.StateSelector(mode="trees", project_names=()) + )[0] + corrupt_candidate.state_path.write_text('{"broken":', encoding="utf-8") + + loaded = repo.load_latest() + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual((loaded.run_id, loaded.mode), ("run-main", "main")) + self.assertEqual(load_state(str(runtime_root / "run_state.json")).run_id, "run-main") + + def test_non_strict_project_lookup_filters_projects_before_newest_mode(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + repo = RuntimeStateRepository( + runtime_root=runtime_dir / "scope", + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_resume_state( + state=RunState( + run_id="run-tree", + mode="trees", + services={ + "FeatureA Backend": ServiceRecord( + name="FeatureA Backend", + type="backend", + cwd="/tmp/FeatureA/backend", + project="FeatureA", + ) + }, + metadata={"project_names": ["FeatureA"]}, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + repo.save_resume_state( + state=RunState( + run_id="run-main", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + ) + }, + metadata={"project_names": ["Main"]}, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + loaded = repo.load_latest( + mode="main", + strict_mode_match=False, + project_names=["FeatureA"], + ) + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual((loaded.run_id, loaded.mode), ("run-tree", "trees")) + + def test_resuming_older_mode_reactivates_it_without_changing_project_ownership_precedence(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + main_state = RunState( + run_id="run-main", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/main/backend", + project="Main", + ) + }, + ) + tree_state = RunState( + run_id="run-tree", + mode="trees", + services={ + "FeatureA Backend": ServiceRecord( + name="FeatureA Backend", + type="backend", + cwd="/tmp/FeatureA/backend", + project="FeatureA", + ) + }, + ) + for state in (main_state, tree_state): + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + self.assertEqual(repo.load_latest().run_id, "run-tree") # type: ignore[union-attr] + + repo.save_resume_state( + state=main_state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + current = repo.load_latest() + self.assertIsNotNone(current) + assert current is not None + self.assertEqual((current.run_id, current.mode), ("run-main", "main")) + tree = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["FeatureA"], + ) + self.assertIsNotNone(tree) + assert tree is not None + self.assertEqual(tree.run_id, "run-tree") + + def test_corrupt_primary_and_backup_after_deactivation_fails_closed_without_resurrection(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + repo.save_resume_state( + state=RunState( + run_id="retired-run", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/backend", + project="Main", + ) + }, + ), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + self.assertTrue(repo.deactivate_run("retired-run")) + historical_state = repo.run_dir_path("retired-run") / "run_state.json" + self.assertTrue(historical_state.is_file()) + repo.run_index.index_path.write_text('{"broken":', encoding="utf-8") + repo.run_index.backup_path.write_text('{"broken":', encoding="utf-8") + + with self.assertRaisesRegex(RuntimeError, "both the primary and backup"): + repo.load_latest() + + self.assertTrue(historical_state.is_file()) + self.assertFalse((runtime_root / "run_state.json").exists()) + + def test_failed_legacy_registry_publish_cleans_staged_revision_directories(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_state = runtime_root / "runs" / "legacy-run" / "run_state.json" + legacy_state.parent.mkdir(parents=True, exist_ok=True) + dump_state( + RunState( + run_id="legacy-run", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/backend", + project="Main", + ) + }, + ), + str(legacy_state), + ) + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + + with patch.object(repo.run_index, "replace_all", side_effect=OSError("registry unavailable")): + with self.assertRaisesRegex(OSError, "registry unavailable"): + repo.load_latest() + + revisions = legacy_state.parent / "revisions" + self.assertEqual(list(revisions.iterdir()) if revisions.is_dir() else [], []) + self.assertTrue(legacy_state.is_file()) + self.assertFalse((runtime_root / "run_registry.json").exists()) + + def test_read_only_and_scoped_purges_never_mutate_legacy_runtime(self) -> None: + for compat_mode in ( + RuntimeStateRepository.COMPAT_READ_ONLY, + RuntimeStateRepository.SCOPED_ONLY, + ): + with self.subTest(compat_mode=compat_mode), tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + legacy_run = legacy_root / "runs" / "legacy-run" + legacy_run.mkdir(parents=True, exist_ok=True) + sentinels = { + legacy_root / "run_state.json": "legacy-state", + legacy_root / "events.jsonl": "legacy-event", + legacy_root / ".last_state": "legacy-pointer", + legacy_run / "run_state.json": "legacy-history", + legacy_root / ".events.jsonl.stale.tmp": "legacy-temp", + } + for path, text in sentinels.items(): + path.write_text(text, encoding="utf-8") + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=compat_mode, + ) + + repo.purge(aggressive=True) + + for path, expected in sentinels.items(): + self.assertEqual(path.read_text(encoding="utf-8"), expected) + + def test_first_operation_read_only_purge_cannot_reimport_legacy_after_index_loss(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + legacy_root.mkdir(parents=True) + legacy_state = legacy_root / "run_state.json" + dump_state(RunState(run_id="legacy-run", mode="main"), str(legacy_state)) + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_ONLY, + ) + + repo.purge(aggressive=False) + self.assertTrue(legacy_state.is_file()) + self.assertTrue((runtime_root / "run_registry.json").is_file()) + repo.run_index.index_path.unlink() + repo.run_index.backup_path.unlink() + + with self.assertRaisesRegex(RuntimeError, "both the primary and backup"): + repo.load_latest() + + self.assertTrue(legacy_state.is_file()) + self.assertFalse((runtime_root / "run_state.json").exists()) + + def test_runtime_artifact_directories_reject_symlink_redirection(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + external = Path(tmpdir) / "external" + external.mkdir() + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + runs_root = repo.run_dir_path(None) + runs_root.mkdir(parents=True) + run_link = runs_root / "run-a" + run_link.symlink_to(external, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "symlink"): + repo.run_dir_path("run-a") + run_link.unlink() + + run_dir = repo.run_dir_path("run-a") + run_dir.mkdir() + for name, resolver in ( + ("test-results", lambda: repo.test_results_dir_path("run-a")), + ("tree-diffs", lambda: repo.tree_diffs_dir_path("run-a")), + ): + with self.subTest(name=name): + link = run_dir / name + link.symlink_to(external, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "symlink"): + resolver() + link.unlink() + + root_tree_diffs = runtime_root / "tree-diffs" + root_tree_diffs.symlink_to(external, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "symlink"): + repo.tree_diffs_dir_path() + self.assertEqual(list(external.iterdir()), []) + + def test_runtime_root_retarget_is_rejected_before_lock_or_purge_writes(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + runtime_root.mkdir(parents=True) + legacy_root.mkdir() + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + external = Path(tmpdir) / "external" + external.mkdir() + sentinel = external / "run_state.json" + sentinel.write_text("outside-sentinel", encoding="utf-8") + runtime_root.rmdir() + runtime_root.symlink_to(external, target_is_directory=True) + + with self.assertRaisesRegex(RuntimeError, "symlink"): + repo.purge(aggressive=True) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "outside-sentinel") + self.assertEqual(list(external.iterdir()), [sentinel]) + + def test_legacy_root_retarget_is_rejected_before_compatibility_writes(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + runtime_root.mkdir(parents=True) + legacy_root.mkdir() + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + external = Path(tmpdir) / "external" + external.mkdir() + sentinel = external / "run_state.json" + sentinel.write_text("outside-sentinel", encoding="utf-8") + legacy_root.rmdir() + legacy_root.symlink_to(external, target_is_directory=True) + + with self.assertRaisesRegex(RuntimeError, "symlink"): + repo.save_resume_state( + state=RunState(run_id="run-a", mode="main"), + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + self.assertEqual(sentinel.read_text(encoding="utf-8"), "outside-sentinel") + self.assertEqual(list(external.iterdir()), [sentinel]) + + def test_runtime_update_carries_forward_ports_errors_events_and_readiness(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + state = RunState( + run_id="run-a", + mode="main", + services={ + "Main Backend": ServiceRecord( + name="Main Backend", + type="backend", + cwd="/tmp/backend", + project="Main", + pid=123, + ) + }, + ) + repo.save_run( + state=state, + contexts=[self._context(Path(tmpdir))], + errors=["diagnostic"], + events=[{"event": "initial"}], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {}}, + ) + self.assertTrue( + repo.write_runtime_artifact( + run_id="run-a", + artifact_name="runtime_readiness_report.json", + text='{"passed": true}', + ) + ) + artifact_names = ( + "ports_manifest.json", + "error_report.json", + "events.jsonl", + "runtime_readiness_report.json", + ) + expected = {name: (runtime_root / name).read_text(encoding="utf-8") for name in artifact_names} + + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {"projection": {"Main": {}}}, + ) + + candidate = repo.run_index.candidates(_state_repository.StateSelector(mode="main", project_names=()))[0] + for artifact_name, expected_text in expected.items(): + for root in (runtime_root, legacy_root, candidate.state_path.parent): + with self.subTest(artifact_name=artifact_name, root=root): + actual_text = (root / artifact_name).read_text(encoding="utf-8") + if artifact_name == "error_report.json": + payload = json.loads(actual_text) + self.assertEqual(payload["errors"], ["diagnostic"]) + self.assertEqual(payload["source_run_ids"], ["run-a"]) + else: + self.assertEqual(actual_text, expected_text) + + def test_partial_tree_stop_rebuilds_pointer_inventory_without_resurrecting_removed_project(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + state = RunState( + run_id="run-ab", + mode="trees", + services={ + project: ServiceRecord( + name=project, + type="backend", + cwd=f"/tmp/{project}/backend", + project=project, + ) + for project in ("FeatureA", "FeatureB") + }, + ) + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + pointer_a_name = repo._tree_pointer_name("FeatureA") + pointer_b_name = repo._tree_pointer_name("FeatureB") + assert pointer_a_name is not None and pointer_b_name is not None + old_target = Path((runtime_root / pointer_a_name).read_text(encoding="utf-8").strip()) + self.assertTrue(old_target.is_file()) + + state.services.pop("FeatureA") + repo.save_selected_stop_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + for root in (runtime_root, legacy_root): + self.assertFalse((root / pointer_a_name).exists()) + self.assertTrue((root / pointer_b_name).is_file()) + self.assertTrue(old_target.is_file()) + self.assertIsNone( + repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["FeatureA"], + ) + ) + remaining = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["FeatureB"], + ) + self.assertIsNotNone(remaining) + assert remaining is not None + self.assertEqual(set(remaining.services), {"FeatureB"}) + ports_payload = json.loads((runtime_root / "ports_manifest.json").read_text(encoding="utf-8")) + self.assertEqual( + [project["project"] for project in ports_payload["projects"]], + ["FeatureB"], + ) + + def test_selected_stop_can_preserve_authoritative_metadata_only_tree_owner(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + legacy_root = runtime_dir / "python-engine" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=legacy_root, + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + state = RunState( + run_id="run-metadata-ab", + mode="trees", + metadata={ + "project_names": ["FeatureA", "FeatureB"], + "project_roots": { + "FeatureA": "/tmp/FeatureA", + "FeatureB": "/tmp/FeatureB", + }, + }, + ) + repo.save_selected_stop_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + authoritative_project_names=["FeatureA", "FeatureB"], + ) + + state.metadata["project_names"] = ["FeatureB"] + state.metadata["project_roots"] = {"FeatureB": "/tmp/FeatureB"} + repo.save_selected_stop_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + authoritative_project_names=["FeatureB"], + ) + + candidates = repo.run_index.candidates(StateSelector(mode="trees", project_names=())) + self.assertEqual( + [(candidate.run_id, candidate.project_names) for candidate in candidates], + [("run-metadata-ab", ("featureb",))], + ) + self.assertIsNone( + repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["FeatureA"], + ) + ) + remaining = repo.load_latest( + mode="trees", + strict_mode_match=True, + project_names=["FeatureB"], + ) + self.assertIsNotNone(remaining) + assert remaining is not None + self.assertEqual(remaining.services, {}) + self.assertEqual(remaining.requirements, {}) + self.assertEqual(remaining.metadata["project_names"], ["FeatureB"]) + self.assertEqual(remaining.metadata["project_roots"], {"FeatureB": "/tmp/FeatureB"}) + current = json.loads(repo.run_state_path().read_text(encoding="utf-8")) + self.assertEqual(current["metadata"]["project_names"], ["FeatureB"]) + pointer_a = repo._tree_pointer_name("FeatureA") + pointer_b = repo._tree_pointer_name("FeatureB") + assert pointer_a is not None and pointer_b is not None + self.assertFalse((runtime_root / pointer_a).exists()) + self.assertTrue((runtime_root / pointer_b).exists()) + + def test_lossy_tree_names_and_mixed_case_get_distinct_stable_pointers(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + projects = ("Feature/API v2", "Feature_API_v2") + for index, project in enumerate(projects, start=1): + repo.save_run( + state=RunState(run_id=f"run-{index}", mode="trees"), + contexts=[SimpleNamespace(name=project, root=Path(tmpdir) / f"tree-{index}", ports={})], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + + pointer_names = [repo._tree_pointer_name(project) for project in projects] + self.assertTrue(all(pointer_names)) + self.assertEqual(len(set(pointer_names)), 2) + pointed_runs = { + load_state((runtime_root / str(pointer_name)).read_text(encoding="utf-8").strip()).run_id + for pointer_name in pointer_names + } + self.assertEqual(pointed_runs, {"run-1", "run-2"}) + self.assertEqual( + {pointer.name for pointer in runtime_root.glob(".last_state.trees.*")}, + set(pointer_names), + ) + + def test_post_commit_per_run_alias_failure_is_repaired_on_next_read(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) / "runtime" + runtime_root = runtime_dir / "scope" + repo = RuntimeStateRepository( + runtime_root=runtime_root, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id="repo-123", + compat_mode=RuntimeStateRepository.SCOPED_ONLY, + ) + run_alias = repo.run_dir_path("run-a") / "run_state.json" + original_write_text = repo._write_text + failed = False + + def fail_first_run_alias(path: Path, text: str) -> None: + nonlocal failed + if path == run_alias and not failed: + failed = True + raise OSError("alias unavailable") + original_write_text(path, text) + + repo._write_text = fail_first_run_alias # type: ignore[method-assign] + repo.save_run( + state=RunState(run_id="run-a", mode="main"), + contexts=[self._context(Path(tmpdir))], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda _state: {}, + ) + self.assertTrue(failed) + self.assertFalse(run_alias.exists()) + repo._write_text = original_write_text # type: ignore[method-assign] + + loaded = repo.load_latest() + + self.assertIsNotNone(loaded) + self.assertTrue(run_alias.is_file()) + self.assertEqual(load_state(str(run_alias)).run_id, "run-a") if __name__ == "__main__": diff --git a/tests/python/state/test_state_repository_hardening.py b/tests/python/state/test_state_repository_hardening.py new file mode 100644 index 00000000..7a84f305 --- /dev/null +++ b/tests/python/state/test_state_repository_hardening.py @@ -0,0 +1,638 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from envctl_engine.state import dump_state, load_state +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord +from envctl_engine.state.persistence import fsync_directory +from envctl_engine.state.repository import RuntimeStateRepository +from envctl_engine.state.run_index import StateSelector + + +class StateRepositoryHardeningTests(unittest.TestCase): + @staticmethod + def _repository( + root: Path, + *, + scope: str = "scope-a", + compat_mode: str = RuntimeStateRepository.SCOPED_ONLY, + ) -> RuntimeStateRepository: + runtime_dir = root / "runtime" + return RuntimeStateRepository( + runtime_root=runtime_dir / scope, + runtime_legacy_root=runtime_dir / "python-engine", + runtime_dir=runtime_dir, + runtime_scope_id=scope, + compat_mode=compat_mode, + ) + + @staticmethod + def _state( + run_id: str, + project: str, + *, + port: int = 8000, + metadata: dict[str, object] | None = None, + ) -> RunState: + return RunState( + run_id=run_id, + mode="main" if project.casefold() == "main" else "trees", + services={ + f"{project} Backend": ServiceRecord( + name=f"{project} Backend", + type="backend", + cwd=f"/tmp/{project}/backend", + project=project, + requested_port=port, + actual_port=port, + status="running", + ) + }, + metadata=dict(metadata or {}), + ) + + @staticmethod + def _context(root: Path, project: str, port: int) -> object: + return SimpleNamespace( + name=project, + root=root / project, + ports={ + "backend": SimpleNamespace( + requested=port, + assigned=port, + final=port, + source="requested", + retries=0, + ) + }, + ) + + @staticmethod + def _save_resume(repo: RuntimeStateRepository, state: RunState) -> None: + repo.save_resume_state( + state=state, + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda saved: {"run_id": saved.run_id}, + ) + + @staticmethod + def _save_run( + repo: RuntimeStateRepository, + state: RunState, + *, + context: object, + errors: list[str] | None = None, + ) -> None: + repo.save_run( + state=state, + contexts=[context], + errors=list(errors or []), + events=[{"event": state.run_id}], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda saved: {"run_id": saved.run_id}, + ) + + def test_clean_save_scavenges_crash_temps_for_rewritten_alias_targets(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + state = self._state("run-a", "Main") + self._save_resume(repo, state) + run_dir = repo.run_dir_path("run-a") + stale_root_alias = repo.runtime_root / ".run_state.json.abcdefgh.tmp" + stale_run_alias = run_dir / ".run_state.json.ijklmnop.tmp" + stale_index = repo.runtime_root / ".run_index.json.qrstuvwx.tmp" + for path in (stale_root_alias, stale_run_alias, stale_index): + path.write_text("crash debris", encoding="utf-8") + + self._save_resume(repo, state) + + self.assertFalse(stale_root_alias.exists()) + self.assertFalse(stale_run_alias.exists()) + self.assertFalse(stale_index.exists()) + + def test_revision_retention_keeps_indexed_revision_and_one_grace_revision(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + revisions_dir = repo.run_dir_path("run-a") / "revisions" + external = root / "external" + external.mkdir() + + with patch( + "envctl_engine.state.repository.fsync_directory", + wraps=fsync_directory, + ) as synced_directory: + for port in range(8100, 8106): + self._save_resume(repo, self._state("run-a", "feature-a", port=port)) + real_revisions = [ + path for path in revisions_dir.iterdir() if path.is_dir() and not path.is_symlink() + ] + self.assertLessEqual(len(real_revisions), 2) + + candidate = repo.run_index.candidates(StateSelector(mode="trees", project_names=()))[0] + real_revisions = [path for path in revisions_dir.iterdir() if path.is_dir() and not path.is_symlink()] + self.assertEqual(len(real_revisions), 2) + self.assertIn(candidate.state_path.parent, real_revisions) + self.assertTrue(any(call.args == (revisions_dir,) for call in synced_directory.call_args_list)) + + external_sentinel = external / "sentinel" + external_sentinel.write_text("safe", encoding="utf-8") + revision_link = revisions_dir / "foreign-link" + revision_link.symlink_to(external, target_is_directory=True) + self._save_resume(repo, self._state("run-a", "feature-a", port=9000)) + self.assertTrue(revision_link.is_symlink()) + self.assertEqual(external_sentinel.read_text(encoding="utf-8"), "safe") + + def test_revision_pruning_is_deferred_until_failed_promotion_is_repaired(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + for port in (8100, 8101): + self._save_resume(repo, self._state("run-a", "feature-a", port=port)) + revisions_dir = repo.run_dir_path("run-a") / "revisions" + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 2) + + with patch.object( + repo, + "_promote_latest_active_state", + side_effect=OSError("alias promotion unavailable"), + ): + self._save_resume(repo, self._state("run-a", "feature-a", port=8102)) + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 3) + + loaded = repo.load_latest(mode="trees", strict_mode_match=True) + self.assertIsNotNone(loaded) + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 2) + + def test_supersession_prunes_failed_promotion_backlog_for_retired_run(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + for port in (8100, 8101): + self._save_resume(repo, self._state("run-old", "feature-a", port=port)) + revisions_dir = repo.run_dir_path("run-old") / "revisions" + with patch.object( + repo, + "_promote_latest_active_state", + side_effect=OSError("alias promotion unavailable"), + ): + for port in (8102, 8103, 8104): + self._save_resume(repo, self._state("run-old", "feature-a", port=port)) + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 5) + + replacement = self._state( + "run-new", + "feature-a", + port=8200, + metadata={"state_source_run_ids": ["run-old"]}, + ) + self._save_resume(repo, replacement) + + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 2) + payload = json.loads(repo.run_index.index_path.read_text(encoding="utf-8")) + self.assertEqual([entry["run_id"] for entry in payload["entries"]], ["run-new"]) + self.assertIn("run-old", payload["retired_run_ids"]) + + def test_revision_cleanup_io_failures_do_not_lose_state_and_are_retried(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + for port in (8100, 8101): + self._save_resume(repo, self._state("run-old", "feature-a", port=port)) + revisions_dir = repo.run_dir_path("run-old") / "revisions" + with patch.object( + repo, + "_promote_latest_active_state", + side_effect=OSError("alias promotion unavailable"), + ): + for port in (8102, 8103, 8104): + self._save_resume(repo, self._state("run-old", "feature-a", port=port)) + + replacement = self._state( + "run-new", + "feature-a", + port=8200, + metadata={"state_source_run_ids": ["run-old"]}, + ) + with patch( + "envctl_engine.state.repository.shutil.rmtree", + side_effect=OSError("cleanup unavailable"), + ): + self._save_resume(repo, replacement) + + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 5) + cleanup_dir = repo.runtime_root / repo._REVISION_CLEANUP_DIR_NAME + self.assertEqual(len(list(cleanup_dir.glob("*.json"))), 1) + for index in range(100): + clean_history = repo.run_dir_path(f"inactive-{index}") / "revisions" / "only-revision" + clean_history.mkdir(parents=True) + original_prune = repo._prune_run_revisions_unlocked + inspected_run_ids: list[str] = [] + + def recording_prune(candidate: object) -> None: + inspected_run_ids.append(str(getattr(candidate, "run_id", ""))) + original_prune(candidate) # type: ignore[arg-type] + + with patch.object( + repo, + "_prune_run_revisions_unlocked", + side_effect=recording_prune, + ): + loaded = repo.load_latest(mode="trees", strict_mode_match=True) + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual(loaded.run_id, "run-new") + self.assertEqual(len([path for path in revisions_dir.iterdir() if path.is_dir()]), 2) + self.assertEqual(set(inspected_run_ids), {"run-new", "run-old"}) + self.assertFalse(cleanup_dir.exists()) + + with patch( + "envctl_engine.state.repository.fsync_directory", + side_effect=OSError("directory fsync failed"), + ): + for port in (8201, 8202): + self._save_resume(repo, self._state("run-new", "feature-a", port=port)) + loaded = repo.load_latest(mode="trees", strict_mode_match=True) + self.assertIsNotNone(loaded) + self.assertLessEqual( + len([path for path in (repo.run_dir_path("run-new") / "revisions").iterdir() if path.is_dir()]), + 2, + ) + + def test_superseded_cleanup_is_queued_when_alias_promotion_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + with patch.object( + repo, + "_promote_latest_active_state", + side_effect=OSError("alias promotion unavailable"), + ): + for port in range(8100, 8105): + self._save_resume(repo, self._state("run-old", "feature-a", port=port)) + self._save_resume( + repo, + self._state( + "run-new", + "feature-a", + port=8200, + metadata={"state_source_run_ids": ["run-old"]}, + ), + ) + + old_revisions = repo.run_dir_path("run-old") / "revisions" + cleanup_dir = repo.runtime_root / repo._REVISION_CLEANUP_DIR_NAME + self.assertEqual( + len([path for path in old_revisions.iterdir() if path.is_dir()]), + 5, + ) + self.assertEqual(len(list(cleanup_dir.glob("*.json"))), 1) + + loaded = repo.load_latest(mode="trees", strict_mode_match=True) + + self.assertIsNotNone(loaded) + self.assertLessEqual( + len([path for path in old_revisions.iterdir() if path.is_dir()]), + 2, + ) + self.assertFalse(cleanup_dir.exists()) + + def test_current_ancillary_alias_signatures_detect_tampering_and_source_updates(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository( + root, + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + self._save_run( + repo, + self._state("run-a", "Main"), + context=self._context(root, "Main", 8000), + errors=["expected error"], + ) + self.assertTrue( + repo.write_runtime_artifact( + run_id="run-a", + artifact_name="runtime_readiness_report.json", + text='{"passed": true}', + ) + ) + candidate = repo.run_index.candidates(StateSelector(mode="main", project_names=()))[0] + scoped_ports = repo.ports_manifest_path() + expected_ports = scoped_ports.read_text(encoding="utf-8") + scoped_ports.write_text("x" * len(expected_ports), encoding="utf-8") + repo.load_latest() + self.assertEqual(scoped_ports.read_text(encoding="utf-8"), expected_ports) + + legacy_error = repo.runtime_legacy_root / "error_report.json" + expected_error = legacy_error.read_text(encoding="utf-8") + legacy_error.write_text("x" * len(expected_error), encoding="utf-8") + repo.load_latest() + repaired_error = json.loads(legacy_error.read_text(encoding="utf-8")) + self.assertEqual(repaired_error["errors"], ["expected error"]) + self.assertEqual( + legacy_error.read_text(encoding="utf-8"), + repo.error_report_path().read_text(encoding="utf-8"), + ) + + source_events = candidate.state_path.parent / "events.jsonl" + updated_events = '{"event": "late-source-update"}\n' + source_events.write_text(updated_events, encoding="utf-8") + repo.load_latest() + self.assertEqual( + (repo.runtime_root / "events.jsonl").read_text(encoding="utf-8"), + updated_events, + ) + self.assertEqual( + (repo.runtime_legacy_root / "events.jsonl").read_text(encoding="utf-8"), + updated_events, + ) + + def test_per_run_alias_manifest_uses_signatures_and_repairs_tampering(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + self._save_run( + repo, + self._state("run-a", "Main"), + context=self._context(root, "Main", 8000), + ) + self.assertTrue( + repo.write_runtime_artifact( + run_id="run-a", + artifact_name="runtime_readiness_report.json", + text='{"passed": true}', + ) + ) + repo.load_latest() + candidate = repo.run_index.candidates(StateSelector(mode="main", project_names=()))[0] + + observed_reads: list[Path] = [] + original_read_text = Path.read_text + + def recording_read_text(path: Path, *args: object, **kwargs: object) -> str: + observed_reads.append(path) + return original_read_text(path, *args, **kwargs) + + indexed = repo._load_valid_index_states( + [candidate], + allowed_root=str(repo.runtime_dir), + ) + observed_reads.clear() + with patch.object(Path, "read_text", recording_read_text): + repo._reconcile_run_aliases_unlocked(indexed) + + source_dir = candidate.state_path.parent + run_dir = repo.run_dir_path("run-a") + compared_artifacts = {source_dir / artifact_name for artifact_name in repo._RUN_ALIAS_ARTIFACT_NAMES} | { + run_dir / artifact_name for artifact_name in repo._RUN_ALIAS_ARTIFACT_NAMES + } + self.assertTrue(compared_artifacts.isdisjoint(observed_reads)) + + tampered_alias = run_dir / "events.jsonl" + expected_events = (source_dir / "events.jsonl").read_text(encoding="utf-8") + tampered_alias.write_text("x" * len(expected_events), encoding="utf-8") + repo.load_latest() + self.assertEqual(tampered_alias.read_text(encoding="utf-8"), expected_events) + + def test_deactivate_does_not_retire_unknown_run_ids(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + repo = self._repository(Path(tmpdir)) + self._save_resume(repo, self._state("run-active", "Main")) + + self.assertFalse(repo.deactivate_run("never-active")) + self.assertTrue(repo.deactivate_runs(["run-active", "also-never-active"])) + + payload = json.loads(repo.run_index.index_path.read_text(encoding="utf-8")) + self.assertEqual(payload["retired_run_ids"], ["run-active"]) + + def test_legacy_migration_preserves_failed_health_state_and_all_artifacts(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + runtime_dir = root / "runtime" + legacy_root = runtime_dir / "python-engine" + legacy_root.mkdir(parents=True) + failed_state = RunState( + run_id="legacy-failed", + mode="main", + requirements={ + "Main": RequirementsResult( + project="Main", + health="degraded", + failures=["redis unavailable"], + components={ + "redis": { + "enabled": True, + "success": False, + "runtime_status": "unreachable", + } + }, + ) + }, + metadata={"failed": True}, + ) + dump_state(failed_state, str(legacy_root / "run_state.json")) + artifacts = { + "runtime_map.json": '{"legacy": true}', + "ports_manifest.json": '{"projects": [{"project": "Main", "ports": {}}]}', + "error_report.json": '{"errors": ["redis unavailable"]}', + "events.jsonl": '{"event": "failed"}\n', + "runtime_readiness_report.json": '{"passed": false}', + } + for artifact_name, text in artifacts.items(): + (legacy_root / artifact_name).write_text(text, encoding="utf-8") + + repo = self._repository( + root, + compat_mode=RuntimeStateRepository.COMPAT_READ_ONLY, + ) + loaded = repo.load_latest(mode="main", strict_mode_match=True) + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertTrue(loaded.metadata["failed"]) + self.assertEqual(loaded.requirements["Main"].health, "degraded") + self.assertEqual( + loaded.requirements["Main"].component("redis")["runtime_status"], + "unreachable", + ) + candidate = repo.run_index.candidates(StateSelector(mode="main", project_names=()))[0] + for artifact_name, expected in artifacts.items(): + self.assertEqual( + (candidate.state_path.parent / artifact_name).read_text(encoding="utf-8"), + expected, + ) + + def test_unbound_diagnostics_do_not_replace_active_aliases_and_purge_cleans_them(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository( + root, + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + runtime_root = repo.runtime_root + legacy_root = repo.runtime_legacy_root + no_run_event = '{"event": "no-run"}\n' + self.assertTrue( + repo.write_runtime_artifact( + run_id=None, + artifact_name="events.jsonl", + text=no_run_event, + ) + ) + self.assertEqual((runtime_root / "events.jsonl").read_text(encoding="utf-8"), no_run_event) + self.assertEqual((legacy_root / "events.jsonl").read_text(encoding="utf-8"), no_run_event) + + self._save_run( + repo, + self._state("run-a", "Main"), + context=self._context(root, "Main", 8000), + ) + active_event = (runtime_root / "events.jsonl").read_text(encoding="utf-8") + diagnostic_event = '{"event": "doctor"}\n' + self.assertTrue( + repo.write_runtime_artifact( + run_id=None, + artifact_name="events.jsonl", + text=diagnostic_event, + ) + ) + self.assertEqual((runtime_root / "events.jsonl").read_text(encoding="utf-8"), active_event) + self.assertEqual((legacy_root / "events.jsonl").read_text(encoding="utf-8"), active_event) + self.assertEqual( + (runtime_root / "diagnostics" / "events.jsonl").read_text(encoding="utf-8"), + diagnostic_event, + ) + + repo.purge(aggressive=False) + self.assertFalse((runtime_root / "diagnostics").exists()) + + def test_shared_legacy_pointers_are_only_removed_by_their_own_scope(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo_a = self._repository( + root, + scope="scope-a", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + repo_b = self._repository( + root, + scope="scope-b", + compat_mode=RuntimeStateRepository.COMPAT_READ_WRITE, + ) + self._save_run( + repo_a, + self._state("run-a", "feature-a"), + context=self._context(root, "feature-a", 8100), + ) + self._save_run( + repo_b, + self._state("run-b", "feature-b"), + context=self._context(root, "feature-b", 8200), + ) + legacy_root = repo_a.runtime_legacy_root + pointer_b_name = repo_b._tree_pointer_name("feature-b") + assert pointer_b_name is not None + pointer_b = legacy_root / pointer_b_name + generic_pointer = legacy_root / ".last_state" + self.assertIn("scope-b", Path(pointer_b.read_text(encoding="utf-8")).parts) + self.assertIn("scope-b", Path(generic_pointer.read_text(encoding="utf-8")).parts) + + self.assertTrue(repo_a.deactivate_run("run-a")) + + self.assertTrue(pointer_b.is_file()) + self.assertTrue(generic_pointer.is_file()) + self.assertIn("scope-b", Path(pointer_b.read_text(encoding="utf-8")).parts) + self.assertIn("scope-b", Path(generic_pointer.read_text(encoding="utf-8")).parts) + + def test_partial_restart_ports_merge_current_context_with_preserved_project(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + initial = RunState( + run_id="run-ab", + mode="trees", + services={ + **self._state("unused", "feature-a", port=8100).services, + **self._state("unused", "feature-b", port=8200).services, + }, + ) + repo.save_run( + state=initial, + contexts=[ + self._context(root, "feature-a", 8100), + self._context(root, "feature-b", 8200), + ], + errors=[], + events=[], + emit=lambda *_args, **_kwargs: None, + runtime_map_builder=lambda saved: {"run_id": saved.run_id}, + ) + restarted = RunState( + run_id="run-restart-a", + mode="trees", + services={ + **self._state("unused", "feature-a", port=8111).services, + **self._state("unused", "feature-b", port=8200).services, + }, + metadata={"state_source_run_ids": ["run-ab"]}, + ) + self._save_run( + repo, + restarted, + context=self._context(root, "feature-a", 8111), + ) + + payload = json.loads(repo.ports_manifest_path().read_text(encoding="utf-8")) + ports_by_project = { + project["project"]: project["ports"]["backend"]["final"] for project in payload["projects"] + } + self.assertEqual(ports_by_project, {"feature-a": 8111, "feature-b": 8200}) + + def test_aggregate_aliases_merge_independent_owner_ports_and_errors(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + repo = self._repository(root) + self._save_run( + repo, + self._state("run-a", "feature-a", port=8100), + context=self._context(root, "feature-a", 8100), + errors=["feature-a failed"], + ) + self._save_run( + repo, + self._state("run-b", "feature-b", port=8200), + context=self._context(root, "feature-b", 8200), + errors=["feature-b failed"], + ) + + current = load_state(str(repo.run_state_path())) + self.assertEqual(set(current.services), {"feature-a Backend", "feature-b Backend"}) + ports = json.loads(repo.ports_manifest_path().read_text(encoding="utf-8")) + errors = json.loads(repo.error_report_path().read_text(encoding="utf-8")) + self.assertEqual( + {project["project"] for project in ports["projects"]}, + {"feature-a", "feature-b"}, + ) + self.assertEqual(set(errors["errors"]), {"feature-a failed", "feature-b failed"}) + + self.assertTrue(repo.deactivate_run("run-a")) + remaining = load_state(str(repo.run_state_path())) + ports = json.loads(repo.ports_manifest_path().read_text(encoding="utf-8")) + errors = json.loads(repo.error_report_path().read_text(encoding="utf-8")) + self.assertEqual(set(remaining.services), {"feature-b Backend"}) + self.assertEqual( + {project["project"] for project in ports["projects"]}, + {"feature-b"}, + ) + self.assertEqual(errors["errors"], ["feature-b failed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/state/test_state_roundtrip.py b/tests/python/state/test_state_roundtrip.py index 781aad67..b130670f 100644 --- a/tests/python/state/test_state_roundtrip.py +++ b/tests/python/state/test_state_roundtrip.py @@ -6,7 +6,7 @@ REPO_ROOT = Path(__file__).resolve().parents[3] PYTHON_ROOT = REPO_ROOT / "python" -from envctl_engine.state.models import RunState, ServiceRecord +from envctl_engine.state.models import RequirementsResult, RunState, ServiceRecord from envctl_engine.state import dump_state, load_legacy_shell_state, load_state, merge_states @@ -25,11 +25,22 @@ def test_json_state_roundtrip_preserves_services(self) -> None: pid=12345, requested_port=8000, actual_port=8001, + port_lock_session="planner-session-1", listener_pids=[12345, 12346], status="running", started_at=1234.5, ) }, + requirements={ + "Tree Alpha": RequirementsResult( + project="Tree Alpha", + db={ + "enabled": True, + "final": 5432, + "port_lock_session": "planner-session-1", + }, + ) + }, metadata={"source": "python"}, ) @@ -38,8 +49,16 @@ def test_json_state_roundtrip_preserves_services(self) -> None: self.assertEqual(loaded.run_id, "run-123") self.assertEqual(loaded.services["Tree Alpha Backend"].actual_port, 8001) + self.assertEqual( + loaded.services["Tree Alpha Backend"].port_lock_session, + "planner-session-1", + ) self.assertEqual(loaded.services["Tree Alpha Backend"].listener_pids, [12345, 12346]) self.assertEqual(loaded.services["Tree Alpha Backend"].started_at, 1234.5) + self.assertEqual( + loaded.requirements["Tree Alpha"].db["port_lock_session"], + "planner-session-1", + ) self.assertEqual(loaded.metadata["source"], "python") def test_legacy_shell_state_is_loaded_without_source_execution(self) -> None: diff --git a/tests/python/state/test_state_shell_compatibility.py b/tests/python/state/test_state_shell_compatibility.py index f293f9a3..a2ef3bca 100644 --- a/tests/python/state/test_state_shell_compatibility.py +++ b/tests/python/state/test_state_shell_compatibility.py @@ -134,6 +134,21 @@ def test_engine_runtime_resolves_state_from_pointer_files(self) -> None: assert loaded is not None self.assertEqual(loaded.mode, "main") self.assertIn("Main Backend", loaded.services) + primary_index = runtime.runtime_root / "run_index.json" + backup_index = runtime.runtime_root / "run_index.backup.json" + self.assertEqual( + primary_index.read_text(encoding="utf-8"), + backup_index.read_text(encoding="utf-8"), + ) + migrated_pointer = Path((runtime.runtime_root / ".last_state.main").read_text(encoding="utf-8").strip()) + self.assertTrue(migrated_pointer.is_file()) + self.assertTrue(migrated_pointer.is_relative_to(runtime.runtime_root / "runs")) + self.assertTrue(state_path.is_file()) + + loaded_again = runtime._try_load_existing_state() + self.assertIsNotNone(loaded_again) + assert loaded_again is not None + self.assertEqual(loaded_again.run_id, loaded.run_id) if __name__ == "__main__":