Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion contracts/python_engine_parity_manifest.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"generated_at": "2026-07-06T17:49:55+00:00",
"generated_at": "2026-07-15T22:24:17+00:00",
"engine_default": "python",
"modes": {
"main": {
Expand Down
2 changes: 2 additions & 0 deletions docs/reference/commands.md
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,8 @@ Behavior:
- deletes imported worktrees when a PR closes or merges
- comments with CPU, memory, disk, Docker, and process evidence when the runner
is too loaded to deploy
- counts only controller states that still consume runtime capacity; pending
labels by themselves do not count as active previews
- runs storage-preserving `envctl blast-all` cleanup when active preview count
reaches zero, passing keep-volume flags so leftover containers and processes
are killed without deleting copied runtime storage
Expand Down
39 changes: 29 additions & 10 deletions python/envctl_engine/pr_preview_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@
r"^ENVCTL_[A-Z][A-Z0-9_]*_ENV__[A-Z_][A-Z0-9_]*$"
)
FAILED_START_STATUSES = frozenset({"public_route_unhealthy", "start_failed"})
CAPACITY_CONSUMING_PREVIEW_STATUSES = frozenset({"running", "starting", "qa_user_failed", "public_route_failed"})

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Count failed stops as capacity-consuming

stop() records status = "stop_failed" when envctl stop returns nonzero while leaving the state’s project intact, which is exactly a case where the runtime may still be alive. Because this allow-list omits stop_failed, later starts ignore that preview in max_other_active_previews even if the PR still has the deploy label, such as after a failed manual stop, so the controller can admit more previews onto an already occupied runner.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Include failed-start cleanup states in capacity

When envctl start fails or the public-route health check fails, the controller saves start_failed/public_route_unhealthy before attempting cleanup; if cleanup_failed_start_preview() returns nonzero, that state remains while the partially started runtime may still be alive and the label is left for retry. Since this capacity allow-list excludes both statuses, later PRs ignore that occupied runtime and can pass ENVCTL_PREVIEW_MAX_OTHER_ACTIVE even though the earlier start did not clean up.

Useful? React with 👍 / 👎.

PUBLIC_ROUTE_HEALTH_DEADLINE_SECONDS = 30.0
PUBLIC_ROUTE_HEALTH_INTERVAL_SECONDS = 2.0
PUBLIC_ROUTE_REQUEST_TIMEOUT_SECONDS = 5.0
Expand Down Expand Up @@ -1213,7 +1214,7 @@ def run_capture(argv: list[str], *, timeout: float) -> subprocess.CompletedProce

def overload_reasons(
stats: MachineStats,
active_prs: list[PullRequestInfo],
active_previews: list[PreviewState],
current_pr_number: int,
config: ControllerConfig,
) -> list[str]:
Expand All @@ -1236,7 +1237,7 @@ def overload_reasons(
reasons.append(
f"free disk is {stats.disk_free_percent:.1f}%, below threshold {config.min_disk_free_percent:.1f}%"
)
other_active = [pr for pr in active_prs if pr.number != current_pr_number]
other_active = [state for state in active_previews if state.pr_number != current_pr_number]
if len(other_active) >= config.max_other_active_previews:
reasons.append(
f"{len(other_active)} other envctl previews are already active, "
Expand Down Expand Up @@ -1895,16 +1896,26 @@ def start(
)
return 0

active_prs = self.active_prs_with_label()
active_previews = self.capacity_consuming_preview_states()

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Persist a starting state before admitting capacity

With this switch to capacity_consuming_preview_states(), admission no longer observes PRs that have the label but have not yet written a state file. In the start() path I inspected, no starting state is saved before envctl import/envctl start; the first save happens later in failure/success branches. If two label or synchronize jobs run at the same time, both can read zero capacity-consuming previews and both pass ENVCTL_PREVIEW_MAX_OTHER_ACTIVE, oversubscribing the runner; the previous label-based query would count the other labeled PR.

Useful? React with 👍 / 👎.

stats = collect_machine_stats(self.config.preview_root, self.runner)
reasons = overload_reasons(stats, active_prs, pr.number, self.config)
reasons = overload_reasons(
stats,
active_previews,
pr.number,
self.config,
)
if reasons:
state = self.load_state(pr.number)
if state and state.project:
self.stop(pr, reason="self-hosted machine overloaded", comment=False)
self.comment(
pr.number,
self.render_overload_comment(pr, stats, reasons, active_prs),
self.render_overload_comment(
pr,
stats,
reasons,
active_previews,
),
)
self.remove_label(pr.number)
return 0
Expand Down Expand Up @@ -3165,6 +3176,13 @@ def load_all_states(self) -> list[PreviewState]:
continue
return states

def capacity_consuming_preview_states(self) -> list[PreviewState]:
return [
state
for state in self.load_all_states()
if state.project and state.status in CAPACITY_CONSUMING_PREVIEW_STATUSES
]

def load_state(self, pr_number: int) -> PreviewState | None:
path = self.state_path(pr_number)
if not path.exists():
Expand Down Expand Up @@ -3440,11 +3458,11 @@ def render_overload_comment(
pr: PullRequestInfo,
stats: MachineStats,
reasons: list[str],
active_prs: list[PullRequestInfo],
active_previews: list[PreviewState],
) -> str:
other_prs = [item for item in active_prs if item.number != pr.number]
other_previews = [state for state in active_previews if state.pr_number != pr.number]
other_lines = (
"\n".join(f"- #{item.number}: [{item.title}]({item.url}) on `{item.head_ref}`" for item in other_prs)
"\n".join(f"- PR #{state.pr_number}: `{state.status}` on `{state.project}`" for state in other_previews)
or "- None"
)
process_lines = "\n".join(f" - `{line}`" for line in stats.top_processes) or " - unavailable"
Expand Down Expand Up @@ -3475,8 +3493,9 @@ def render_overload_comment(
f"({stats.disk_free_percent:.1f}%; "
f"threshold {self.config.min_disk_free_percent:.1f}%)\n"
f"- Docker containers: {docker_line}\n"
f"- Active PRs with `{self.config.label}`: {len(active_prs)}\n\n"
"Other active PRs with this label:\n"
"- Capacity-consuming envctl previews: "
f"{len(active_previews)}\n\n"
"Other capacity-consuming previews:\n"
f"{other_lines}\n\n"
"Top CPU processes:\n"
f"{process_lines}"
Expand Down
94 changes: 83 additions & 11 deletions tests/python/runtime/test_pr_preview_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -496,17 +496,19 @@ def test_overload_reasons_report_real_threshold_breaches(tmp_path):
docker_error=None,
top_processes=(),
)
other = controller.PullRequestInfo(
number=123,
title="Other",
url="https://example.test/pr/123",
state="OPEN",
merged=False,
other = controller.PreviewState(
pr_number=123,
label="deploy-app",
project="feature/other",
root=str(tmp_path / "other"),
head_ref="feature/other",
head_sha="abc",
head_repo_name="pele-monorepo",
head_repo_owner="OrgPele",
labels=("deploy-app",),
status="running",
label_added_at="2026-07-15T17:00:00Z",
started_at="2026-07-15T17:00:00Z",
expires_at="2026-07-15T17:30:00Z",
updated_at="2026-07-15T17:00:00Z",
endpoints={},
)

reasons = controller.overload_reasons(stats, [other], 456, config)
Expand All @@ -517,7 +519,7 @@ def test_overload_reasons_report_real_threshold_breaches(tmp_path):
assert any("other envctl previews" in reason for reason in reasons)


def test_overloaded_start_comments_stats_lists_other_prs_and_removes_label(
def test_overloaded_start_comments_stats_lists_other_previews_and_removes_label(
tmp_path,
monkeypatch,
):
Expand All @@ -532,6 +534,22 @@ def test_overloaded_start_comments_stats_lists_other_prs_and_removes_label(
],
)
instance = controller.PreviewController(config, runner)
instance.save_state(
controller.PreviewState(
pr_number=123,
label="deploy-app",
project="feature/other",
root=str(tmp_path / "other"),
head_ref="feature/other",
head_sha="abc",
status="running",
label_added_at="2026-07-15T17:00:00Z",
started_at="2026-07-15T17:00:00Z",
expires_at="2026-07-15T17:30:00Z",
updated_at="2026-07-15T17:00:00Z",
endpoints={},
)
)
stats = controller.MachineStats(
load_1m=8.0,
load_5m=5.0,
Expand Down Expand Up @@ -561,10 +579,64 @@ def test_overloaded_start_comments_stats_lists_other_prs_and_removes_label(
comment = runner.comments[-1]
assert "self-hosted machine is overloaded" in comment
assert "Load average: 8.00, 5.00, 3.00" in comment
assert "#123: [Other preview]" in comment
assert "PR #123: `running` on `feature/other`" in comment
assert "Capacity-consuming envctl previews: 1" in comment
assert "111 python 250.0 5.0" in comment


def test_capacity_admission_ignores_other_labeled_prs_without_preview_state(
tmp_path,
):
controller = load_controller()
root = tmp_path / "control" / "trees" / "imported" / "feature-demo"
root.mkdir(parents=True)
runner = FakeRunner(
controller,
active_prs=[
pr_list_payload(789, "Current", head_ref="feature/demo"),
pr_list_payload(123, "Requested one", head_ref="feature/one"),
pr_list_payload(124, "Requested two", head_ref="feature/two"),
pr_list_payload(125, "Requested three", head_ref="feature/three"),
],
projects={
"projects": [
{
"name": "feature/demo",
"root": str(root),
"running": True,
}
]
},
endpoints={
"frontend": {"port": 9000},
"backend": {"port": 8000},
},
root_to_branch={str(root): "feature/demo"},
)
base_config = make_config(controller, tmp_path)
config = controller.ControllerConfig(
**{
**base_config.__dict__,
"max_other_active_previews": 1,
}
)
instance = controller.PreviewController(config, runner)

exit_code = instance.run_pull_request_event(
pr_payload(
action="labeled",
labels=["deploy-app"],
event_label="deploy-app",
)
)

assert exit_code == 0
assert command_argvs(runner, "envctl", "import")
assert command_argvs(runner, "envctl", "start")
assert runner.removed_labels == []
assert instance.load_state(789).status == "running"


def test_overloaded_start_stops_stale_tracked_preview_before_label_removal(
tmp_path,
monkeypatch,
Expand Down
Loading