diff --git a/fork-ci/actions/pr-queue/README.md b/fork-ci/actions/pr-queue/README.md index 0cc43caa726bf..9f6c4cadf9f1a 100644 --- a/fork-ci/actions/pr-queue/README.md +++ b/fork-ci/actions/pr-queue/README.md @@ -8,9 +8,10 @@ only for a head that holds exactly the change the review read. This folder is built in four stacked pull requests: 1. the review rules and the config (`lens.py`, `config.py`); -2. the queue itself (`pr_queue.py`, section "The queue"): this part; +2. the queue itself (`pr_queue.py`, section "The queue"); 3. armed pull requests that cannot merge (a red required check, or one that - ended without passing or never reported) are told once and passed over; + ended without passing or never reported) are told once and passed over: + this part; 4. the "red only because the base was red" rule, the reusable workflow `.github/workflows/fork-pr-queue.yml`, `action.yml` and the usage sections of this file. @@ -110,6 +111,25 @@ first, and acts on the first one it can: A pull request in a merge state the queue neither updates nor waits for (a conflict, `DIRTY`) is passed over, and the run says so. +An armed pull request that cannot merge does not hold the queue. It is told +why on the pull request, once per head, check and reason, and passed over: + +- a required check whose newest run failed (`FAILURE`, `ERROR`, `TIMED_OUT`, + or `CANCELLED` after a red with nothing green since), behind or not. Its + author fixes it; nothing in the queue re-runs it. +- up to date, and a required check's newest run ended without passing and + without a red (`CANCELLED`, `STARTUP_FAILURE`, `STALE`, ...): nothing + re-runs it by itself. The comment says to re-run the CI run or push. +- up to date, and a required check never reported although no workflow runs + on the head any more. The comment says to push or merge the base: a re-run + repeats the jobs of the workflow files the head was built with, so it cannot + add a check those files do not have (one added or renamed since). + +Only the newest run of a check counts, and a check not in `required_checks` is +ignored. Only the queue's own comment (posted with its token) counts as told: +a copy of it by anyone else does not silence it. When GitHub cannot say which workflows still run on the head, the +pull request is not called stuck: it holds the queue, and the run says why. + `gh` must be logged in for the repository (`GH_TOKEN`). Git fetches into the queue's own bare repository (`--cache`) and gets its login from `gh auth git-credential` only, never from a helper the machine configured. @@ -177,4 +197,8 @@ everything project-specific comes from the config; `STATE` names its repository; exit 3 (was 2) means "may be off and silent", so that 2 can mean "refused"; the step that adopted an older queue's state file is gone; and the queue's own advice in a comment keeps its backticks and `` placeholder -(`lens.no_lens_text(keep_markdown=True)`), which the source dropped. +(`lens.no_lens_text(keep_markdown=True)`), which the source dropped. For a +required check that never reported, the comment says to push or merge the +base; the source said to re-run, which cannot add a check the head's workflow +files do not have. With two red checks the comment names the first by name +(the source named whichever the check list gave first). diff --git a/fork-ci/actions/pr-queue/pr_queue.py b/fork-ci/actions/pr-queue/pr_queue.py index b7731e2034db0..b9f29311c0637 100755 --- a/fork-ci/actions/pr-queue/pr_queue.py +++ b/fork-ci/actions/pr-queue/pr_queue.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 """A merge queue for a repository whose branch ruleset requires pull requests to be up to date with their base (README.md). Each run brings exactly ONE waiting PR up to date: the oldest open, non-draft PR with auto-merge on that is -BEHIND its base. Its CI runs, it merges itself, and the run after the next push to the base moves the queue on. An -armed PR that is already up to date holds the queue while its required checks run. +BEHIND its base and has no failing required check. Its CI runs, it merges itself, and the run after the next push to +the base moves the queue on. An armed PR that is already up to date holds the queue while its required checks run. One +whose required check ended without passing (cancelled) or never reported cannot merge and nothing re-runs that check: +it is told once per head and passed over, as is a red one. Auto-merge only ever merges a head someone checked. Whether GitHub checks --match-head-commit again after the head moved is not documented, and "armed" does not say at which head (a plain push keeps auto-merge on), so the queue relies @@ -38,7 +40,9 @@ SHA = re.compile(r'[0-9a-f]{40}') REPO_RE = re.compile(r'[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9._-]+') GONE_RE = re.compile(r"not our ref|unadvertised object|couldn't find remote ref|no such remote ref", re.I) +RED_STATES = ('FAILURE', 'ERROR', 'TIMED_OUT') WAITS_FOR = ('BLOCKED', 'UNSTABLE', 'CLEAN', 'HAS_HOOKS') # up-to-date merge states whose checks the queue waits for +NEVER = 'never reported' UPDATES = 3 # holds() follows at most this many queue updates in a row GIT_MIN = (2, 38) # git merge-tree --write-tree REFUSED, SILENT = 2, 3 @@ -264,6 +268,106 @@ def same_change(self, base, n, head, read): got = self.fetch(base, n, head, read) return holds(self.git, head, read) if got else got + # --- which armed PRs wait, and why ------------------------------------------------------------------------------ + def red_checks(self, pr): + """{name: rollup item of its newest run} of each required check whose newest run really failed: the rollup + lists every run on the head, and an older red must not outvote a newer run (lens.latest_checks). CANCELLED + alone is not a failure (a newer push or update cancels the old run), but a red that only a cancelled run + followed still stands: nothing ever passed.""" + rollup = pr.get('statusCheckRollup') or [] + name_of = lambda c: c.get('name') or c.get('context') + state_of = lambda c: (c.get('conclusion') or c.get('state') or '').upper() + out = {} + for c in lens.latest_checks(rollup): + name = name_of(c) + if name not in self.cfg.required_checks: + continue + if state_of(c) in RED_STATES or (state_of(c) == 'CANCELLED' and + any(name_of(o) == name and state_of(o) in RED_STATES for o in rollup)): + if (c.get('startedAt') or '') >= ((out.get(name) or {}).get('startedAt') or ''): + out[name] = c + return out + + def failing(self, pr): + return next(iter(sorted(self.red_checks(pr))), None) + + def required_now(self, rollup): + """lens.check_state of each required check's newest run, by name: the ruleset takes the newest run of a name, + whichever workflow ran it.""" + newest = {} + for c in lens.latest_checks(rollup): + name, when = c.get('name') or c.get('context'), c.get('startedAt') or '9999' + if name in self.cfg.required_checks and (name not in newest or when >= newest[name][0]): + newest[name] = (when, lens.check_state(c)) + return {name: state for name, (_, state) in newest.items()} + + def workflow_runs(self, head): + """The workflow runs on commit `head` (status, workflowName), or None when GitHub could not say (printed).""" + if not SHA.fullmatch(head or ''): + return None + r = self.gh('run', 'list', '--repo', self.repo, '--commit', head, '--limit', '50', '--json', + 'status,workflowName') + if r.returncode != 0: + print(f'gh run list --commit {head[:12]} failed: {r.stderr.strip()[:200]}') + return None + try: + return json.loads(r.stdout) + except ValueError: + print(f'gh run list --commit {head[:12]} gave no JSON') + return None + + def stuck(self, p): + """An up-to-date armed PR that cannot merge and that nothing moves on any more: (check, what) for a required + check whose newest run ended without passing and without a red (CANCELLED, STARTUP_FAILURE, STALE...), or that + never reported although no workflow runs on the head any more. None: it may still merge (green or running).""" + now = self.required_now(p.get('statusCheckRollup') or []) + ended = sorted((name, state) for name, state in now.items() if state not in ('ok', 'running')) + if ended: + return ended[0][0], f'ended {ended[0][1]}' + missing = sorted(self.cfg.required_checks - set(now)) + if 'running' in now.values() or not missing: + return None + if not (p.get('statusCheckRollup') or []): + print(f'#{p["number"]}: no check has started on its head yet') + return None + head = (p.get('headRefOid') or '').lower() + runs = self.workflow_runs(head) + if runs is None: # unknown is not "nothing runs": wait, and say it + print(f'#{p["number"]}: required check {", ".join(missing)} has not reported, and which workflows still ' + f'run on {head[:12] or "its head"} is unknown: waiting') + return None + if any((x.get('status') or '').lower() != 'completed' for x in runs): + return None + return missing[0], NEVER + + def stuck_fix(self, what, base): + if what == NEVER: # a re-run repeats the run's own workflow file, which has no job for this check + return (f'A re-run cannot add it: a re-run repeats the jobs of the workflow files this head was built ' + f'with, and none of them reported it. Push again (`{self.cfg.push_command}`) or merge {base} into ' + 'it, so that the current workflows run') + return 'Nothing re-runs it by itself: re-run the CI run (Actions, "Re-run failed jobs") or push' + + def tell_red(self, n, check, what='is red', fix='Fix the cause and push'): + """An armed PR the queue skips says why on the PR, once per head, check and reason. No answer from GitHub: the + next run tries again. Told means a comment of the queue's own login (viewerDidAuthor) that starts with the mark: + anyone can post a copy of it on a public repository.""" + pr = self.view(n) + if not pr: + print(f'#{n}: could not read it to say why it waits') + return + mark = f'required check {check} {what} on {(pr.get("headRefOid") or "?")[:12]}' + told = f'pr_queue: {mark},' + if any(c.get('viewerDidAuthor') is True and (c.get('body') or '').startswith(told) + for c in pr.get('comments') or []): + return + also = '.' + if check == self.cfg.head_check and what == 'is red': + also = f', or if a new lens read or override covers this head: `{self.cfg.ready(n)}`.' + r = self.gh('pr', 'comment', str(n), '--repo', self.repo, '--body', + f'{told} so this armed PR waits and the queue passes over it. {fix}' + also) + if r.returncode != 0: + print(f'#{n}: could not comment: {r.stderr.strip()[:300]}') + # --- steps 2-4 -------------------------------------------------------------------------------------------------- def view(self, n): r = self.gh('pr', 'view', str(n), '--repo', self.repo, '--json', VIEW) @@ -517,10 +621,23 @@ def run(self, dry): n = p['number'] if p.get('isDraft') or not p.get('autoMergeRequest'): continue + bad = self.failing(p) + if bad: + print(f'skip #{n}: required check {bad} failed (its author fixes it)') + if not dry: + self.tell_red(n, bad) + continue if p.get('mergeStateStatus') != 'BEHIND': if p.get('mergeStateStatus') not in WAITS_FOR: print(f'skip #{n}: merge state {p.get("mergeStateStatus")}, not one the queue updates or waits for') continue + why = self.stuck(p) + if why: + print(f'skip #{n}: up to date, but required check {why[0]} {why[1]} and nothing re-runs it: it ' + 'cannot merge, so it does not hold the queue') + if not dry: + self.tell_red(n, why[0], why[1], self.stuck_fix(why[1], p.get('baseRefName') or 'its base')) + continue print(f'#{n} is up to date and waiting for its checks: queue is moving') return 0 print(f'update #{n} {(p.get("title") or "")[:70]}') diff --git a/fork-ci/actions/pr-queue/pr_queue.test.py b/fork-ci/actions/pr-queue/pr_queue.test.py index 91903cf6b71be..2977c5e870c0b 100644 --- a/fork-ci/actions/pr-queue/pr_queue.test.py +++ b/fork-ci/actions/pr-queue/pr_queue.test.py @@ -949,6 +949,251 @@ def test_a_cache_with_another_config_or_open_to_others_is_rebuilt(self): self.assertEqual(os.stat(self.q.cache).st_mode & 0o777, 0o700) +def own(comment): + """A PR comment for a fake `gh pr view`: a plain body is one the queue's own login posted.""" + return comment if isinstance(comment, dict) else {'body': comment, 'viewerDidAuthor': True} + + +class RedChecks(unittest.TestCase): + """Only the newest run of a check counts, and an armed PR the queue skips says why on the PR.""" + H = 'd4' * 20 + + def setUp(self): + d = tempfile.mkdtemp(prefix='pr-queue-red-') + self.addCleanup(shutil.rmtree, d, True) + self.q = make_queue(d) + + @staticmethod + def rollup(*runs, name='review-check'): + return [{'__typename': 'CheckRun', 'workflowName': 'ci', 'name': name, 'startedAt': s, 'conclusion': c} + for s, c in runs] + + def failing(self, *runs, name='review-check'): + return self.q.failing({'statusCheckRollup': self.rollup(*runs, name=name)}) + + def test_an_older_red_under_a_newer_green_is_not_failing(self): + self.assertIsNone(self.failing(('2026-11-09T01:00:00Z', 'FAILURE'), ('2026-11-09T02:00:00Z', 'SUCCESS'))) + + def test_a_newer_red_over_an_older_green_is_failing(self): + self.assertEqual(self.failing(('2026-11-09T02:00:00Z', 'FAILURE'), ('2026-11-09T01:00:00Z', 'SUCCESS')), + 'review-check') + + def test_a_newest_timed_out_run_is_failing_and_a_cancelled_one_is_not(self): + self.assertEqual(self.failing(('2026-11-09T02:00:00Z', 'TIMED_OUT')), 'review-check') + self.assertIsNone(self.failing(('2026-11-09T02:00:00Z', 'CANCELLED'))) + + def test_a_red_that_only_a_cancelled_run_followed_still_stands(self): + self.assertEqual(self.failing(('2026-11-09T01:00:00Z', 'FAILURE'), ('2026-11-09T02:00:00Z', 'CANCELLED')), + 'review-check') + + def test_a_green_that_a_cancelled_run_followed_is_not_red(self): + self.assertIsNone(self.failing(('2026-11-09T01:00:00Z', 'SUCCESS'), ('2026-11-09T02:00:00Z', 'CANCELLED'))) + + def test_a_check_that_is_not_required_is_ignored(self): + self.assertIsNone(self.failing(('2026-11-09T02:00:00Z', 'FAILURE'), name='lint')) + + def test_two_red_checks_name_the_first_by_name(self): + rollup = self.rollup(('2026-11-09T02:00:00Z', 'FAILURE'), name='test') + \ + self.rollup(('2026-11-09T02:00:00Z', 'FAILURE'), name='build') + self.assertEqual(self.q.failing({'statusCheckRollup': rollup}), 'build') + + def sweep(self, state='BEHIND', runs=(('2026-11-09T02:00:00Z', 'FAILURE'),), comments=(), head=None, dry=False, + fail=(), name='review-check'): + self.posted, self.updated = [], [] + listed = [{'number': 7, 'title': 't', 'isDraft': False, 'autoMergeRequest': {'m': 1}, 'mergeStateStatus': state, + 'createdAt': '2026-11-07T00:00:00Z', 'statusCheckRollup': self.rollup(*runs, name=name)}] + + def gh(*a): + if a[:2] == ('pr', 'list'): + return ok(json.dumps(listed)) + if a[:2] == ('pr', 'view'): + if 'view' in fail: + return failed('boom') + return ok(json.dumps({'headRefOid': head or self.H, 'comments': [own(c) for c in comments]})) + if a[:2] == ('pr', 'comment'): + self.posted.append(a[-1]) + return failed('boom') if 'comment' in fail else ok() + raise AssertionError(a) + self.q.gh = gh + self.q.update = lambda n, d: self.updated.append(n) or 0 + return run_quiet(self.q.run, dry)[0] + + def test_a_red_armed_pr_is_told_whether_or_not_it_is_behind(self): + for state in ('BEHIND', 'BLOCKED', 'UNSTABLE'): + with self.subTest(state=state): + self.sweep(state) + self.assertEqual((len(self.posted), self.updated), (1, [])) + self.assertIn(f'required check review-check is red on {self.H[:12]}', self.posted[0]) + + def test_a_red_then_cancelled_armed_pr_is_told_and_does_not_hold_the_queue(self): + runs = (('2026-11-09T01:00:00Z', 'FAILURE'), ('2026-11-09T02:00:00Z', 'CANCELLED')) + for state in ('BLOCKED', 'BEHIND'): + with self.subTest(state=state): + self.sweep(state, runs=runs) + self.assertEqual((len(self.posted), self.updated), (1, [])) + + def test_the_message_promises_no_re_run(self): # nothing in the queue re-runs a red check + for name in ('review-check', 'build'): + with self.subTest(name=name): + self.sweep(name=name) + self.assertNotIn('re-run', self.posted[0]) + + def test_only_the_head_check_is_pointed_at_the_ready_command(self): # the lenses are about the head check alone + self.sweep() + self.assertIn('`tool ready 7`', self.posted[0]) + self.sweep(name='build') + self.assertNotIn('tool ready', self.posted[0]) + + def test_it_is_told_once_per_head(self): + self.sweep() + first = self.posted[0] + self.sweep(comments=[first]) + self.assertEqual(self.posted, []) + self.sweep(comments=[first], head='e5' * 20) + self.assertEqual(len(self.posted), 1, 'a new head that is red is a new message') + + def test_only_the_queues_own_comment_counts_as_told(self): # anyone can post a copy of the mark + self.sweep() + first = self.posted[0] + for copy in ({'body': first, 'viewerDidAuthor': False}, {'body': first}, + {'body': '> ' + first, 'viewerDidAuthor': True}): + with self.subTest(copy=copy): + self.sweep(comments=[copy]) + self.assertEqual(self.posted, [first], 'a copy of the mark silenced the queue') + + def test_a_dry_run_posts_nothing(self): + self.sweep(dry=True) + self.assertEqual(self.posted, []) + + def test_a_failed_comment_or_read_does_not_stop_the_queue(self): + for fail in ('comment', 'view'): + with self.subTest(fail=fail): + self.assertEqual(self.sweep(fail=(fail,)), 0) + + def test_an_older_red_no_longer_skips_the_pr(self): + self.sweep(runs=(('2026-11-09T01:00:00Z', 'FAILURE'), ('2026-11-09T02:00:00Z', 'SUCCESS'))) + self.assertEqual((self.posted, self.updated), ([], [7])) + + def test_the_message_can_never_count_as_a_lens(self): + self.sweep() + post = {'body': self.posted[0], 'author': {'login': 'owner'}} + self.assertIsNone(REVIEW.lens_problem(post)) + self.assertEqual(REVIEW.reviewed_head({'comments': [post], 'reviews': []}), (False, None, 'no lens comments')) + + +class StuckUpToDate(unittest.TestCase): + """An armed, up-to-date PR whose required check ended without passing (cancelled) or never reported cannot merge, + and nothing re-runs that check. It is told once per head and passed over; it does not hold the queue.""" + H1, H2 = 'e1' * 20, 'f2' * 20 + + def setUp(self): + d = tempfile.mkdtemp(prefix='pr-queue-stuck-') + self.addCleanup(shutil.rmtree, d, True) + self.q = make_queue(d) + + @staticmethod + def checks(**states): # every required check green, except the ones named (None: the check never reported) + out = [] + for name in sorted(CFG.required_checks): + state = states.get(name.replace('-', '_'), 'SUCCESS') + if state is None: + continue + run = {'__typename': 'CheckRun', 'workflowName': 'ci', 'name': name, 'startedAt': '2026-11-11T01:00:00Z', + 'status': 'COMPLETED', 'conclusion': state} + if state == 'IN_PROGRESS': + run.update(status='IN_PROGRESS', conclusion='') + out.append(run) + return out + + def sweep(self, first, comments=(), runs='[]', runs_fail=False, dry=False): + """#1 (oldest): up to date (BLOCKED) with rollup `first`; #2: BEHIND and green.""" + self.posted, self.updated, self.asked = [], [], [] + listed = [{'number': 1, 'title': 't', 'isDraft': False, 'autoMergeRequest': {'m': 1}, 'mergeStateStatus': 'BLOCKED', + 'createdAt': '2026-11-07T00:00:01Z', 'statusCheckRollup': first, 'headRefOid': self.H1, + 'baseRefName': 'main'}, + {'number': 2, 'title': 't', 'isDraft': False, 'autoMergeRequest': {'m': 1}, 'mergeStateStatus': 'BEHIND', + 'createdAt': '2026-11-07T00:00:02Z', 'statusCheckRollup': self.checks(), 'headRefOid': self.H2, + 'baseRefName': 'main'}] + + def gh(*a): + if a[:2] == ('pr', 'list'): + return ok(json.dumps(listed)) + if a[:2] == ('pr', 'view'): + return ok(json.dumps({'headRefOid': self.H1, 'comments': [own(c) for c in comments]})) + if a[:2] == ('pr', 'comment'): + self.posted.append(a[-1]) + return ok() + if a[:2] == ('run', 'list'): + self.asked.append(a[a.index('--commit') + 1]) + return failed('boom') if runs_fail else ok(runs) + raise AssertionError(a) + self.q.gh = gh + self.q.update = lambda n, d: self.updated.append(n) or 0 + return run_quiet(self.q.run, dry) + + def test_a_cancelled_required_check_is_told_and_passed_over(self): + code, out = self.sweep(self.checks(build='CANCELLED')) + self.assertEqual((code, self.updated, len(self.posted)), (0, [2], 1), out) + self.assertIn(f'required check build ended CANCELLED on {self.H1[:12]}', self.posted[0]) + self.assertIn('Re-run failed jobs', self.posted[0]) + self.assertNotIn('tool ready', self.posted[0]) + + def test_a_rerun_tells_it_once_per_head_and_still_moves_on(self): + self.sweep(self.checks(review_check='CANCELLED')) + first = self.posted[0] + self.sweep(self.checks(review_check='CANCELLED'), comments=[first]) + self.assertEqual((self.posted, self.updated), ([], [2])) + self.assertNotIn('tool ready', first, 'a cancelled head check needs a re-run, not new lenses') + + def test_other_ended_states_count_too(self): # check_state's rule: only an unfinished check is running + for state in ('STARTUP_FAILURE', 'STALE', 'ACTION_REQUIRED'): + with self.subTest(state=state): + self.sweep(self.checks(test=state)) + self.assertEqual(self.updated, [2]) + self.assertIn(f'ended {state}', self.posted[0]) + self.assertIn('Re-run failed jobs', self.posted[0]) + + def test_a_re_run_of_the_cancelled_check_holds_the_queue_again(self): # the newest run counts: it is running + rerun = self.checks(build='CANCELLED') + [{'__typename': 'CheckRun', 'workflowName': 'ci', 'name': 'build', + 'status': 'QUEUED', 'conclusion': ''}] + code, out = self.sweep(rerun) + self.assertEqual((code, self.updated, self.posted), (0, [], []), out) + self.assertIn('#1 is up to date and waiting', out) + + def test_green_or_running_checks_still_hold_the_queue(self): + for first in (self.checks(), self.checks(review_check='IN_PROGRESS'), []): + with self.subTest(first=len(first)): + code, out = self.sweep(first) + self.assertEqual((code, self.updated, self.posted, self.asked), (0, [], [], []), out) + + def test_a_required_check_that_never_reports_is_told_once_nothing_runs(self): + code, out = self.sweep(self.checks(test=None), runs='[{"status": "completed", "workflowName": "ci"}]') + self.assertEqual((self.asked, self.updated, len(self.posted)), ([self.H1], [2], 1), out) + self.assertIn('required check test never reported', self.posted[0]) + + def test_a_check_that_never_reported_is_not_told_to_re_run(self): # the source told it to re-run + # A re-run repeats the jobs of the workflow files the head was built with: a check added to the workflows + # after that never runs on a re-run. Only a push or a merge of the base runs the current workflows. + self.sweep(self.checks(test=None), runs='[{"status": "completed", "workflowName": "ci"}]') + told = self.posted[0] + self.assertNotIn('Re-run failed jobs', told, 'a re-run cannot add a check its workflow files do not have') + self.assertIn('A re-run cannot add it', told) + self.assertIn('`tool push`', told) + self.assertIn('merge main into it', told) + + def test_a_missing_check_waits_while_a_workflow_runs_or_when_runs_are_unknown(self): + for kw in ({'runs': '[{"status": "in_progress", "workflowName": "ci"}]'}, {'runs_fail': True}, {'runs': 'x'}): + with self.subTest(**kw): + code, out = self.sweep(self.checks(test=None), **kw) + self.assertEqual((code, self.updated, self.posted), (0, [], []), out) + self.assertIn('is unknown: waiting', out, 'an unknown answer is said, not treated as "nothing runs"') + + def test_a_dry_run_posts_nothing_and_says_what_it_skips(self): + code, out = self.sweep(self.checks(build='CANCELLED'), dry=True) + self.assertEqual((self.posted, self.updated), ([], [2])) + self.assertIn('skip #1: up to date, but required check build ended CANCELLED', out) + + # --- the source project's lens-review cases for the queue ------------------------------------------------------------ HEAD = 'def wipe(player, island):\n' READ = HEAD + " if is_admin(player):\n log.info('wipe')\n wipe_island(island)\n" # only an admin wipes