fork: Fork tests on the repository's own fleet runners, chosen per run (CI_RUNNERS)

.github/fork-ci-tests.txt

@@ -7,6 +7,7 @@
 # is a note; one that was present and then deleted fails.
 .github/scripts/fork-ci-tests.test.cjs
 .github/scripts/fork-contracts.test.py
+.github/scripts/fork-fleet.test.py
 .github/scripts/fork-follow.test.sh
 .github/scripts/fork-release-workflow.test.py
 .github/scripts/fork-release.test.sh

.github/fork-fleet.json

@@ -0,0 +1,60 @@
+{
+  "_doc": [
+    "Where fork-tests.yml's Linux jobs run: this repository's own fleet of self-hosted runners, or GitHub-hosted.",
+    "The one place for every fact about this repository and its fleet: .github/scripts/fork-fleet.py has none.",
+    "Read by `route` (a step of fork-tests.yml's changes job), `watch` (fork-tests.yml), `sweep` (fork-fleet.yml),",
+    "and on each fleet host by `dispatch` and `gate` (an installed copy of main's script and of this file).",
+    "The setting is the repository variable named in `variable`: auto (unset too), fleet or github.",
+    "Runbook, owner steps and the failure table: .github/fork-fleet.md."
+  ],
+  "repository": "scuttletoby/t3code",
+  "variable": "CI_RUNNERS",
+  "hosted": "ubuntu-24.04",
+  "trusted": {
+    "workflows": [".github/workflows/fork-tests.yml"],
+    "actors": ["scuttletoby"],
+    "push_branches": ["main"]
+  },
+  "upstream_sync": {
+    "_doc": [
+      "Upstream's code runs on GitHub-hosted runners only (owner decision). The daily sync pushes main with the owner's",
+      "token, so its push run would pass `trusted`: scripts/fork-follow-push.sh tags each commit it moves main to as",
+      "<tag_prefix><sha> before the move, and a push run of a tagged commit never uses the fleet. Keep the tags: a",
+      "run can be re-run for 30 days."
+    ],
+    "tag_prefix": "fork-follow/synced/"
+  },
+  "pools": [
+    {
+      "_doc": "The owner's PC: its dispatcher runs on the always-on host and starts runners on the PC over ssh.",
+      "name": "pc",
+      "label": "fleet-pc-t3code",
+      "slots": 1,
+      "beacon": "FLEET_BEACON_PC",
+      "launch": ["/opt/fork-fleet/launch-pc"],
+      "waits": false
+    },
+    {
+      "_doc": "The always-on host (the owner's workspace, not a VM of its own): each runner runs as its own user in its own network namespace with internet only (fork-fleet.md, Containment). Fleet mode queues here.",
+      "name": "vm",
+      "label": "fleet-vm-t3code",
+      "slots": 2,
+      "beacon": "FLEET_BEACON_VM",
+      "launch": ["/opt/fork-fleet/fork-fleet-launch.sh"],
+      "waits": true
+    }
+  ],
+  "beacon_fresh_seconds": 180,
+  "stuck_seconds": 600,
+  "fleet_stuck_seconds": 7200,
+  "dispatch": { "every_seconds": 20, "beacon_seconds": 60, "idle_passes": 3 },
+  "jobs": [
+    { "job": "test", "shards": 2, "gates": ["test"], "memory_mb": 4096 },
+    { "job": "test_web", "shards": 2, "gates": ["test_web"], "memory_mb": 4096 },
+    { "job": "build", "gates": ["build", "typecheck"], "memory_mb": 4096 },
+    { "job": "lint", "gates": [], "memory_mb": 3072 },
+    { "job": "rust", "gates": ["rust"], "memory_mb": 2048 },
+    { "job": "test_server", "shards": 8, "gates": ["test_server"], "memory_mb": 3072 },
+    { "job": "release_smoke", "gates": ["release_smoke"], "memory_mb": 2048 }
+  ]
+}

.github/scripts/fork-fleet-hook.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Fork only: every fleet runner's job-started hook (ACTIONS_RUNNER_HOOK_JOB_STARTED, which fork-fleet-launch.sh sets
+# from a root-only file). Installed root-owned next to root-owned copies of main's fork-fleet.py and fork-fleet.json;
+# the runner user can change none of them. `gate` kills the job's Runner.Worker when the job may not run here.
+exec /usr/bin/python3 -I "$(dirname -- "$0")/fork-fleet.py" --config "$(dirname -- "$0")/fork-fleet.json" gate

.github/scripts/fork-follow-push.sh

@@ -6,8 +6,11 @@
 #   check            Fails unless FORK_PUSH_TOKEN is set, accepted by GitHub and may push here.
 #   candidate SHA    Force-pushes SHA as the branch fork-follow/candidate.
 #   advance PREV SHA Moves SERIES_BRANCH from PREV to SHA with a lease. First keeps PREV as the tag
-#                    fork-follow/pre-<BASE_TAG>-<PREV, 12 hex>, so the old series stays reachable.
-#                    Already at SHA (a re-run) is success; moved elsewhere is a refusal.
+#                    fork-follow/pre-<BASE_TAG>-<PREV, 12 hex>, so the old series stays reachable,
+#                    then tags SHA as fork-follow/synced/<SHA>: the push run of a tagged commit never
+#                    uses the fleet (upstream code runs GitHub-hosted only; tag_prefix in
+#                    .github/fork-fleet.json, which must say the same). Already at SHA (a re-run) tags
+#                    it and is success; moved elsewhere is a refusal.
 #   cleanup          Deletes the branch fork-follow/candidate; a failure is only a warning.
 #
 # Environment: FORK_PUSH_TOKEN, FORK_PUSH_ENVIRONMENT (the GitHub Environment that should hold the
@@ -25,6 +28,7 @@ SERIES_BRANCH="${SERIES_BRANCH:-main}"
 REPORT="${REPORT:-fork-follow-report.md}"
 OUTPUT="${GITHUB_OUTPUT:-/dev/stdout}"
 CANDIDATE_BRANCH=fork-follow/candidate
+SYNCED_TAG_PREFIX=fork-follow/synced/
 err="$(mktemp)"
 trap 'rm -f "$err"' EXIT
 
@@ -83,13 +87,20 @@ cmd_candidate() {
   echo "Pushed ${sha:0:12} to $CANDIDATE_BRANCH."
 }
 
+tag_synced() { # tag_synced SHA OUTCOME: the tag keeping the push run of SHA off the fleet, made before main is SHA
+  git push "$PUSH_URL" "$1:refs/tags/$SYNCED_TAG_PREFIX$1" > /dev/null 2> "$err" \
+    || die "Could not tag ${1:0:12} as $SYNCED_TAG_PREFIX$1, $2." \
+      "Without the tag, the push run of a synced commit may use the fleet; upstream code runs on GitHub-hosted runners only. git said:" "$(git_said)"
+}
+
 cmd_advance() {
   local prev="${1:?PREV}" sha="${2:?SHA}" now backup
   : "${BASE_TAG:?BASE_TAG}"
   with_token
   now="$(remote_ref "refs/heads/$SERIES_BRANCH")" \
     || die "Could not read $SERIES_BRANCH of $REPOSITORY, so it was not moved. git said:" "$(git_said)"
   if [[ "$now" == "$sha" ]]; then
+    tag_synced "$sha" "so a re-run of its push run could use the fleet" # an earlier run moved it
     echo "moved=already" >> "$OUTPUT"
     echo "$SERIES_BRANCH is already at ${sha:0:12}."
     return 0
@@ -101,6 +112,7 @@ cmd_advance() {
   git push "$PUSH_URL" "$prev:refs/tags/$backup" > /dev/null 2> "$err" \
     || die "Could not keep the old series ${prev:0:12} as tag $backup, so $SERIES_BRANCH was not moved. git said:" "$(git_said)"
   echo "backup_tag=$backup" >> "$OUTPUT"
+  tag_synced "$sha" "so $SERIES_BRANCH was not moved"
   git push --force-with-lease="refs/heads/$SERIES_BRANCH:$prev" "$PUSH_URL" "$sha:refs/heads/$SERIES_BRANCH" > /dev/null 2> "$err" \
     || die "Could not move $SERIES_BRANCH from ${prev:0:12} to ${sha:0:12}." \
       "If a rule protects $SERIES_BRANCH, the token's user must be allowed to bypass it. The old series is kept as tag $backup. git said:" "$(git_said)"

.github/scripts/fork-follow.test.sh

@@ -418,11 +418,25 @@ check "push: a refused candidate push fails with git's reason" \
 rm -f "$hook"
 push candidate "$next"
 check "push: the candidate goes to fork-follow/candidate" eval '[[ "$(series fork-follow/candidate)" == "$next" ]] && has candidate_pushed=true "$root/push.out"'
+# The synced tag keeps the push run of the synced commit off the fleet (fork-fleet.py made_by_the_sync), so
+# main may only move once it exists: a refused tag push leaves main where it was.
+synced_tag() { git -C "$root/gh/me/t3code.git" rev-parse -q --verify "refs/tags/fork-follow/synced/$1" 2> /dev/null; }
+printf '#!/bin/sh\nwhile read -r old new ref; do case "$ref" in refs/tags/fork-follow/synced/*) echo "no synced tags" >&2; exit 1;; esac; done\n' > "$hook" && chmod +x "$hook"
+rm -f "$root/report.md"
+check "push: advance refuses to move the series when the synced tag cannot be pushed" \
+  eval '! push advance "$prev" "$next" && [[ "$(series)" == "$prev" ]] && has "Could not tag ${next:0:12} as fork-follow/synced/$next, so main was not moved" "$root/report.md" && has "no synced tags" "$root/report.md"'
+printf '#!/bin/sh\nwhile read -r old new ref; do echo "$ref" >> "%s"; done\n' "$root/pushed-refs" > "$hook"
+rm -f "$root/pushed-refs"
 push advance "$prev" "$next"
 check "push: advance moves the series and keeps the old one as a tag" \
   eval '[[ "$(series)" == "$next" && "$(git -C "$root/gh/me/t3code.git" rev-parse "refs/tags/fork-follow/pre-v0.1.10-${prev:0:12}")" == "$prev" ]]'
+check "push: advance tags the new series fork-follow/synced/<sha> before it moves the series" \
+  eval '[[ "$(synced_tag "$next")" == "$next" ]] && [[ "$(grep -n "" "$root/pushed-refs" | grep -E "refs/tags/fork-follow/synced/$next\$|refs/heads/main\$" | cut -d: -f2 | tr "\n" " ")" == "refs/tags/fork-follow/synced/$next refs/heads/main " ]]'
+rm -f "$hook"
+git -C "$root/gh/me/t3code.git" tag -d "fork-follow/synced/$next" > /dev/null
 push advance "$prev" "$next"
 check "push: advance again after it moved (re-run) succeeds without a push" has moved=already "$root/push.out"
+check "push: advance again after it moved (re-run) makes a missing synced tag" eval '[[ "$(synced_tag "$next")" == "$next" ]]'
 work git commit -q --allow-empty -m "fork: merged meanwhile"
 work git push -q origin HEAD:main
 moved="$(series)"

.github/workflows/fork-fleet.yml

@@ -0,0 +1,55 @@
+name: Fork fleet sweep
+
+# Fork only. In auto and github mode (repository variable CI_RUNNERS; unset is
+# auto), a trusted Fork tests run that the fleet did not finish runs again,
+# GitHub-hosted:
+# - a fleet job queued 10 minutes (stuck_seconds) without a runner: the run is
+#   cancelled (if fork-tests.yml's fleet_watch has not done it already) and
+#   re-run;
+# - a run that ended red because a fleet runner stopped during a job (no failed
+#   step of the job's own): re-run. A run a person cancelled stays cancelled.
+# Only trusted runs are touched, by the same rule as the dispatcher's (GitHub's
+# record of the run, each pull request read from the API), never a stranger's.
+# The sweep's re-run is triggered by github-actions, which is not a trusted
+# actor: it is GitHub-hosted, and the sweep never touches it, so each run is
+# re-run once at most. Nothing is kept between sweeps: each reads the runs of
+# the last 24 hours (the latest attempt of each) from GitHub, so a sweep that
+# stopped half-way is finished by the next. Fleet mode re-runs nothing: it only
+# cancels, with an error, a trusted run whose fleet job waited 2 hours
+# (fleet_stuck_seconds; a re-run of failed jobs has no fleet_watch of its own).
+# .github/scripts/fork-fleet.py sweep; runbook .github/fork-fleet.md.
+on:
+  schedule:
+    - cron: "*/10 * * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: fork-fleet-sweep
+  cancel-in-progress: false
+
+jobs:
+  sweep:
+    name: Sweep
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    permissions:
+      actions: write
+      contents: read
+      pull-requests: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          sparse-checkout: |
+            /.github/fork-fleet.json
+            /.github/scripts/fork-fleet.py
+          sparse-checkout-cone-mode: false
+
+      - name: Cancel and re-run what the fleet did not finish
+        env:
+          FLEET_MODE: ${{ vars.CI_RUNNERS }}
+          GITHUB_TOKEN: ${{ github.token }}
+        run: python3 -I .github/scripts/fork-fleet.py sweep

.github/workflows/fork-release.yml

@@ -146,6 +146,13 @@ jobs:
   tests:
     name: Tests
     needs: [preflight]
+    # A called workflow's job may not ask for more than its caller grants, even a job that is
+    # skipped: fork-tests.yml's fleet_watch asks for actions: write (it cancels a run stuck waiting
+    # for the fleet; never in a release run, which is GitHub-hosted). fork-tests.yml's own
+    # top-level permissions keep its other jobs at contents: read.
+    permissions:
+      actions: write
+      contents: read
     uses: ./.github/workflows/fork-tests.yml
     with:
       ref: ${{ needs.preflight.outputs.ref }}