diff --git a/.env.example b/.env.example index b3e3658..f63588a 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,6 @@ # Copy to .env and fill in. Never commit .env. -# Variables interpolated by docker-compose.yml for the 3-runner X64 layout. +# Variables interpolated by docker-compose.yml. Per-host examples with the +# target labels live in hosts/ (hetzner, beelink, rpi5). # GitHub App that registers the runners (required) APP_ID= @@ -9,10 +10,18 @@ APP_LOGIN= # Optional — values match the compose defaults RUNNER_NAME_PREFIX=runner -LABELS=self-hosted,Linux,X64,beelink,docker,large +LABELS=self-hosted,Linux,X64,docker,x86,large,light,beelink RUNNER_GROUP=Default -DISABLE_AUTO_UPDATE=false -# EPHEMERAL is hard-coded to "false" in docker-compose.yml so the three -# runners stay persistent and keep their per-runner cache volumes. -# Setting it here has no effect. +# Runner self-update. The upstream entrypoint tests this with `[ -n ... ]`, +# so ANY non-empty value disables auto-update - "false" included. Leave it +# unset/empty to keep auto-update on; set it to e.g. "true" to turn it off. +# DISABLE_AUTO_UPDATE=true + +# EPHEMERAL is hard-coded to "true" in docker-compose.yml (each runner takes +# one job, then a fresh container registers again; the caches live on the +# named volumes). Setting it here has no effect. + +# Nightly pruner (see pruner.sh) +# CACHE_CAP_BYTES=21474836480 +# PRUNE_INTERVAL_SECONDS=86400 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 282bc11..a8d47f2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -78,6 +78,8 @@ jobs: VERSION=${{ steps.versions.outputs.runner_tag }} JAVA_VERSION=${{ matrix.java_version }} NODE_VERSION=${{ matrix.node_version }} + RUNNER_IMAGE_TAG=pr-${{ github.event.pull_request.number }} + RUNNER_IMAGE_REVISION=${{ github.event.pull_request.head.sha }} gate: name: All builds passed diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index bc367e9..18ee23c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -73,6 +73,17 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 + # Image identity baked into the image as RUNNER_IMAGE_TAG / + # RUNNER_IMAGE_REVISION (Dockerfile ARG -> ENV). The tag is the fully + # pinned one, which never changes meaning; the revision is the commit. + - name: Docker metadata (image identity) + id: meta + uses: docker/metadata-action@v6 + with: + images: ghcr.io/${{ github.repository_owner }}/github-runner + tags: | + type=raw,value=${{ steps.versions.outputs.runner_version }}-jdk${{ matrix.java_version }}-sdk${{ steps.versions.outputs.compile_sdk }}-node${{ matrix.node_version }} + - name: Build and push by digest id: build uses: docker/build-push-action@v7 @@ -89,6 +100,8 @@ jobs: VERSION=${{ steps.versions.outputs.runner_tag }} JAVA_VERSION=${{ matrix.java_version }} NODE_VERSION=${{ matrix.node_version }} + RUNNER_IMAGE_TAG=${{ steps.meta.outputs.version }} + RUNNER_IMAGE_REVISION=${{ github.sha }} outputs: type=image,name=ghcr.io/${{ github.repository_owner }}/github-runner,push-by-digest=true,name-canonical=true,push=true - name: Export digest diff --git a/Dockerfile b/Dockerfile index 26471d6..d90f1a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,11 +101,28 @@ LABEL maintainer="asen.lekov@dodi.li" # the only other option, `safe.directory=*`, turns the check off for everything. ENV TAR_OPTIONS=--no-same-owner +# Tool caches that outlive a job. Jobs run as root (HOME=/root), so these are +# the tools' own defaults made explicit: docker-compose.yml mounts one named +# volume on each of them. Mount the specific directories, never ~/.cache as a +# whole, or it would shadow the pnpm volume at ~/.cache/pnpm. +# - mise: tool installs (data) and downloads (cache). jdx/mise-action and +# `mise install` fill these. +# - hk: HK_CACHE_DIR is its config/tool cache. HK_PKL_CACHE_DIR defaults to +# ~/.cache/pklr, a sibling of ~/.cache/hk, so it is pinned underneath +# HK_CACHE_DIR to live on the same volume. +ENV MISE_DATA_DIR=/root/.local/share/mise \ + MISE_CACHE_DIR=/root/.cache/mise \ + HK_CACHE_DIR=/root/.cache/hk \ + HK_PKL_CACHE_DIR=/root/.cache/hk/pklr + # Job hooks: JOB_STARTED is the poison sweep (covers OOM / killed runner); # JOB_COMPLETED is the tidy-up. Neither wipes package / Gradle caches. COPY pre-job.sh /usr/local/bin/pre-job.sh COPY cleanup.sh /usr/local/bin/cleanup.sh -RUN chmod +x /usr/local/bin/pre-job.sh /usr/local/bin/cleanup.sh +# pruner.sh is not a hook: the `pruner` compose service runs it nightly +# (LRU cache caps + docker prune), keeping those scans off the job path. +COPY pruner.sh /usr/local/bin/pruner.sh +RUN chmod +x /usr/local/bin/pre-job.sh /usr/local/bin/cleanup.sh /usr/local/bin/pruner.sh ENV ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/pre-job.sh ENV ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/cleanup.sh @@ -117,6 +134,15 @@ ENV ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/cleanup.sh # registration happens instead. See preflight-entrypoint.sh for details. COPY preflight-entrypoint.sh /usr/local/bin/preflight-entrypoint.sh RUN chmod +x /usr/local/bin/preflight-entrypoint.sh +# Image identity. deploy.yml passes the image tag and the commit it was built +# from; jobs read them (e.g. the runner report) to tell which image they ran +# on. They default to "dev" for local builds. Declared last: the values change +# on every build, so anything after them would never hit the layer cache. +ARG RUNNER_IMAGE_TAG=dev +ARG RUNNER_IMAGE_REVISION=dev +ENV RUNNER_IMAGE_TAG=$RUNNER_IMAGE_TAG \ + RUNNER_IMAGE_REVISION=$RUNNER_IMAGE_REVISION + ENTRYPOINT ["/usr/local/bin/preflight-entrypoint.sh"] # Declaring ENTRYPOINT resets the CMD inherited from the base image, so the # base image's CMD is restated here. See diff --git a/README.md b/README.md index 727228e..c74c492 100644 --- a/README.md +++ b/README.md @@ -17,39 +17,142 @@ Linux hosts run natively without emulation. To run the container, check out the environment variables from the base image: https://github.com/myoung34/docker-github-actions-runner#environment-variables -[`docker-compose.yml`](docker-compose.yml) is the X64 beelink layout — three -persistent runners, each with its own work **and** cache volumes — not a -side file. After pulling a new image on the host: +[`docker-compose.yml`](docker-compose.yml) is the X64 layout (beelink by +default; hetzner uses the same file with its own `.env`) — three ephemeral +runners, each with its own work **and** cache volumes, plus a nightly +`pruner`. Per-host `.env` examples with the target labels are in +[`hosts/`](hosts/). After pulling a new image on a host: ``` docker compose pull && docker compose up -d ``` Existing named `runner-work*` volumes keep their data; new cache volumes -start empty and warm on first jobs. Do not share one bun/gradle volume +start empty and warm on first jobs. Do not share one bun/gradle/mise volume across the three containers: concurrent jobs would corrupt the same tree. +## Runner pools and labels + +Jobs pick runners by capability label, never by machine name. Each host's +`.env` sets `LABELS` (the runner binary adds `self-hosted`, the OS and the +architecture itself): + +| Pool | `.env` example | `LABELS` | Caches | +|------|----------------|----------|--------| +| hetzner (X64) | [`hosts/hetzner.env.example`](hosts/hetzner.env.example) | `self-hosted,Linux,X64,docker,x86,light,agent,hetzner` | all volumes | +| beelink (X64) | [`hosts/beelink.env.example`](hosts/beelink.env.example) | `self-hosted,Linux,X64,docker,x86,large,light,beelink` | all volumes | +| rpi5 (ARM64) | [`hosts/rpi5.env.example`](hosts/rpi5.env.example) | `self-hosted,Linux,ARM64,docker,agent,rpi5` | work only | + +| Capability label | Meaning | +|------------------|---------| +| `light` | default checks, release, deploy steps (X64 only: no rpi5) | +| `large` | heavy jobs: Gradle/Android, Next.js builds (beelink) | +| `agent` | review, triage, assist (hetzner and rpi5) | + +The rpi5 pool is an agent/utility pool: it has no `light` label and must not +mount the heavy cache volumes. Its `.env` example sets +`COMPOSE_FILE=docker-compose.yml:hosts/rpi5.compose.yml`, which layers +[`hosts/rpi5.compose.yml`](hosts/rpi5.compose.yml) over the base file so each +runner keeps only `docker.sock` and its work volume (and the pruner only +prunes Docker). + +### Rolling out a label change + +Runners are ephemeral and re-register on their next job, so changing labels is +a `.env` edit plus a recreate: + +1. Copy the host's `hosts/.env.example` values into its `.env` (keep the + real `APP_*` values). On rpi5 also keep the `COMPOSE_FILE` line. +2. `docker compose pull && docker compose up -d`. +3. Check the pool in the org's runner list (Settings, Actions, Runners): each + runner should show the labels above. `docker compose ps` shows `healthy` + once `Runner.Listener` is up. + +## Ephemeral runners + +`EPHEMERAL` is `"true"` in the compose file. The upstream entrypoint +(`myoung34/github-runner`) only tests `[ -n "${EPHEMERAL}" ]` (and the same for +`DISABLE_AUTO_UPDATE`), so any non-empty value, `"false"` included, passes +`--ephemeral` to `config.sh`. The previous `EPHEMERAL: "false"` therefore +never kept runners persistent: they have always been ephemeral, one job per +container, restarted by `restart: unless-stopped`. The compose file now says +what actually happens. Each job gets a clean container; the caches survive on +the named volumes below. + +`DISABLE_AUTO_UPDATE` is empty by default. Set it to any non-empty value in +`.env` to disable runner self-update (`false` disables it too). + +## Health check + +Each runner reports healthy while `Runner.Listener` is alive +(`pgrep -f Runner.Listener`, exec form: under `CMD-SHELL` the wrapping +`sh -c` would match its own command line). The container restarts after each +ephemeral job; `start_period` (120 s) keeps the re-registration from counting +as failures. + +## Image identity + +The image carries `RUNNER_IMAGE_TAG` (the fully pinned tag, e.g. +`2.338.0-jdk21-sdk36-node24`) and `RUNNER_IMAGE_REVISION` (the commit it was +built from) as environment variables, set from build args by `deploy.yml` +(`build.yml` sets `pr-` and the head SHA). Jobs can print them to tell +which image they ran on. Local builds default to `dev`. + ## X64 cache volumes -This compose is for the **X64** class (`beelink`, `docker`, `large`). System -labels (`Linux`, `X64`) are still applied by the runner binary; set -`LABELS` in `.env` to `beelink,docker,large` (do not add a `build` label). -`EPHEMERAL` stays `false` so the volumes survive between jobs. +This compose is for the **X64** class (`beelink`, `hetzner`). The `LABELS` +default is the beelink set; hosts override it in `.env` (see the table above). +The runners are ephemeral, but the volumes survive between containers and +jobs. Each runner gets its own: -| Mount | Volume (runner / runner-2 / runner-3) | Why | -|-------|----------------------------------------|-----| -| `/tmp/runner/work` | `runner-work*` | Job workspace. Cleanup spares `$RUNNER_WORKDIR`. | -| `/root/.bun` | `runner-bun*` | Bun install cache | -| `/root/.npm` | `runner-npm*` | npm cache | -| `/root/.cache/pnpm` | `runner-pnpm*` | pnpm store | -| `/root/.gradle` | `runner-gradle*` | Gradle modules, build-cache, transforms | -| `/root/.cargo` | `runner-cargo*` | Cargo registry | - -Budget ~20 GB per runner (enforced by the post-job hook with LRU, not a -wipe), ~60 GB per beelink. ARM64 / Pi hosts should **not** mount these -cache volumes — those runners isolate to `RUNNER_TEMP`. +| Mount | Volume (runner / runner-2 / runner-3) | Why | Capped by pruner | +|-------|----------------------------------------|-----|------------------| +| `/tmp/runner/work` | `runner-work*` | Job workspace. Cleanup spares `$RUNNER_WORKDIR`. | no | +| `/root/.bun` | `runner-bun*` | Bun install cache | yes | +| `/root/.npm` | `runner-npm*` | npm cache | no | +| `/root/.cache/pnpm` | `runner-pnpm*` | pnpm store | no | +| `/root/.gradle` | `runner-gradle*` | Gradle modules, build-cache, transforms | yes | +| `/root/.cargo` | `runner-cargo*` | Cargo registry | no | +| `/root/.rustup` | `runner-rustup*` | Rust toolchains (issue #8) | no | +| `/root/.konan` | `runner-konan*` | Kotlin/Native toolchain and dependencies | no | +| `/root/.local/share/mise` | `runner-mise-data*` | mise tool installs (`MISE_DATA_DIR`) | yes, whole tool versions | +| `/root/.cache/mise` | `runner-mise-cache*` | mise downloads (`MISE_CACHE_DIR`) | yes | +| `/root/.cache/hk` | `runner-hk*` | hk cache, incl. pkl (`HK_CACHE_DIR`, `HK_PKL_CACHE_DIR`) | yes | + +The Dockerfile sets `MISE_DATA_DIR`, `MISE_CACHE_DIR`, `HK_CACHE_DIR` and +`HK_PKL_CACHE_DIR` so the tools write where the volumes are mounted. The +`~/.cache` directory is never mounted as a whole: that would shadow the pnpm +volume, so each subdirectory has its own volume. `jdx/mise-action` should use +`cache: false` on these runners; the volumes replace its GitHub-cache step. + +Budget ~20 GB per capped cache directory (enforced by the nightly pruner with +LRU, not a wipe). The remaining volumes (npm, pnpm, cargo, rustup, konan) are +not capped yet (issue #7 tracks the bun/gradle-only caps; hard-linked stores +need care). ARM64 / Pi hosts should **not** mount these cache volumes — those +runners isolate to `RUNNER_TEMP`, see `hosts/rpi5.compose.yml`. + +## Maintenance: per-job cleanup and the nightly pruner + +`cleanup.sh` (the `JOB_COMPLETED` hook) only does the cheap tidy-up after each +job: `/tmp` outside the workdir, Gradle daemon/journals/locks, runner `_diag` +logs. The expensive scans used to run there on every job and cost 21–39 s each +(issue #16). They now run once a day in the `pruner` compose service, which +uses the same image with [`pruner.sh`](pruner.sh) as its entrypoint and +mounts only the capped cache volumes plus `docker.sock`: + +- LRU size caps (`CACHE_CAP_BYTES`, default 20 GiB) on gradle, bun, the mise + cache and the hk cache, evicting by last access time; mise installs are + evicted as whole `tool/version` directories so a tool is never left half + deleted (`MISE_DATA_CAP_BYTES`). +- `docker system prune -af --filter until=24h` +- `docker builder prune -af --filter until=72h` + +Named volumes are never pruned. The first run is 5 minutes after start +(`PRUNE_INITIAL_DELAY`), then every `PRUNE_INTERVAL_SECONDS` (86400). Run it +once by hand with +`docker compose run --rm -e PRUNE_RUN_ONCE=true pruner`. Do **not** tmpfs `/tmp`. `RUNNER_WORKDIR` is `/tmp/runner/work`; a tmpfs or a blanket `find /tmp -delete` hides or wipes the work volume. @@ -58,7 +161,8 @@ or a blanket `find /tmp -delete` hides or wipes the work volume. Named volumes are unused if the job relocates the caches. On these X64 self-hosted runners, workflows must **not** isolate `GRADLE_USER_HOME` or -`BUN_INSTALL_CACHE_DIR` (or `npm_config_cache` / `CARGO_HOME`) to +`BUN_INSTALL_CACHE_DIR` (or `npm_config_cache` / `CARGO_HOME` / +`MISE_DATA_DIR` / `MISE_CACHE_DIR` / `HK_CACHE_DIR`) to `$RUNNER_TEMP`. That is the `setup-stack` local-home mode in `dodi-smart/.github` — `deps-verify` stays isolated; `pr-checks` uses the default homes so the volumes are actually read. @@ -180,7 +284,7 @@ docker build -f Dockerfile . Build args can be used to customize the image. See the top of the [`Dockerfile`](Dockerfile) for the current defaults and available args (`VERSION`, `JAVA_VERSION`, `NODE_VERSION`, `COMPILE_SDK`, `BUILD_TOOLS`, -`NDK_VERSION`, `SDK_TOOLS`). +`NDK_VERSION`, `SDK_TOOLS`, `RUNNER_IMAGE_TAG`, `RUNNER_IMAGE_REVISION`). Example: ``` diff --git a/cleanup.sh b/cleanup.sh index 30aafe1..5fbfef3 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -2,87 +2,72 @@ # Post-job cleanup script for GitHub Actions runner. # Invoked automatically via ACTIONS_RUNNER_HOOK_JOB_COMPLETED after each job. # -# Persist package and Gradle/Cargo caches on the named volumes. Cap each -# runner's gradle-home and bun-home at ~20 GB with LRU eviction — do not -# wholesale wipe. JOB_STARTED (pre-job.sh) is the poison sweep for a killed -# runner process; this hook is the tidy-up when the job finishes normally. +# Persist package, Gradle/Cargo and mise/hk caches on the named volumes. This +# hook only does the CHEAP tidy-up that has to happen after every job. The +# expensive size-cap scans (du over multi-GB trees, LRU eviction) used to run +# here and cost 21-39 s per job (issue #16); they now live in pruner.sh, run +# nightly by the `pruner` compose service. JOB_STARTED (pre-job.sh) is the +# poison sweep for a killed runner process. # # Intentionally not using set -e: individual cleanup failures should not # prevent other cleanups from running. # # KEEP: $HOME/.bun, $HOME/.npm, $HOME/.cache/pnpm, $HOME/.cache/yarn, +# $HOME/.cache/mise, $HOME/.local/share/mise, $HOME/.cache/hk, +# $HOME/.konan, $HOME/.rustup, # Gradle modules-2 / build-cache-* / transforms-*, Cargo registry. # DELETE: /tmp contents except $RUNNER_WORKDIR (and $RUNNER_TEMP if under # /tmp), Gradle daemon/journals/locks, runner _diag logs. -CACHE_CAP_BYTES="${CACHE_CAP_BYTES:-21474836480}" # 20 GiB - echo "[cleanup] Running post-job cleanup..." BEFORE=$(df -h / | awk 'NR==2 {print $4}') -# True when $1 is $2, a descendant of $2, or an ancestor of $2. -_spared() { - local path="$1" - local keep="$2" - [ -z "$keep" ] && return 1 - case "$path" in - "$keep"|"$keep"/*) return 0 ;; - esac - case "$keep" in - "$path"/*) return 0 ;; - esac +# True when $1 is $2 or lives inside $2. +_inside() { + [ -n "$2" ] || return 1 + case "$1" in "$2"|"$2"/*) return 0 ;; esac + return 1 +} + +# True when $1 is a strict ancestor of $2. +_ancestor_of() { + [ -n "$2" ] || return 1 + case "$2" in "$1"/*) return 0 ;; esac return 1 } # RUNNER_WORKDIR is /tmp/runner/work on the beelinks. A blanket # `find /tmp -delete` wipes the work volume. Spare the workdir (and # RUNNER_TEMP if it sits under /tmp), plus their ancestors under /tmp. +# +# Walk top-down and never descend into a spared tree: the work volume holds +# checkouts plus node_modules, and `find -depth` (which disables -prune) used +# to walk every path in it on every job. Only strict ancestors of a spared +# path are entered, so the cost is the handful of entries outside the workdir. +clean_dir() { + local dir="$1" workdir="$2" temp="$3" path + while IFS= read -r -d '' path; do + if _inside "$path" "$workdir" || _inside "$path" "$temp"; then + continue + fi + if _ancestor_of "$path" "$workdir" || _ancestor_of "$path" "$temp"; then + clean_dir "$path" "$workdir" "$temp" + continue + fi + rm -rf "$path" 2>/dev/null || true + done < <(find "$dir" -mindepth 1 -maxdepth 1 -print0 2>/dev/null) +} + clean_tmp() { local workdir="${RUNNER_WORKDIR:-/tmp/runner/work}" local temp="${RUNNER_TEMP:-}" workdir="${workdir%/}" temp="${temp%/}" - if [ -n "$temp" ] && [ "$temp" != "/tmp" ] && [ "${temp#/tmp/}" = "$temp" ]; then - temp="" - fi - if [ "$temp" = "/tmp" ]; then - temp="" - fi + # Only a RUNNER_TEMP strictly under /tmp needs sparing. + case "$temp" in /tmp/*) ;; *) temp="" ;; esac - local path - while IFS= read -r -d '' path; do - _spared "$path" "$workdir" && continue - _spared "$path" "$temp" && continue - rm -rf "$path" 2>/dev/null || true - done < <(find /tmp -mindepth 1 -depth -print0 2>/dev/null) -} - -# LRU-evict oldest regular files until $1 is at or under CACHE_CAP_BYTES. -# Does not wholesale wipe the tree. -lru_cap() { - local dir="$1" - [ -d "$dir" ] || return 0 - local size - size=$(du -sb "$dir" 2>/dev/null | awk '{print $1}') - [ -n "$size" ] || return 0 - if [ "$size" -le "$CACHE_CAP_BYTES" ]; then - return 0 - fi - local excess=$((size - CACHE_CAP_BYTES)) - local freed=0 - echo "[cleanup] $dir is ${size} bytes (cap ${CACHE_CAP_BYTES}); LRU-evicting oldest files" - local rec fsize path - while IFS= read -r -d '' rec; do - [ "$freed" -ge "$excess" ] && break - rec="${rec#* }" - fsize="${rec%% *}" - path="${rec#* }" - [ -n "$path" ] && [ -n "$fsize" ] || continue - rm -f "$path" 2>/dev/null || true - freed=$((freed + fsize)) - done < <(find "$dir" -type f -printf '%T@\t%s\t%p\0' 2>/dev/null | sort -z -n) - find "$dir" -type d -empty -delete 2>/dev/null || true + clean_dir /tmp "$workdir" "$temp" } clean_tmp @@ -92,14 +77,13 @@ clean_tmp # (/home/runner) runner users. rm -rf "$HOME"/.gradle/daemon/ 2>/dev/null || true rm -rf "$HOME"/.gradle/caches/journal-* 2>/dev/null || true -find "$HOME"/.gradle -name '*.lock' -delete 2>/dev/null || true +# Shallow on purpose: Gradle's lock files sit at most three levels under +# caches/, and a find over the whole (multi-GB) tree is what made this slow. +find "$HOME"/.gradle/caches -maxdepth 3 -name '*.lock' -delete 2>/dev/null || true # Runner diagnostic logs find /actions-runner/_diag -name '*.log' -delete 2>/dev/null || true find /runner-data/_diag -name '*.log' -delete 2>/dev/null || true -lru_cap "$HOME"/.gradle -lru_cap "$HOME"/.bun - AFTER=$(df -h / | awk 'NR==2 {print $4}') echo "[cleanup] Post-job cleanup complete. Disk free: ${BEFORE} -> ${AFTER}" diff --git a/docker-compose.yml b/docker-compose.yml index 0194ee8..7991910 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,10 @@ -# Beelink X64 layout: three persistent runners, each with its own work and -# cache volumes. Do not share bun/gradle/npm/cargo volumes across services — -# concurrent jobs would corrupt the same tree. YAML merge of volumes: -# replaces the list, so docker.sock is repeated on each service. +# X64 runner layout (beelink by default; hetzner uses the same file with its own +# .env, see hosts/): three runners, each with its own work and cache volumes, +# plus a nightly `pruner`. Do not share bun/gradle/npm/cargo/mise volumes +# across runner services - concurrent jobs would corrupt the same tree. YAML +# merge of volumes: replaces the list, so docker.sock is repeated on each +# service. ARM64 / rpi5 hosts layer hosts/rpi5.compose.yml on top (no heavy +# cache volumes), see README. x-runner: &runner image: ghcr.io/dodi-smart/github-runner restart: unless-stopped @@ -13,10 +16,32 @@ x-runner: &runner RUNNER_WORKDIR: /tmp/runner/work RUNNER_SCOPE: org ORG_NAME: ${APP_LOGIN} - LABELS: ${LABELS:-self-hosted,Linux,X64,beelink,docker,large} + LABELS: ${LABELS:-self-hosted,Linux,X64,docker,x86,large,light,beelink} RUNNER_GROUP: ${RUNNER_GROUP:-Default} - DISABLE_AUTO_UPDATE: ${DISABLE_AUTO_UPDATE:-false} - EPHEMERAL: "false" + # The upstream entrypoint only tests these with `[ -n "$VAR" ]`, so ANY + # non-empty value - including "false" - turns the flag on. + # + # EPHEMERAL: "true" is what these runners have always actually done: the old + # `EPHEMERAL: "false"` passed --ephemeral too. Each runner takes exactly one + # job, deregisters, exits, and `restart: unless-stopped` brings up a fresh + # container that registers again. That is intended: a clean container per + # job, while the named volumes below carry the caches across containers. + EPHEMERAL: "true" + # Empty (the default) leaves runner self-update on. Set it in .env to ANY + # non-empty value to disable updates; `DISABLE_AUTO_UPDATE=false` disables + # them too. See .env.example. + DISABLE_AUTO_UPDATE: ${DISABLE_AUTO_UPDATE:-} + # Runner.Listener is the long-lived process of a registered runner. Exec form + # on purpose: under CMD-SHELL the wrapping `sh -c` carries the pattern in its + # own command line and `pgrep -f` would match itself, healthy forever. The + # container restarts after every job (ephemeral), so start_period covers + # re-registration. + healthcheck: + test: ["CMD", "pgrep", "-f", "Runner.Listener"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 120s volumes: - /var/run/docker.sock:/var/run/docker.sock security_opt: @@ -33,6 +58,11 @@ services: - runner-pnpm:/root/.cache/pnpm - runner-gradle:/root/.gradle - runner-cargo:/root/.cargo + - runner-rustup:/root/.rustup + - runner-konan:/root/.konan + - runner-mise-data:/root/.local/share/mise + - runner-mise-cache:/root/.cache/mise + - runner-hk:/root/.cache/hk runner-2: <<: *runner volumes: @@ -43,6 +73,11 @@ services: - runner-pnpm-2:/root/.cache/pnpm - runner-gradle-2:/root/.gradle - runner-cargo-2:/root/.cargo + - runner-rustup-2:/root/.rustup + - runner-konan-2:/root/.konan + - runner-mise-data-2:/root/.local/share/mise + - runner-mise-cache-2:/root/.cache/mise + - runner-hk-2:/root/.cache/hk runner-3: <<: *runner volumes: @@ -53,6 +88,45 @@ services: - runner-pnpm-3:/root/.cache/pnpm - runner-gradle-3:/root/.gradle - runner-cargo-3:/root/.cargo + - runner-rustup-3:/root/.rustup + - runner-konan-3:/root/.konan + - runner-mise-data-3:/root/.local/share/mise + - runner-mise-cache-3:/root/.cache/mise + - runner-hk-3:/root/.cache/hk + # Nightly maintenance, off the job path (issue #16: the same scans inside the + # per-job cleanup.sh cost 21-39 s on every job). Same image, pruner.sh as the + # entrypoint. It needs only the caches it caps (not work/npm/pnpm/...), and + # docker.sock to prune the host daemon: `docker system prune -af --filter + # until=24h` and `docker builder prune -af --filter until=72h`. Named volumes + # are never pruned. Tunables: PRUNE_INTERVAL_SECONDS, CACHE_CAP_BYTES, ... + # (see pruner.sh). + pruner: + image: ghcr.io/dodi-smart/github-runner + restart: unless-stopped + entrypoint: ["/usr/local/bin/pruner.sh"] + environment: + CACHES_ROOT: /caches + CACHE_CAP_BYTES: ${CACHE_CAP_BYTES:-21474836480} + PRUNE_INTERVAL_SECONDS: ${PRUNE_INTERVAL_SECONDS:-86400} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - runner-gradle:/caches/1/gradle + - runner-bun:/caches/1/bun + - runner-mise-cache:/caches/1/mise-cache + - runner-hk:/caches/1/hk + - runner-mise-data:/caches/1/mise-data + - runner-gradle-2:/caches/2/gradle + - runner-bun-2:/caches/2/bun + - runner-mise-cache-2:/caches/2/mise-cache + - runner-hk-2:/caches/2/hk + - runner-mise-data-2:/caches/2/mise-data + - runner-gradle-3:/caches/3/gradle + - runner-bun-3:/caches/3/bun + - runner-mise-cache-3:/caches/3/mise-cache + - runner-hk-3:/caches/3/hk + - runner-mise-data-3:/caches/3/mise-data + security_opt: + - label:disable volumes: runner-work: {} @@ -73,3 +147,18 @@ volumes: runner-cargo: {} runner-cargo-2: {} runner-cargo-3: {} + runner-rustup: {} + runner-rustup-2: {} + runner-rustup-3: {} + runner-konan: {} + runner-konan-2: {} + runner-konan-3: {} + runner-mise-data: {} + runner-mise-data-2: {} + runner-mise-data-3: {} + runner-mise-cache: {} + runner-mise-cache-2: {} + runner-mise-cache-3: {} + runner-hk: {} + runner-hk-2: {} + runner-hk-3: {} diff --git a/hosts/beelink.env.example b/hosts/beelink.env.example new file mode 100644 index 0000000..682fd4d --- /dev/null +++ b/hosts/beelink.env.example @@ -0,0 +1,18 @@ +# Host .env for the beelink X64 pool (3 runners, the default docker-compose.yml). +# Copy to .env next to docker-compose.yml. Never commit .env. +# +# Target labels (CI redesign): +# large -> heavy jobs (Gradle/Android, Next.js builds) +# light -> overflow for light jobs, resolved by GitHub's own queue +# The runner binary adds self-hosted/Linux/X64 itself; they are listed to keep +# the whole set visible in one place. +APP_ID= +APP_PRIVATE_KEY= +APP_LOGIN= + +RUNNER_NAME_PREFIX=beelink +LABELS=self-hosted,Linux,X64,docker,x86,large,light,beelink +RUNNER_GROUP=Default + +# Runners are ephemeral (set in docker-compose.yml). Leave DISABLE_AUTO_UPDATE +# unset: ANY non-empty value, even "false", disables runner self-update. diff --git a/hosts/hetzner.env.example b/hosts/hetzner.env.example new file mode 100644 index 0000000..1101a97 --- /dev/null +++ b/hosts/hetzner.env.example @@ -0,0 +1,20 @@ +# Host .env for the hetzner X64 pool. If a host runs more than one stack, give +# each stack its own RUNNER_NAME_PREFIX. +# Copy to .env next to docker-compose.yml. Never commit .env. +# +# Target labels (CI redesign): capability labels the GitHub queue resolves. +# light -> default checks, release, deploy steps +# agent -> review, triage, assist +# No `large`: heavy jobs (Gradle/Android, Next.js builds) go to the beelinks. +# The runner binary adds self-hosted/Linux/X64 itself; they are listed to keep +# the whole set visible in one place. +APP_ID= +APP_PRIVATE_KEY= +APP_LOGIN= + +RUNNER_NAME_PREFIX=hetzner-a +LABELS=self-hosted,Linux,X64,docker,x86,light,agent,hetzner +RUNNER_GROUP=Default + +# Runners are ephemeral (set in docker-compose.yml). Leave DISABLE_AUTO_UPDATE +# unset: ANY non-empty value, even "false", disables runner self-update. diff --git a/hosts/rpi5.compose.yml b/hosts/rpi5.compose.yml new file mode 100644 index 0000000..4fa7fe7 --- /dev/null +++ b/hosts/rpi5.compose.yml @@ -0,0 +1,27 @@ +# Layer on top of docker-compose.yml for the ARM64 rpi5 hosts: +# +# docker compose -f docker-compose.yml -f hosts/rpi5.compose.yml up -d +# +# hosts/rpi5.env.example sets COMPOSE_FILE so a plain `docker compose ...` +# does exactly that. The rpi5 pool is the agent/utility pool: it must not +# mount the heavy X64 cache volumes (bun, npm, pnpm, gradle, cargo, rustup, +# konan, mise, hk) - those runners isolate to RUNNER_TEMP and the SD/NVMe is +# small. Each runner keeps only docker.sock and its work volume, and the +# pruner (which only caps cache volumes) does docker pruning alone. +# `!override` replaces the base volume lists instead of merging into them. +services: + runner: + volumes: !override + - /var/run/docker.sock:/var/run/docker.sock + - runner-work:/tmp/runner/work + runner-2: + volumes: !override + - /var/run/docker.sock:/var/run/docker.sock + - runner-work-2:/tmp/runner/work + runner-3: + volumes: !override + - /var/run/docker.sock:/var/run/docker.sock + - runner-work-3:/tmp/runner/work + pruner: + volumes: !override + - /var/run/docker.sock:/var/run/docker.sock diff --git a/hosts/rpi5.env.example b/hosts/rpi5.env.example new file mode 100644 index 0000000..325e23b --- /dev/null +++ b/hosts/rpi5.env.example @@ -0,0 +1,22 @@ +# Host .env for the rpi5 ARM64 pool (agent and utility runners). +# Copy to .env next to docker-compose.yml. Never commit .env. +# +# Target labels (CI redesign): agent + utility pool. NOT `light` (light jobs +# assume X64: Flutter, bun, zavet failed when ARM64 runners carried it) and +# NOT `large`. +# The runner binary adds self-hosted/Linux/ARM64 itself; they are listed to +# keep the whole set visible in one place. +APP_ID= +APP_PRIVATE_KEY= +APP_LOGIN= + +RUNNER_NAME_PREFIX=rpi5 +LABELS=self-hosted,Linux,ARM64,docker,agent,rpi5 +RUNNER_GROUP=Default + +# Do not mount the heavy cache volumes on the Pi: layer the override that +# keeps only docker.sock + the work volume (see hosts/rpi5.compose.yml). +COMPOSE_FILE=docker-compose.yml:hosts/rpi5.compose.yml + +# Runners are ephemeral (set in docker-compose.yml). Leave DISABLE_AUTO_UPDATE +# unset: ANY non-empty value, even "false", disables runner self-update. diff --git a/pruner.sh b/pruner.sh new file mode 100755 index 0000000..58f6998 --- /dev/null +++ b/pruner.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# Nightly pruner. Run by the `pruner` compose service (same image, this script +# as the entrypoint), NOT by a runner job hook. +# +# These scans used to sit in cleanup.sh and ran after every job, costing +# 21-39 s of runner time each time and freeing nothing (issue #16). Here they +# run once a day, off the job path: +# 1. LRU size caps on the per-runner cache volumes (mounted under +# $CACHES_ROOT//, see docker-compose.yml). +# 2. docker system prune / builder prune on the host daemon (docker.sock). +# +# Env: +# PRUNE_INTERVAL_SECONDS sleep between runs (default 86400) +# PRUNE_INITIAL_DELAY sleep before the first run (default 300; keeps a +# `docker compose up -d` from running a scan at once) +# PRUNE_RUN_ONCE=true run once and exit (manual / testing) +# CACHES_ROOT where the cache volumes are mounted (/caches) +# CACHE_CAP_BYTES cap for each file-level cache dir (20 GiB) +# MISE_DATA_CAP_BYTES cap for each mise installs tree (20 GiB) +# DOCKER_PRUNE_UNTIL / DOCKER_BUILDER_PRUNE_UNTIL (24h / 72h) +# +# Intentionally not using set -e: one failing step must not stop the others. + +CACHES_ROOT="${CACHES_ROOT:-/caches}" +CACHE_CAP_BYTES="${CACHE_CAP_BYTES:-21474836480}" # 20 GiB +MISE_DATA_CAP_BYTES="${MISE_DATA_CAP_BYTES:-21474836480}" # 20 GiB +DOCKER_PRUNE_UNTIL="${DOCKER_PRUNE_UNTIL:-24h}" +DOCKER_BUILDER_PRUNE_UNTIL="${DOCKER_BUILDER_PRUNE_UNTIL:-72h}" + +log() { echo "[pruner] $(date -u +%FT%TZ) $*"; } + +dir_bytes() { + du -sb "$1" 2>/dev/null | awk '{print $1}' +} + +# Print how many bytes dir $1 is over cap $2. Returns 1 (printing nothing) when +# the dir is missing, unreadable, or already within the cap. +excess_bytes() { + local dir="$1" cap="$2" size + [ -d "$dir" ] || return 1 + size=$(dir_bytes "$dir") + [ -n "$size" ] || return 1 + if [ "$size" -le "$cap" ]; then + log "$dir ${size}B <= cap ${cap}B" >&2 + return 1 + fi + log "$dir is ${size}B (cap ${cap}B)" >&2 + echo $((size - cap)) +} + +# LRU-evict regular files, least recently accessed first, until $1 is at or +# under $2 bytes. Ordered by atime (a read counts as use; Gradle never touches +# mtime on read, so mtime order evicted the most-used dependencies first). +# Volumes mounted relatime still refresh atime on the first read after a +# change. Does not wholesale wipe the tree. +lru_cap() { + local dir="$1" cap="${2:-$CACHE_CAP_BYTES}" + local excess freed=0 rec fsize path + excess=$(excess_bytes "$dir" "$cap") || return 0 + log "evicting least recently used files from $dir" + while IFS= read -r -d '' rec; do + [ "$freed" -ge "$excess" ] && break + rec="${rec#* }" # drop atime + fsize="${rec%% *}" + path="${rec#* }" + [ -n "$path" ] && [ -n "$fsize" ] || continue + rm -f "$path" 2>/dev/null || true + freed=$((freed + fsize)) + done < <(find "$dir" -type f -printf '%A@\t%s\t%p\0' 2>/dev/null | sort -z -n) + find "$dir" -type d -empty -delete 2>/dev/null || true + log "$dir freed ~${freed}B" +} + +# mise installs (installs//) are only useful whole: deleting +# single files would leave a tool that exists but is broken. Evict whole +# / directories instead, oldest install first, until the +# installs tree fits under the cap. +lru_cap_mise_installs() { + local data="$1" cap="${2:-$MISE_DATA_CAP_BYTES}" + local installs="$data/installs" + local excess freed=0 rec path dsize + excess=$(excess_bytes "$installs" "$cap") || return 0 + log "evicting oldest tool versions from $installs" + while IFS= read -r -d '' rec; do + [ "$freed" -ge "$excess" ] && break + path="${rec#* }" + [ -d "$path" ] || continue + dsize=$(dir_bytes "$path") + rm -rf "$path" 2>/dev/null || true + freed=$((freed + ${dsize:-0})) + done < <(find "$installs" -mindepth 2 -maxdepth 2 -type d -printf '%T@\t%p\0' 2>/dev/null | sort -z -n) + log "$installs freed ~${freed}B" +} + +prune_caches() { + local runner + for runner in "$CACHES_ROOT"/*/; do + [ -d "$runner" ] || continue + runner="${runner%/}" + log "capping caches under $runner" + lru_cap "$runner/gradle" + lru_cap "$runner/bun" + lru_cap "$runner/mise-cache" + lru_cap "$runner/hk" + lru_cap_mise_installs "$runner/mise-data" + done +} + +prune_docker() { + if ! docker info >/dev/null 2>&1; then + log "docker daemon unreachable; skipping docker prune" + return 0 + fi + log "docker system prune -af --filter until=${DOCKER_PRUNE_UNTIL}" + docker system prune -af --filter "until=${DOCKER_PRUNE_UNTIL}" 2>&1 | sed 's/^/[pruner] /' + log "docker builder prune -af --filter until=${DOCKER_BUILDER_PRUNE_UNTIL}" + docker builder prune -af --filter "until=${DOCKER_BUILDER_PRUNE_UNTIL}" 2>&1 | sed 's/^/[pruner] /' +} + +run_once() { + local before after + before=$(df -h / | awk 'NR==2 {print $4}') + prune_caches + prune_docker + after=$(df -h / | awk 'NR==2 {print $4}') + log "done. Disk free: ${before} -> ${after}" +} + +if [ "${PRUNE_RUN_ONCE:-}" = "true" ]; then + run_once + exit 0 +fi + +sleep "${PRUNE_INITIAL_DELAY:-300}" +while true; do + run_once + sleep "${PRUNE_INTERVAL_SECONDS:-86400}" +done