diff --git a/.env.example b/.env.example index f63588a..99d6cc1 100644 --- a/.env.example +++ b/.env.example @@ -1,27 +1,28 @@ -# Copy to .env and fill in. Never commit .env. -# Variables interpolated by docker-compose.yml. Per-host examples with the -# target labels live in hosts/ (hetzner, beelink, rpi5). +# Copy to .env next to docker-compose.yml and fill in. Never commit .env. # GitHub App that registers the runners (required) APP_ID= APP_PRIVATE_KEY= -# Org login the runners register against (also used as ORG_NAME) +# Organization the runners register to APP_LOGIN= -# Optional — values match the compose defaults +# Runner identity. The runner adds self-hosted, the OS and the architecture to +# LABELS itself; add the capability labels your workflows select on. RUNNER_NAME_PREFIX=runner -LABELS=self-hosted,Linux,X64,docker,x86,large,light,beelink +LABELS=docker RUNNER_GROUP=Default -# Runner self-update. The upstream entrypoint tests this with `[ -n ... ]`, -# so ANY non-empty value disables auto-update - "false" included. Leave it -# unset/empty to keep auto-update on; set it to e.g. "true" to turn it off. -# DISABLE_AUTO_UPDATE=true +# Small hosts without room for per-runner caches (e.g. ARM64 boards): +# COMPOSE_FILE=docker-compose.yml:hosts/no-caches.compose.yml -# EPHEMERAL is hard-coded to "true" in docker-compose.yml (each runner takes -# one job, then a fresh container registers again; the caches live on the -# named volumes). Setting it here has no effect. +# Action archive cache (see README). Space-separated owner/repo@ref lists. +# PREFETCH_SOURCES=my-org/my-workflows@main # repos scanned for `uses:` +# PREFETCH_EXTRA=actions/checkout@v4 # actions fetched as-is -# Nightly pruner (see pruner.sh) +# Other tunables (defaults shown) +# RUNNER_IMAGE=ghcr.io/dodi-smart/github-runner # CACHE_CAP_BYTES=21474836480 # PRUNE_INTERVAL_SECONDS=86400 +# PREFETCH_INTERVAL_SECONDS=3600 +# Any non-empty value disables runner self-update ("false" too). +# DISABLE_AUTO_UPDATE=true diff --git a/README.md b/README.md index c74c492..2918c53 100644 --- a/README.md +++ b/README.md @@ -1,296 +1,175 @@ # docker-github-actions-runner-android -Based off https://github.com/myoung34/docker-github-actions-runner but adds the -android sdk into the container so we don't need to run the setup-android action -every time from: https://github.com/android-actions/setup-android -The Android SDK requires Java, so the container includes a Temurin JDK as well, -which means you can also skip the setup-java action: -https://github.com/actions/setup-java +A self-hosted GitHub Actions runner image, plus a Docker Compose setup for +running it at scale. The image builds on +[myoung34/github-runner](https://github.com/myoung34/docker-github-actions-runner) +and adds: -Node.js LTS is also baked in (see `matrix.json` for available versions), so -`actions/setup-node` is unnecessary for LTS workflows. +- a Temurin JDK, the Android SDK and Node.js LTS, so jobs can skip + `setup-java`, `setup-android` and `setup-node`; +- multi-arch images (`linux/amd64`, `linux/arm64`). -Images are published as multi-arch manifests for both `linux/amd64` and -`linux/arm64`, so Apple Silicon Macs (via Docker Desktop / OrbStack) and ARM -Linux hosts run natively without emulation. +The Compose setup adds: -To run the container, check out the environment variables from the base image: -https://github.com/myoung34/docker-github-actions-runner#environment-variables +- ephemeral runners with persistent cache volumes; +- a `pruner` service that does the maintenance; +- a local action archive cache that avoids GitHub's download rate limits. -[`docker-compose.yml`](docker-compose.yml) is the X64 layout (beelink by -default; hetzner uses the same file with its own `.env`) — three ephemeral -runners, each with its own work **and** cache volumes, plus a nightly -`pruner`. Per-host `.env` examples with the target labels are in -[`hosts/`](hosts/). After pulling a new image on a host: +## Quick start -``` -docker compose pull && docker compose up -d -``` +1. Create a GitHub App for runner registration. See the + [base image docs](https://github.com/myoung34/docker-github-actions-runner#environment-variables). +2. Copy [`.env.example`](.env.example) to `.env` and set `APP_ID`, + `APP_PRIVATE_KEY` and `APP_LOGIN`. +3. Start the runners: + ```bash + docker compose up -d + ``` -Existing named `runner-work*` volumes keep their data; new cache volumes -start empty and warm on first jobs. Do not share one bun/gradle/mise volume -across the three containers: concurrent jobs would corrupt the same tree. - -## Runner pools and labels - -Jobs pick runners by capability label, never by machine name. Each host's -`.env` sets `LABELS` (the runner binary adds `self-hosted`, the OS and the -architecture itself): - -| Pool | `.env` example | `LABELS` | Caches | -|------|----------------|----------|--------| -| hetzner (X64) | [`hosts/hetzner.env.example`](hosts/hetzner.env.example) | `self-hosted,Linux,X64,docker,x86,light,agent,hetzner` | all volumes | -| beelink (X64) | [`hosts/beelink.env.example`](hosts/beelink.env.example) | `self-hosted,Linux,X64,docker,x86,large,light,beelink` | all volumes | -| rpi5 (ARM64) | [`hosts/rpi5.env.example`](hosts/rpi5.env.example) | `self-hosted,Linux,ARM64,docker,agent,rpi5` | work only | - -| Capability label | Meaning | -|------------------|---------| -| `light` | default checks, release, deploy steps (X64 only: no rpi5) | -| `large` | heavy jobs: Gradle/Android, Next.js builds (beelink) | -| `agent` | review, triage, assist (hetzner and rpi5) | - -The rpi5 pool is an agent/utility pool: it has no `light` label and must not -mount the heavy cache volumes. Its `.env` example sets -`COMPOSE_FILE=docker-compose.yml:hosts/rpi5.compose.yml`, which layers -[`hosts/rpi5.compose.yml`](hosts/rpi5.compose.yml) over the base file so each -runner keeps only `docker.sock` and its work volume (and the pruner only -prunes Docker). - -### Rolling out a label change - -Runners are ephemeral and re-register on their next job, so changing labels is -a `.env` edit plus a recreate: - -1. Copy the host's `hosts/.env.example` values into its `.env` (keep the - real `APP_*` values). On rpi5 also keep the `COMPOSE_FILE` line. -2. `docker compose pull && docker compose up -d`. -3. Check the pool in the org's runner list (Settings, Actions, Runners): each - runner should show the labels above. `docker compose ps` shows `healthy` - once `Runner.Listener` is up. - -## Ephemeral runners - -`EPHEMERAL` is `"true"` in the compose file. The upstream entrypoint -(`myoung34/github-runner`) only tests `[ -n "${EPHEMERAL}" ]` (and the same for -`DISABLE_AUTO_UPDATE`), so any non-empty value, `"false"` included, passes -`--ephemeral` to `config.sh`. The previous `EPHEMERAL: "false"` therefore -never kept runners persistent: they have always been ephemeral, one job per -container, restarted by `restart: unless-stopped`. The compose file now says -what actually happens. Each job gets a clean container; the caches survive on -the named volumes below. - -`DISABLE_AUTO_UPDATE` is empty by default. Set it to any non-empty value in -`.env` to disable runner self-update (`false` disables it too). - -## Health check - -Each runner reports healthy while `Runner.Listener` is alive -(`pgrep -f Runner.Listener`, exec form: under `CMD-SHELL` the wrapping -`sh -c` would match its own command line). The container restarts after each -ephemeral job; `start_period` (120 s) keeps the re-registration from counting -as failures. - -## Image identity - -The image carries `RUNNER_IMAGE_TAG` (the fully pinned tag, e.g. -`2.338.0-jdk21-sdk36-node24`) and `RUNNER_IMAGE_REVISION` (the commit it was -built from) as environment variables, set from build args by `deploy.yml` -(`build.yml` sets `pr-` and the head SHA). Jobs can print them to tell -which image they ran on. Local builds default to `dev`. - -## X64 cache volumes - -This compose is for the **X64** class (`beelink`, `hetzner`). The `LABELS` -default is the beelink set; hosts override it in `.env` (see the table above). -The runners are ephemeral, but the volumes survive between containers and -jobs. - -Each runner gets its own: - -| Mount | Volume (runner / runner-2 / runner-3) | Why | Capped by pruner | -|-------|----------------------------------------|-----|------------------| -| `/tmp/runner/work` | `runner-work*` | Job workspace. Cleanup spares `$RUNNER_WORKDIR`. | no | -| `/root/.bun` | `runner-bun*` | Bun install cache | yes | -| `/root/.npm` | `runner-npm*` | npm cache | no | -| `/root/.cache/pnpm` | `runner-pnpm*` | pnpm store | no | -| `/root/.gradle` | `runner-gradle*` | Gradle modules, build-cache, transforms | yes | -| `/root/.cargo` | `runner-cargo*` | Cargo registry | no | -| `/root/.rustup` | `runner-rustup*` | Rust toolchains (issue #8) | no | -| `/root/.konan` | `runner-konan*` | Kotlin/Native toolchain and dependencies | no | -| `/root/.local/share/mise` | `runner-mise-data*` | mise tool installs (`MISE_DATA_DIR`) | yes, whole tool versions | -| `/root/.cache/mise` | `runner-mise-cache*` | mise downloads (`MISE_CACHE_DIR`) | yes | -| `/root/.cache/hk` | `runner-hk*` | hk cache, incl. pkl (`HK_CACHE_DIR`, `HK_PKL_CACHE_DIR`) | yes | - -The Dockerfile sets `MISE_DATA_DIR`, `MISE_CACHE_DIR`, `HK_CACHE_DIR` and -`HK_PKL_CACHE_DIR` so the tools write where the volumes are mounted. The -`~/.cache` directory is never mounted as a whole: that would shadow the pnpm -volume, so each subdirectory has its own volume. `jdx/mise-action` should use -`cache: false` on these runners; the volumes replace its GitHub-cache step. - -Budget ~20 GB per capped cache directory (enforced by the nightly pruner with -LRU, not a wipe). The remaining volumes (npm, pnpm, cargo, rustup, konan) are -not capped yet (issue #7 tracks the bun/gradle-only caps; hard-linked stores -need care). ARM64 / Pi hosts should **not** mount these cache volumes — those -runners isolate to `RUNNER_TEMP`, see `hosts/rpi5.compose.yml`. - -## Maintenance: per-job cleanup and the nightly pruner - -`cleanup.sh` (the `JOB_COMPLETED` hook) only does the cheap tidy-up after each -job: `/tmp` outside the workdir, Gradle daemon/journals/locks, runner `_diag` -logs. The expensive scans used to run there on every job and cost 21–39 s each -(issue #16). They now run once a day in the `pruner` compose service, which -uses the same image with [`pruner.sh`](pruner.sh) as its entrypoint and -mounts only the capped cache volumes plus `docker.sock`: - -- LRU size caps (`CACHE_CAP_BYTES`, default 20 GiB) on gradle, bun, the mise - cache and the hk cache, evicting by last access time; mise installs are - evicted as whole `tool/version` directories so a tool is never left half - deleted (`MISE_DATA_CAP_BYTES`). -- `docker system prune -af --filter until=24h` -- `docker builder prune -af --filter until=72h` - -Named volumes are never pruned. The first run is 5 minutes after start -(`PRUNE_INITIAL_DELAY`), then every `PRUNE_INTERVAL_SECONDS` (86400). Run it -once by hand with -`docker compose run --rm -e PRUNE_RUN_ONCE=true pruner`. - -Do **not** tmpfs `/tmp`. `RUNNER_WORKDIR` is `/tmp/runner/work`; a tmpfs -or a blanket `find /tmp -delete` hides or wipes the work volume. - -### Workflows must use the volume homes - -Named volumes are unused if the job relocates the caches. On these X64 -self-hosted runners, workflows must **not** isolate `GRADLE_USER_HOME` or -`BUN_INSTALL_CACHE_DIR` (or `npm_config_cache` / `CARGO_HOME` / -`MISE_DATA_DIR` / `MISE_CACHE_DIR` / `HK_CACHE_DIR`) to -`$RUNNER_TEMP`. That is the `setup-stack` local-home mode in -`dodi-smart/.github` — `deps-verify` stays isolated; `pr-checks` uses the -default homes so the volumes are actually read. - -The other half of cache poison is GitHub `actions/cache` with -`restore-keys` re-importing a partial bun tarball. Use an exact key, or -do not upload the bun install cache at all. +[`docker-compose.yml`](docker-compose.yml) starts three runners and a `pruner`. +Each runner is **ephemeral**: it takes one job, then restarts and registers +again with a clean container. Only the named volumes survive between jobs. -## Image tags +To update a host, run `docker compose pull && docker compose up -d`. -Images are published to GitHub Container Registry (GHCR): -- `ghcr.io/dodi-smart/github-runner` - -### Tag scheme - -Each image is built with a specific combination of base runner version, JDK -version, Node major version, and Android SDK level. Tags let you choose how -tightly to pin: - -| Tag | Example | What it pins | What floats | -|-----|---------|--------------|-------------| -| `latest` | `latest` | Nothing | Runner, JDK, Node, and SDK all track latest | -| `jdk` | `jdk21` | JDK major version | Runner, Node (default), and SDK track latest | -| `node` | `node22` | Node major version | Runner, JDK (default), and SDK track latest | -| `jdk-node` | `jdk21-node22` | JDK + Node | Runner and SDK track latest | -| `-jdk-sdk-node` | `2.332.0-jdk21-sdk36-node22` | Everything | Nothing — fully pinned | - -### Which tag should I use? - -- **`latest`** — always the newest runner, default JDK, default Node LTS, and - latest Android SDK. Good for staying current, but builds may break if a new - SDK introduces incompatibilities. -- **`jdk21`** / **`jdk17`** — pin the JDK version but still get runner, Node, - and SDK updates automatically. A good balance for most users. -- **`jdk21-node22`** — pin both JDK and Node majors; runner and SDK still - track latest. Useful when an LTS-pinned Node is part of your build's - contract. -- **`2.332.0-jdk21-sdk36-node22`** — fully pinned. Use this for reproducible - builds where you want complete control over when to upgrade. Old pinned tags - are never overwritten; they remain in the registry from previous builds. - -### What's in each image? - -Every image includes: -- The [myoung34/github-runner](https://github.com/myoung34/docker-github-actions-runner) base image -- [Temurin JDK](https://adoptium.net/) (the version in the tag) -- [Node.js](https://nodejs.org/) LTS (the version in the tag) — installed - from NodeSource, replacing the base image's older system Node -- Android SDK platform matching the SDK level in the tag (e.g., `sdk36` = `platforms;android-36`) -- Latest Android build-tools and NDK for that SDK level -- Android SDK command-line tools, platform-tools, and cmake -- Multi-arch manifest covering `linux/amd64` and `linux/arm64` - -Projects that need additional SDK platforms (e.g., an older `compileSdk`) can -install them at build time: -```yaml -- run: sdkmanager "platforms;android-28" -``` +## Fit it to your setup -### Currently available JDK versions +**One host or many.** Run the same Compose file on every host and change only +`.env`. To run more or fewer runners per host, add or remove `runner-N` +services, along with their volumes. -The build matrix is defined in [`matrix.json`](matrix.json). Check that file -for the current list of JDK versions and which one is the default. +**Different kinds of machines.** Give each host the capability labels your +workflows select on, through `LABELS` in its `.env`. For example, `build` for +general jobs and `heavy` for memory-hungry ones. Workflows then ask for +`runs-on: [self-hosted, Linux, build]`. Label by what a machine can do, not by +its name. A host can carry several labels, so a large machine labelled +`build,heavy` also picks up general jobs when the others are busy. -### Currently available Node versions +**Small hosts.** Boards such as a Raspberry Pi have little disk. Layer +[`hosts/no-caches.compose.yml`](hosts/no-caches.compose.yml) by setting +`COMPOSE_FILE` in `.env`. Those runners skip the per-runner cache volumes. -Also defined in [`matrix.json`](matrix.json) — see `node_versions` for the -current LTS majors and `default_node` for the version applied to short tags -like `latest` and `jdk`. +**Repository or enterprise runners.** The Compose file registers runners for +an organization (`RUNNER_SCOPE: org`). For other scopes, change the runner +environment to what the +[base image](https://github.com/myoung34/docker-github-actions-runner#environment-variables) +supports. -## Automated updates +**Your own image.** Set `RUNNER_IMAGE` to use a fork or a pinned tag. See +[Image tags](#image-tags). -Versions are kept up to date automatically: +## Caches -| Component | Mechanism | -|-----------|-----------| -| Base runner image | [Renovate](https://github.com/renovatebot/renovate) tracks `myoung34/github-runner` Docker tags | -| GitHub Actions versions | Renovate (`config:recommended`) | -| New JDK major versions | Weekly workflow queries the [Adoptium API](https://api.adoptium.net) and opens a PR | -| New Node LTS majors | Weekly workflow queries the [Node Release schedule](https://github.com/nodejs/Release/blob/main/schedule.json) and opens a PR | -| Android SDK platform level | Weekly workflow queries `sdkmanager` and opens a PR | -| Android build-tools | Weekly workflow queries `sdkmanager` and opens a PR | -| Android NDK | Weekly workflow queries `sdkmanager` and opens a PR | +Each runner gets its own volume for these caches: -Every pull request is validated by [`build.yml`](.github/workflows/build.yml), -which builds each JDK against the default Node, each Node against the default -JDK, and one `linux/arm64` build of the default pair. Its `All builds passed` -job is the single check to require in branch protection. +- bun, npm and pnpm +- Gradle +- Cargo and rustup +- Kotlin/Native +- mise and hk -Because this repository is a fork, `renovate.json` sets -`"forkProcessing": "enabled"`. The Mend-hosted Renovate App leaves fork -processing disabled for org-wide ("All repositories") installations, so without -that flag Renovate silently skips the repository and the base image is never -bumped. +The image sets `MISE_DATA_DIR`, `MISE_CACHE_DIR` and `HK_CACHE_DIR` so those +tools write there. -The weekly SDK/JDK/Node workflow opens its PR with `GITHUB_TOKEN`, which means -GitHub does not run `build.yml` on it. Upstream solves this with an -`AUTOMERGE_PAT` secret and `gh pr merge --auto`; this fork does not, so those -PRs are merged manually and the builds run on merge. +- **Never share a cache volume between runners.** Concurrent jobs would + corrupt it. +- **Keep the default cache locations in workflows.** If a job moves + `GRADLE_USER_HOME` or `BUN_INSTALL_CACHE_DIR` into `$RUNNER_TEMP`, the + volumes go unused. +- **Turn off the GitHub cache where a volume replaces it.** For example, use + `cache: false` on `jdx/mise-action`. +- **Don't put `/tmp` on tmpfs.** The job workspace is `/tmp/runner/work`. -## Runner registration preflight +## Action archive cache -The image wraps the base entrypoint with -[`preflight-entrypoint.sh`](preflight-entrypoint.sh). When -`CONFIGURED_ACTIONS_RUNNER_FILES_DIR` is set, the upstream entrypoint reuses -persisted registration credentials without checking them, so a runner that was -removed on GitHub's side crash-loops forever instead of re-registering. The -preflight asks GitHub whether the persisted runner still exists and wipes the -stored registration only on a definitive `404`, letting a fresh `ACCESS_TOKEN` -registration take over. Timeouts, missing variables, and unparseable state all -leave the configuration untouched. +Ephemeral runners download every `uses:` action on every job. Several runners +behind one IP address soon hit `429 Too Many Requests` in "Set up job". -## Building locally +The runner can read action archives from a local directory before it downloads +anything. Set `ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE` to that directory. Here it +is `/opt/actionarchivecache`, mounted read-only into every runner. The runner +never writes this directory itself, so the `pruner` fills it every hour. It +uses the GitHub App's token, so the downloads count against the App's rate +limit rather than your IP address. It fetches: +- every `uses:` found in the repositories listed in `PREFETCH_SOURCES`, such as + the repo that holds your reusable workflows; +- the actions listed in `PREFETCH_EXTRA`. + +With neither variable set, the cache stays empty and jobs download as usual. A +cache miss always falls back to a normal download. + +To fill the cache straight away: + +```bash +docker compose run --rm pruner prefetch ``` -docker build -f Dockerfile . -``` -Build args can be used to customize the image. See the top of the -[`Dockerfile`](Dockerfile) for the current defaults and available args -(`VERSION`, `JAVA_VERSION`, `NODE_VERSION`, `COMPILE_SDK`, `BUILD_TOOLS`, -`NDK_VERSION`, `SDK_TOOLS`, `RUNNER_IMAGE_TAG`, `RUNNER_IMAGE_REVISION`). +To confirm it works, re-run a job with debug logging. "Set up job" then shows +`Copied action archive '/opt/actionarchivecache/...'`. + +## Maintenance + +The `pruner` service ([`pruner.sh`](pruner.sh)) keeps slow work off the job +path: + +- **Nightly:** trims the Gradle, bun, mise and hk caches to `CACHE_CAP_BYTES` + each (default 20 GiB), dropping the least recently used files first. It then + runs `docker system prune --filter until=24h` and + `docker builder prune --filter until=72h`. Named volumes are never pruned. +- **Hourly:** fills the action archive cache, capped at 2 GiB. + +After each job, [`cleanup.sh`](cleanup.sh) does only a quick tidy-up: `/tmp`, +Gradle daemons and locks, and runner logs. + +To run the nightly pass once by hand: -Example: +```bash +docker compose run --rm -e PRUNE_RUN_ONCE=true pruner ``` -docker build --build-arg JAVA_VERSION=17 --build-arg NODE_VERSION=22 --build-arg COMPILE_SDK=35 -f Dockerfile . + +## Image tags + +Images are published to `ghcr.io/dodi-smart/github-runner`. + +| Tag | Pins | +|---|---| +| `latest` | Nothing | +| `jdk21` | JDK major | +| `node22` | Node major | +| `jdk21-node22` | JDK and Node majors | +| `2.338.0-jdk21-sdk36-node24` | Everything: runner, JDK, SDK and Node. Never overwritten | + +[`matrix.json`](matrix.json) lists the available JDK and Node versions and the +defaults. Each image sets `RUNNER_IMAGE_TAG` and `RUNNER_IMAGE_REVISION`, so a +job can tell which image it ran on. + +To use another Android platform, install it in the job: +`sdkmanager "platforms;android-28"`. + +## Notes + +- `EPHEMERAL` and `DISABLE_AUTO_UPDATE` count as set for any non-empty value, + `"false"` included. The upstream entrypoint only checks whether they are + empty. +- [`preflight-entrypoint.sh`](preflight-entrypoint.sh) wipes a stored runner + registration only after GitHub confirms that runner is gone (`404`). A + removed runner then registers again instead of crash-looping. + +## Development + +[`build.yml`](.github/workflows/build.yml) builds every pull request. Its +`All builds passed` check is the one to require. + +Renovate keeps the base image and the actions up to date; `forkProcessing` is +on because this repo is a fork. A weekly workflow opens pull requests for new +JDK, Node LTS, Android SDK, build-tools and NDK versions. + +To build locally: + +```bash +docker build --build-arg JAVA_VERSION=17 --build-arg NODE_VERSION=22 . ``` -## Inspiration -- https://github.com/kriskda/docker-github-action-android-container (comes with emulator which we don't need) -- https://github.com/jordond/docker-android-github-runner (desire more configurability - re:version of myoung34 container, etc) +The build args and their defaults are at the top of the +[`Dockerfile`](Dockerfile). diff --git a/cleanup.sh b/cleanup.sh index 5fbfef3..29dfb19 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -36,7 +36,7 @@ _ancestor_of() { return 1 } -# RUNNER_WORKDIR is /tmp/runner/work on the beelinks. A blanket +# RUNNER_WORKDIR is /tmp/runner/work in docker-compose.yml. A blanket # `find /tmp -delete` wipes the work volume. Spare the workdir (and # RUNNER_TEMP if it sits under /tmp), plus their ancestors under /tmp. # diff --git a/docker-compose.yml b/docker-compose.yml index 7991910..12a4542 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,11 @@ -# X64 runner layout (beelink by default; hetzner uses the same file with its own -# .env, see hosts/): three runners, each with its own work and cache volumes, -# plus a nightly `pruner`. Do not share bun/gradle/npm/cargo/mise volumes -# across runner services - concurrent jobs would corrupt the same tree. YAML -# merge of volumes: replaces the list, so docker.sock is repeated on each -# service. ARM64 / rpi5 hosts layer hosts/rpi5.compose.yml on top (no heavy -# cache volumes), see README. +# Three ephemeral runners, each with its own work and cache volumes, plus a +# `pruner` for maintenance. Never share a cache volume between runners: +# concurrent jobs would corrupt it. The action archive cache is the exception: +# read-only in every runner, filled by the pruner. `volumes:` lists replace on +# merge, so docker.sock is repeated per service. Small hosts can layer +# hosts/no-caches.compose.yml on top (see README). x-runner: &runner - image: ghcr.io/dodi-smart/github-runner + image: ${RUNNER_IMAGE:-ghcr.io/dodi-smart/github-runner} restart: unless-stopped environment: APP_ID: ${APP_ID} @@ -16,7 +15,7 @@ x-runner: &runner RUNNER_WORKDIR: /tmp/runner/work RUNNER_SCOPE: org ORG_NAME: ${APP_LOGIN} - LABELS: ${LABELS:-self-hosted,Linux,X64,docker,x86,large,light,beelink} + LABELS: ${LABELS:-docker} RUNNER_GROUP: ${RUNNER_GROUP:-Default} # The upstream entrypoint only tests these with `[ -n "$VAR" ]`, so ANY # non-empty value - including "false" - turns the flag on. @@ -31,6 +30,12 @@ x-runner: &runner # non-empty value to disable updates; `DISABLE_AUTO_UPDATE=false` disables # them too. See .env.example. DISABLE_AUTO_UPDATE: ${DISABLE_AUTO_UPDATE:-} + # Ephemeral runners re-download every action archive for every job, and the + # few public IPs of these hosts hit codeload.github.com's 429 limit. The + # runner (>= 2.319) reads /_/.tar.gz before it + # downloads, and only reads: the `pruner` service fills the shared + # runner-action-cache volume (see pruner.sh), mounted read-only below. + ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE: /opt/actionarchivecache # Runner.Listener is the long-lived process of a registered runner. Exec form # on purpose: under CMD-SHELL the wrapping `sh -c` carries the pattern in its # own command line and `pgrep -f` would match itself, healthy forever. The @@ -63,6 +68,7 @@ services: - runner-mise-data:/root/.local/share/mise - runner-mise-cache:/root/.cache/mise - runner-hk:/root/.cache/hk + - runner-action-cache:/opt/actionarchivecache:ro runner-2: <<: *runner volumes: @@ -78,6 +84,7 @@ services: - runner-mise-data-2:/root/.local/share/mise - runner-mise-cache-2:/root/.cache/mise - runner-hk-2:/root/.cache/hk + - runner-action-cache:/opt/actionarchivecache:ro runner-3: <<: *runner volumes: @@ -93,19 +100,22 @@ services: - runner-mise-data-3:/root/.local/share/mise - runner-mise-cache-3:/root/.cache/mise - runner-hk-3:/root/.cache/hk - # Nightly maintenance, off the job path (issue #16: the same scans inside the - # per-job cleanup.sh cost 21-39 s on every job). Same image, pruner.sh as the - # entrypoint. It needs only the caches it caps (not work/npm/pnpm/...), and - # docker.sock to prune the host daemon: `docker system prune -af --filter - # until=24h` and `docker builder prune -af --filter until=72h`. Named volumes - # are never pruned. Tunables: PRUNE_INTERVAL_SECONDS, CACHE_CAP_BYTES, ... - # (see pruner.sh). + - runner-action-cache:/opt/actionarchivecache:ro + # Maintenance off the job path (pruner.sh): nightly cache caps and docker + # prune, hourly action-cache fill. Mounts only what it caps. pruner: - image: ghcr.io/dodi-smart/github-runner + image: ${RUNNER_IMAGE:-ghcr.io/dodi-smart/github-runner} restart: unless-stopped entrypoint: ["/usr/local/bin/pruner.sh"] environment: CACHES_ROOT: /caches + APP_ID: ${APP_ID} # App token for authenticated action downloads + APP_PRIVATE_KEY: ${APP_PRIVATE_KEY} + APP_LOGIN: ${APP_LOGIN} + ACTION_CACHE_CAP_BYTES: ${ACTION_CACHE_CAP_BYTES:-2147483648} + PREFETCH_SOURCES: ${PREFETCH_SOURCES:-} + PREFETCH_EXTRA: ${PREFETCH_EXTRA:-} + PREFETCH_INTERVAL_SECONDS: ${PREFETCH_INTERVAL_SECONDS:-3600} CACHE_CAP_BYTES: ${CACHE_CAP_BYTES:-21474836480} PRUNE_INTERVAL_SECONDS: ${PRUNE_INTERVAL_SECONDS:-86400} volumes: @@ -125,10 +135,12 @@ services: - runner-mise-cache-3:/caches/3/mise-cache - runner-hk-3:/caches/3/hk - runner-mise-data-3:/caches/3/mise-data + - runner-action-cache:/opt/actionarchivecache security_opt: - label:disable volumes: + runner-action-cache: {} # read-only in runners, filled by the pruner runner-work: {} runner-work-2: {} runner-work-3: {} diff --git a/hosts/beelink.env.example b/hosts/beelink.env.example deleted file mode 100644 index 682fd4d..0000000 --- a/hosts/beelink.env.example +++ /dev/null @@ -1,18 +0,0 @@ -# Host .env for the beelink X64 pool (3 runners, the default docker-compose.yml). -# Copy to .env next to docker-compose.yml. Never commit .env. -# -# Target labels (CI redesign): -# large -> heavy jobs (Gradle/Android, Next.js builds) -# light -> overflow for light jobs, resolved by GitHub's own queue -# The runner binary adds self-hosted/Linux/X64 itself; they are listed to keep -# the whole set visible in one place. -APP_ID= -APP_PRIVATE_KEY= -APP_LOGIN= - -RUNNER_NAME_PREFIX=beelink -LABELS=self-hosted,Linux,X64,docker,x86,large,light,beelink -RUNNER_GROUP=Default - -# Runners are ephemeral (set in docker-compose.yml). Leave DISABLE_AUTO_UPDATE -# unset: ANY non-empty value, even "false", disables runner self-update. diff --git a/hosts/hetzner.env.example b/hosts/hetzner.env.example deleted file mode 100644 index 1101a97..0000000 --- a/hosts/hetzner.env.example +++ /dev/null @@ -1,20 +0,0 @@ -# Host .env for the hetzner X64 pool. If a host runs more than one stack, give -# each stack its own RUNNER_NAME_PREFIX. -# Copy to .env next to docker-compose.yml. Never commit .env. -# -# Target labels (CI redesign): capability labels the GitHub queue resolves. -# light -> default checks, release, deploy steps -# agent -> review, triage, assist -# No `large`: heavy jobs (Gradle/Android, Next.js builds) go to the beelinks. -# The runner binary adds self-hosted/Linux/X64 itself; they are listed to keep -# the whole set visible in one place. -APP_ID= -APP_PRIVATE_KEY= -APP_LOGIN= - -RUNNER_NAME_PREFIX=hetzner-a -LABELS=self-hosted,Linux,X64,docker,x86,light,agent,hetzner -RUNNER_GROUP=Default - -# Runners are ephemeral (set in docker-compose.yml). Leave DISABLE_AUTO_UPDATE -# unset: ANY non-empty value, even "false", disables runner self-update. diff --git a/hosts/no-caches.compose.yml b/hosts/no-caches.compose.yml new file mode 100644 index 0000000..4f7cc25 --- /dev/null +++ b/hosts/no-caches.compose.yml @@ -0,0 +1,24 @@ +# For small hosts (e.g. ARM64 boards with little disk): no per-runner cache +# volumes, only docker.sock, the work volume and the action archive cache. +# COMPOSE_FILE=docker-compose.yml:hosts/no-caches.compose.yml (in .env) +# `!override` replaces the base volume lists instead of merging into them. +services: + runner: + volumes: !override + - /var/run/docker.sock:/var/run/docker.sock + - runner-work:/tmp/runner/work + - runner-action-cache:/opt/actionarchivecache:ro + runner-2: + volumes: !override + - /var/run/docker.sock:/var/run/docker.sock + - runner-work-2:/tmp/runner/work + - runner-action-cache:/opt/actionarchivecache:ro + runner-3: + volumes: !override + - /var/run/docker.sock:/var/run/docker.sock + - runner-work-3:/tmp/runner/work + - runner-action-cache:/opt/actionarchivecache:ro + pruner: + volumes: !override + - /var/run/docker.sock:/var/run/docker.sock + - runner-action-cache:/opt/actionarchivecache diff --git a/hosts/rpi5.compose.yml b/hosts/rpi5.compose.yml deleted file mode 100644 index 4fa7fe7..0000000 --- a/hosts/rpi5.compose.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Layer on top of docker-compose.yml for the ARM64 rpi5 hosts: -# -# docker compose -f docker-compose.yml -f hosts/rpi5.compose.yml up -d -# -# hosts/rpi5.env.example sets COMPOSE_FILE so a plain `docker compose ...` -# does exactly that. The rpi5 pool is the agent/utility pool: it must not -# mount the heavy X64 cache volumes (bun, npm, pnpm, gradle, cargo, rustup, -# konan, mise, hk) - those runners isolate to RUNNER_TEMP and the SD/NVMe is -# small. Each runner keeps only docker.sock and its work volume, and the -# pruner (which only caps cache volumes) does docker pruning alone. -# `!override` replaces the base volume lists instead of merging into them. -services: - runner: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock - - runner-work:/tmp/runner/work - runner-2: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock - - runner-work-2:/tmp/runner/work - runner-3: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock - - runner-work-3:/tmp/runner/work - pruner: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock diff --git a/hosts/rpi5.env.example b/hosts/rpi5.env.example deleted file mode 100644 index 325e23b..0000000 --- a/hosts/rpi5.env.example +++ /dev/null @@ -1,22 +0,0 @@ -# Host .env for the rpi5 ARM64 pool (agent and utility runners). -# Copy to .env next to docker-compose.yml. Never commit .env. -# -# Target labels (CI redesign): agent + utility pool. NOT `light` (light jobs -# assume X64: Flutter, bun, zavet failed when ARM64 runners carried it) and -# NOT `large`. -# The runner binary adds self-hosted/Linux/ARM64 itself; they are listed to -# keep the whole set visible in one place. -APP_ID= -APP_PRIVATE_KEY= -APP_LOGIN= - -RUNNER_NAME_PREFIX=rpi5 -LABELS=self-hosted,Linux,ARM64,docker,agent,rpi5 -RUNNER_GROUP=Default - -# Do not mount the heavy cache volumes on the Pi: layer the override that -# keeps only docker.sock + the work volume (see hosts/rpi5.compose.yml). -COMPOSE_FILE=docker-compose.yml:hosts/rpi5.compose.yml - -# Runners are ephemeral (set in docker-compose.yml). Leave DISABLE_AUTO_UPDATE -# unset: ANY non-empty value, even "false", disables runner self-update. diff --git a/pruner.sh b/pruner.sh index 58f6998..8b701ae 100755 --- a/pruner.sh +++ b/pruner.sh @@ -8,6 +8,14 @@ # 1. LRU size caps on the per-runner cache volumes (mounted under # $CACHES_ROOT//, see docker-compose.yml). # 2. docker system prune / builder prune on the host daemon (docker.sock). +# 3. Hourly, in the background: fill the shared action archive cache. +# +# Action archive cache: the runner copies +# $ACTION_CACHE_DIR/_/.tar.gz instead of downloading when it +# exists, but never writes it. prefetch_actions() fills it as the GitHub App, so +# downloads count against the App rather than the host IP (codeload 429s). +# Refs: every `uses:` found in the PREFETCH_SOURCES repos, plus PREFETCH_EXTRA. +# `pruner.sh prefetch` runs it once (rollout warm-up). # # Env: # PRUNE_INTERVAL_SECONDS sleep between runs (default 86400) @@ -17,6 +25,9 @@ # CACHES_ROOT where the cache volumes are mounted (/caches) # CACHE_CAP_BYTES cap for each file-level cache dir (20 GiB) # MISE_DATA_CAP_BYTES cap for each mise installs tree (20 GiB) +# ACTION_CACHE_CAP_BYTES cap for the action archive cache (2 GiB) +# PREFETCH_SOURCES / PREFETCH_EXTRA "owner/repo@ref ..." to scan / fetch as-is +# PREFETCH_INTERVAL_SECONDS prefetch period (default 3600; 0 = off) # DOCKER_PRUNE_UNTIL / DOCKER_BUILDER_PRUNE_UNTIL (24h / 72h) # # Intentionally not using set -e: one failing step must not stop the others. @@ -24,6 +35,11 @@ CACHES_ROOT="${CACHES_ROOT:-/caches}" CACHE_CAP_BYTES="${CACHE_CAP_BYTES:-21474836480}" # 20 GiB MISE_DATA_CAP_BYTES="${MISE_DATA_CAP_BYTES:-21474836480}" # 20 GiB +ACTION_CACHE_DIR=/opt/actionarchivecache # mount point in docker-compose.yml +ACTION_CACHE_CAP_BYTES="${ACTION_CACHE_CAP_BYTES:-2147483648}" # 2 GiB +PREFETCH_SOURCES="${PREFETCH_SOURCES:-}" +PREFETCH_EXTRA="${PREFETCH_EXTRA:-}" +GITHUB_API="https://api.github.com" DOCKER_PRUNE_UNTIL="${DOCKER_PRUNE_UNTIL:-24h}" DOCKER_BUILDER_PRUNE_UNTIL="${DOCKER_BUILDER_PRUNE_UNTIL:-72h}" @@ -104,6 +120,89 @@ prune_caches() { lru_cap "$runner/hk" lru_cap_mise_installs "$runner/mise-data" done + lru_cap "$ACTION_CACHE_DIR" "$ACTION_CACHE_CAP_BYTES" +} + +gh_curl() { + curl -fsSL --retry 3 --connect-timeout 10 --max-time 180 \ + -H "Authorization: Bearer ${GH_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" "$@" +} + +# Print "owner/repo ref" for every remote action under tree $1. Skips local +# (./), docker:// and reusable-workflow (.github/workflows/) refs, which the +# runner never fetches as an action archive; subpath actions (owner/repo/sub) +# share the owner/repo archive. +action_refs_from_tree() { + grep -rhE --include='*.yml' --include='*.yaml' '^[[:space:]-]*uses:[[:space:]]' "$1" 2>/dev/null | + sed -E 's/^[[:space:]-]*uses:[[:space:]]*//; s/["'"'"']//g; s/[[:space:]#].*$//' | + awk -F@ 'NF == 2 && $1 !~ /^(\.|docker:)/ && $1 !~ /[$]/ && $1 !~ /\.github\/workflows\// { + split($1, p, "/"); if (p[2] != "") print p[1] "/" p[2], $2 }' +} + +# Fetch the archive of $1 (owner/repo) at $2 (tag, branch or sha) into the +# cache as _/.tar.gz, via temp file + rename so a runner never +# reads a partial file. Prints 1 when it downloaded something. +prefetch_one() { + local name="$1" ref="$2" sha="$2" dir file tmp + dir="$ACTION_CACHE_DIR/${name//\//_}" + if ! [[ "$sha" =~ ^[0-9a-f]{40}$ ]]; then + sha=$(gh_curl -H 'Accept: application/vnd.github.sha' "$GITHUB_API/repos/$name/commits/$ref") && + [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || { log "prefetch: cannot resolve $name@$ref" >&2; return 1; } + fi + file="$dir/$sha.tar.gz" + [ -s "$file" ] && return 0 + mkdir -p "$dir" && tmp=$(mktemp "$dir/.tmp.XXXXXX") || return 1 + if gh_curl -o "$tmp" "$GITHUB_API/repos/$name/tarball/$sha" && tar -tzf "$tmp" >/dev/null 2>&1; then + chmod 644 "$tmp" && mv -f "$tmp" "$file" && { log "prefetch: cached $name@$sha ($ref)" >&2; echo 1; return 0; } + fi + rm -f "$tmp" + log "prefetch: download failed for $name@$sha" >&2 + return 1 +} + +prefetch_actions() { + local work src refs name ref out n=0 fetched=0 failed=0 token + [ -n "$PREFETCH_SOURCES$PREFETCH_EXTRA" ] || return 0 # cache not configured + if [ -z "${APP_ID:-}" ] || [ -z "${APP_PRIVATE_KEY:-}" ] || [ -z "${APP_LOGIN:-}" ]; then + log "prefetch: APP_ID/APP_PRIVATE_KEY/APP_LOGIN not set; skipping" + return 0 + fi + mkdir -p "$ACTION_CACHE_DIR" || return 0 + # Same \n unescaping as the upstream entrypoint applies to APP_PRIVATE_KEY. + token=$(APP_PRIVATE_KEY="${APP_PRIVATE_KEY//\\n/$'\n'}" bash /app_token.sh 2>/dev/null) + if [ -z "$token" ] || [ "$token" = null ]; then + log "prefetch: could not mint an installation token; skipping" + return 0 + fi + GH_TOKEN="$token" + work=$(mktemp -d) || return 0 + : > "$work/refs" + for src in $PREFETCH_SOURCES; do + mkdir -p "$work/src" + if gh_curl -o "$work/src.tgz" "$GITHUB_API/repos/${src%@*}/tarball/${src#*@}" && + tar -xzf "$work/src.tgz" -C "$work/src"; then + action_refs_from_tree "$work/src" >> "$work/refs" + else + log "prefetch: cannot read source $src" + fi + rm -rf "$work/src" "$work/src.tgz" + done + for src in $PREFETCH_EXTRA; do echo "${src%@*} ${src#*@}" >> "$work/refs"; done + refs=$(sort -u "$work/refs") + rm -rf "$work" + while read -r name ref; do + [ -n "$name" ] && [ -n "$ref" ] || continue + n=$((n + 1)) + if out=$(prefetch_one "$name" "$ref"); then + [ "$out" = 1 ] && fetched=$((fetched + 1)) + else + failed=$((failed + 1)) + fi + done <<< "$refs" + # Stale temp files from an interrupted download. + find "$ACTION_CACHE_DIR" -name '.tmp.*' -mmin +60 -delete 2>/dev/null + log "prefetch: $n refs, $fetched downloaded, $failed failed" + GH_TOKEN="" } prune_docker() { @@ -126,11 +225,26 @@ run_once() { log "done. Disk free: ${before} -> ${after}" } +if [ "${1:-}" = prefetch ]; then + prefetch_actions + exit 0 +fi + if [ "${PRUNE_RUN_ONCE:-}" = "true" ]; then run_once exit 0 fi +if [ "${PREFETCH_INTERVAL_SECONDS:-3600}" -gt 0 ]; then + ( + sleep 30 + while true; do + prefetch_actions + sleep "${PREFETCH_INTERVAL_SECONDS:-3600}" + done + ) & +fi + sleep "${PRUNE_INITIAL_DELAY:-300}" while true; do run_once