From 92c78086228e02f9aa15d01696197a7147bdd90f Mon Sep 17 00:00:00 2001 From: Asen Lekov Date: Thu, 15 Oct 2026 17:27:35 +0300 Subject: [PATCH] fix(runner): serve action archives from a local cache and make the compose setup generic Ephemeral runners download every action archive on every job, and several runners behind one IP hit codeload's 429 limit in "Set up job". The runner reads ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE before downloading but never writes it, so: - mount a shared runner-action-cache volume read-only into every runner and set the env var; - let the pruner fill it hourly, authenticated as the GitHub App, from the `uses:` refs in PREFETCH_SOURCES plus PREFETCH_EXTRA (opt-in, skipped when neither is set), with an LRU cap; - `pruner.sh prefetch` warms it once after a rollout. Make the setup usable by any organization: neutral defaults (LABELS=docker, RUNNER_IMAGE overridable, empty PREFETCH_SOURCES), one generic .env.example, hosts/no-caches.compose.yml for small hosts, and a shorter README that explains how to fit the setup to different hosts and labels. Co-Authored-By: Claude Opus 5.5 --- .env.example | 29 +-- README.md | 411 +++++++++++++----------------------- cleanup.sh | 2 +- docker-compose.yml | 46 ++-- hosts/beelink.env.example | 18 -- hosts/hetzner.env.example | 20 -- hosts/no-caches.compose.yml | 24 +++ hosts/rpi5.compose.yml | 27 --- hosts/rpi5.env.example | 22 -- pruner.sh | 114 ++++++++++ 10 files changed, 328 insertions(+), 385 deletions(-) delete mode 100644 hosts/beelink.env.example delete mode 100644 hosts/hetzner.env.example create mode 100644 hosts/no-caches.compose.yml delete mode 100644 hosts/rpi5.compose.yml delete mode 100644 hosts/rpi5.env.example diff --git a/.env.example b/.env.example index f63588a..99d6cc1 100644 --- a/.env.example +++ b/.env.example @@ -1,27 +1,28 @@ -# Copy to .env and fill in. Never commit .env. -# Variables interpolated by docker-compose.yml. Per-host examples with the -# target labels live in hosts/ (hetzner, beelink, rpi5). +# Copy to .env next to docker-compose.yml and fill in. Never commit .env. # GitHub App that registers the runners (required) APP_ID= APP_PRIVATE_KEY= -# Org login the runners register against (also used as ORG_NAME) +# Organization the runners register to APP_LOGIN= -# Optional — values match the compose defaults +# Runner identity. The runner adds self-hosted, the OS and the architecture to +# LABELS itself; add the capability labels your workflows select on. RUNNER_NAME_PREFIX=runner -LABELS=self-hosted,Linux,X64,docker,x86,large,light,beelink +LABELS=docker RUNNER_GROUP=Default -# Runner self-update. The upstream entrypoint tests this with `[ -n ... ]`, -# so ANY non-empty value disables auto-update - "false" included. Leave it -# unset/empty to keep auto-update on; set it to e.g. "true" to turn it off. -# DISABLE_AUTO_UPDATE=true +# Small hosts without room for per-runner caches (e.g. ARM64 boards): +# COMPOSE_FILE=docker-compose.yml:hosts/no-caches.compose.yml -# EPHEMERAL is hard-coded to "true" in docker-compose.yml (each runner takes -# one job, then a fresh container registers again; the caches live on the -# named volumes). Setting it here has no effect. +# Action archive cache (see README). Space-separated owner/repo@ref lists. +# PREFETCH_SOURCES=my-org/my-workflows@main # repos scanned for `uses:` +# PREFETCH_EXTRA=actions/checkout@v4 # actions fetched as-is -# Nightly pruner (see pruner.sh) +# Other tunables (defaults shown) +# RUNNER_IMAGE=ghcr.io/dodi-smart/github-runner # CACHE_CAP_BYTES=21474836480 # PRUNE_INTERVAL_SECONDS=86400 +# PREFETCH_INTERVAL_SECONDS=3600 +# Any non-empty value disables runner self-update ("false" too). +# DISABLE_AUTO_UPDATE=true diff --git a/README.md b/README.md index c74c492..2918c53 100644 --- a/README.md +++ b/README.md @@ -1,296 +1,175 @@ # docker-github-actions-runner-android -Based off https://github.com/myoung34/docker-github-actions-runner but adds the -android sdk into the container so we don't need to run the setup-android action -every time from: https://github.com/android-actions/setup-android -The Android SDK requires Java, so the container includes a Temurin JDK as well, -which means you can also skip the setup-java action: -https://github.com/actions/setup-java +A self-hosted GitHub Actions runner image, plus a Docker Compose setup for +running it at scale. The image builds on +[myoung34/github-runner](https://github.com/myoung34/docker-github-actions-runner) +and adds: -Node.js LTS is also baked in (see `matrix.json` for available versions), so -`actions/setup-node` is unnecessary for LTS workflows. +- a Temurin JDK, the Android SDK and Node.js LTS, so jobs can skip + `setup-java`, `setup-android` and `setup-node`; +- multi-arch images (`linux/amd64`, `linux/arm64`). -Images are published as multi-arch manifests for both `linux/amd64` and -`linux/arm64`, so Apple Silicon Macs (via Docker Desktop / OrbStack) and ARM -Linux hosts run natively without emulation. +The Compose setup adds: -To run the container, check out the environment variables from the base image: -https://github.com/myoung34/docker-github-actions-runner#environment-variables +- ephemeral runners with persistent cache volumes; +- a `pruner` service that does the maintenance; +- a local action archive cache that avoids GitHub's download rate limits. -[`docker-compose.yml`](docker-compose.yml) is the X64 layout (beelink by -default; hetzner uses the same file with its own `.env`) — three ephemeral -runners, each with its own work **and** cache volumes, plus a nightly -`pruner`. Per-host `.env` examples with the target labels are in -[`hosts/`](hosts/). After pulling a new image on a host: +## Quick start -``` -docker compose pull && docker compose up -d -``` +1. Create a GitHub App for runner registration. See the + [base image docs](https://github.com/myoung34/docker-github-actions-runner#environment-variables). +2. Copy [`.env.example`](.env.example) to `.env` and set `APP_ID`, + `APP_PRIVATE_KEY` and `APP_LOGIN`. +3. Start the runners: + ```bash + docker compose up -d + ``` -Existing named `runner-work*` volumes keep their data; new cache volumes -start empty and warm on first jobs. Do not share one bun/gradle/mise volume -across the three containers: concurrent jobs would corrupt the same tree. - -## Runner pools and labels - -Jobs pick runners by capability label, never by machine name. Each host's -`.env` sets `LABELS` (the runner binary adds `self-hosted`, the OS and the -architecture itself): - -| Pool | `.env` example | `LABELS` | Caches | -|------|----------------|----------|--------| -| hetzner (X64) | [`hosts/hetzner.env.example`](hosts/hetzner.env.example) | `self-hosted,Linux,X64,docker,x86,light,agent,hetzner` | all volumes | -| beelink (X64) | [`hosts/beelink.env.example`](hosts/beelink.env.example) | `self-hosted,Linux,X64,docker,x86,large,light,beelink` | all volumes | -| rpi5 (ARM64) | [`hosts/rpi5.env.example`](hosts/rpi5.env.example) | `self-hosted,Linux,ARM64,docker,agent,rpi5` | work only | - -| Capability label | Meaning | -|------------------|---------| -| `light` | default checks, release, deploy steps (X64 only: no rpi5) | -| `large` | heavy jobs: Gradle/Android, Next.js builds (beelink) | -| `agent` | review, triage, assist (hetzner and rpi5) | - -The rpi5 pool is an agent/utility pool: it has no `light` label and must not -mount the heavy cache volumes. Its `.env` example sets -`COMPOSE_FILE=docker-compose.yml:hosts/rpi5.compose.yml`, which layers -[`hosts/rpi5.compose.yml`](hosts/rpi5.compose.yml) over the base file so each -runner keeps only `docker.sock` and its work volume (and the pruner only -prunes Docker). - -### Rolling out a label change - -Runners are ephemeral and re-register on their next job, so changing labels is -a `.env` edit plus a recreate: - -1. Copy the host's `hosts/.env.example` values into its `.env` (keep the - real `APP_*` values). On rpi5 also keep the `COMPOSE_FILE` line. -2. `docker compose pull && docker compose up -d`. -3. Check the pool in the org's runner list (Settings, Actions, Runners): each - runner should show the labels above. `docker compose ps` shows `healthy` - once `Runner.Listener` is up. - -## Ephemeral runners - -`EPHEMERAL` is `"true"` in the compose file. The upstream entrypoint -(`myoung34/github-runner`) only tests `[ -n "${EPHEMERAL}" ]` (and the same for -`DISABLE_AUTO_UPDATE`), so any non-empty value, `"false"` included, passes -`--ephemeral` to `config.sh`. The previous `EPHEMERAL: "false"` therefore -never kept runners persistent: they have always been ephemeral, one job per -container, restarted by `restart: unless-stopped`. The compose file now says -what actually happens. Each job gets a clean container; the caches survive on -the named volumes below. - -`DISABLE_AUTO_UPDATE` is empty by default. Set it to any non-empty value in -`.env` to disable runner self-update (`false` disables it too). - -## Health check - -Each runner reports healthy while `Runner.Listener` is alive -(`pgrep -f Runner.Listener`, exec form: under `CMD-SHELL` the wrapping -`sh -c` would match its own command line). The container restarts after each -ephemeral job; `start_period` (120 s) keeps the re-registration from counting -as failures. - -## Image identity - -The image carries `RUNNER_IMAGE_TAG` (the fully pinned tag, e.g. -`2.338.0-jdk21-sdk36-node24`) and `RUNNER_IMAGE_REVISION` (the commit it was -built from) as environment variables, set from build args by `deploy.yml` -(`build.yml` sets `pr-` and the head SHA). Jobs can print them to tell -which image they ran on. Local builds default to `dev`. - -## X64 cache volumes - -This compose is for the **X64** class (`beelink`, `hetzner`). The `LABELS` -default is the beelink set; hosts override it in `.env` (see the table above). -The runners are ephemeral, but the volumes survive between containers and -jobs. - -Each runner gets its own: - -| Mount | Volume (runner / runner-2 / runner-3) | Why | Capped by pruner | -|-------|----------------------------------------|-----|------------------| -| `/tmp/runner/work` | `runner-work*` | Job workspace. Cleanup spares `$RUNNER_WORKDIR`. | no | -| `/root/.bun` | `runner-bun*` | Bun install cache | yes | -| `/root/.npm` | `runner-npm*` | npm cache | no | -| `/root/.cache/pnpm` | `runner-pnpm*` | pnpm store | no | -| `/root/.gradle` | `runner-gradle*` | Gradle modules, build-cache, transforms | yes | -| `/root/.cargo` | `runner-cargo*` | Cargo registry | no | -| `/root/.rustup` | `runner-rustup*` | Rust toolchains (issue #8) | no | -| `/root/.konan` | `runner-konan*` | Kotlin/Native toolchain and dependencies | no | -| `/root/.local/share/mise` | `runner-mise-data*` | mise tool installs (`MISE_DATA_DIR`) | yes, whole tool versions | -| `/root/.cache/mise` | `runner-mise-cache*` | mise downloads (`MISE_CACHE_DIR`) | yes | -| `/root/.cache/hk` | `runner-hk*` | hk cache, incl. pkl (`HK_CACHE_DIR`, `HK_PKL_CACHE_DIR`) | yes | - -The Dockerfile sets `MISE_DATA_DIR`, `MISE_CACHE_DIR`, `HK_CACHE_DIR` and -`HK_PKL_CACHE_DIR` so the tools write where the volumes are mounted. The -`~/.cache` directory is never mounted as a whole: that would shadow the pnpm -volume, so each subdirectory has its own volume. `jdx/mise-action` should use -`cache: false` on these runners; the volumes replace its GitHub-cache step. - -Budget ~20 GB per capped cache directory (enforced by the nightly pruner with -LRU, not a wipe). The remaining volumes (npm, pnpm, cargo, rustup, konan) are -not capped yet (issue #7 tracks the bun/gradle-only caps; hard-linked stores -need care). ARM64 / Pi hosts should **not** mount these cache volumes — those -runners isolate to `RUNNER_TEMP`, see `hosts/rpi5.compose.yml`. - -## Maintenance: per-job cleanup and the nightly pruner - -`cleanup.sh` (the `JOB_COMPLETED` hook) only does the cheap tidy-up after each -job: `/tmp` outside the workdir, Gradle daemon/journals/locks, runner `_diag` -logs. The expensive scans used to run there on every job and cost 21–39 s each -(issue #16). They now run once a day in the `pruner` compose service, which -uses the same image with [`pruner.sh`](pruner.sh) as its entrypoint and -mounts only the capped cache volumes plus `docker.sock`: - -- LRU size caps (`CACHE_CAP_BYTES`, default 20 GiB) on gradle, bun, the mise - cache and the hk cache, evicting by last access time; mise installs are - evicted as whole `tool/version` directories so a tool is never left half - deleted (`MISE_DATA_CAP_BYTES`). -- `docker system prune -af --filter until=24h` -- `docker builder prune -af --filter until=72h` - -Named volumes are never pruned. The first run is 5 minutes after start -(`PRUNE_INITIAL_DELAY`), then every `PRUNE_INTERVAL_SECONDS` (86400). Run it -once by hand with -`docker compose run --rm -e PRUNE_RUN_ONCE=true pruner`. - -Do **not** tmpfs `/tmp`. `RUNNER_WORKDIR` is `/tmp/runner/work`; a tmpfs -or a blanket `find /tmp -delete` hides or wipes the work volume. - -### Workflows must use the volume homes - -Named volumes are unused if the job relocates the caches. On these X64 -self-hosted runners, workflows must **not** isolate `GRADLE_USER_HOME` or -`BUN_INSTALL_CACHE_DIR` (or `npm_config_cache` / `CARGO_HOME` / -`MISE_DATA_DIR` / `MISE_CACHE_DIR` / `HK_CACHE_DIR`) to -`$RUNNER_TEMP`. That is the `setup-stack` local-home mode in -`dodi-smart/.github` — `deps-verify` stays isolated; `pr-checks` uses the -default homes so the volumes are actually read. - -The other half of cache poison is GitHub `actions/cache` with -`restore-keys` re-importing a partial bun tarball. Use an exact key, or -do not upload the bun install cache at all. +[`docker-compose.yml`](docker-compose.yml) starts three runners and a `pruner`. +Each runner is **ephemeral**: it takes one job, then restarts and registers +again with a clean container. Only the named volumes survive between jobs. -## Image tags +To update a host, run `docker compose pull && docker compose up -d`. -Images are published to GitHub Container Registry (GHCR): -- `ghcr.io/dodi-smart/github-runner` - -### Tag scheme - -Each image is built with a specific combination of base runner version, JDK -version, Node major version, and Android SDK level. Tags let you choose how -tightly to pin: - -| Tag | Example | What it pins | What floats | -|-----|---------|--------------|-------------| -| `latest` | `latest` | Nothing | Runner, JDK, Node, and SDK all track latest | -| `jdk` | `jdk21` | JDK major version | Runner, Node (default), and SDK track latest | -| `node` | `node22` | Node major version | Runner, JDK (default), and SDK track latest | -| `jdk-node` | `jdk21-node22` | JDK + Node | Runner and SDK track latest | -| `-jdk-sdk-node` | `2.332.0-jdk21-sdk36-node22` | Everything | Nothing — fully pinned | - -### Which tag should I use? - -- **`latest`** — always the newest runner, default JDK, default Node LTS, and - latest Android SDK. Good for staying current, but builds may break if a new - SDK introduces incompatibilities. -- **`jdk21`** / **`jdk17`** — pin the JDK version but still get runner, Node, - and SDK updates automatically. A good balance for most users. -- **`jdk21-node22`** — pin both JDK and Node majors; runner and SDK still - track latest. Useful when an LTS-pinned Node is part of your build's - contract. -- **`2.332.0-jdk21-sdk36-node22`** — fully pinned. Use this for reproducible - builds where you want complete control over when to upgrade. Old pinned tags - are never overwritten; they remain in the registry from previous builds. - -### What's in each image? - -Every image includes: -- The [myoung34/github-runner](https://github.com/myoung34/docker-github-actions-runner) base image -- [Temurin JDK](https://adoptium.net/) (the version in the tag) -- [Node.js](https://nodejs.org/) LTS (the version in the tag) — installed - from NodeSource, replacing the base image's older system Node -- Android SDK platform matching the SDK level in the tag (e.g., `sdk36` = `platforms;android-36`) -- Latest Android build-tools and NDK for that SDK level -- Android SDK command-line tools, platform-tools, and cmake -- Multi-arch manifest covering `linux/amd64` and `linux/arm64` - -Projects that need additional SDK platforms (e.g., an older `compileSdk`) can -install them at build time: -```yaml -- run: sdkmanager "platforms;android-28" -``` +## Fit it to your setup -### Currently available JDK versions +**One host or many.** Run the same Compose file on every host and change only +`.env`. To run more or fewer runners per host, add or remove `runner-N` +services, along with their volumes. -The build matrix is defined in [`matrix.json`](matrix.json). Check that file -for the current list of JDK versions and which one is the default. +**Different kinds of machines.** Give each host the capability labels your +workflows select on, through `LABELS` in its `.env`. For example, `build` for +general jobs and `heavy` for memory-hungry ones. Workflows then ask for +`runs-on: [self-hosted, Linux, build]`. Label by what a machine can do, not by +its name. A host can carry several labels, so a large machine labelled +`build,heavy` also picks up general jobs when the others are busy. -### Currently available Node versions +**Small hosts.** Boards such as a Raspberry Pi have little disk. Layer +[`hosts/no-caches.compose.yml`](hosts/no-caches.compose.yml) by setting +`COMPOSE_FILE` in `.env`. Those runners skip the per-runner cache volumes. -Also defined in [`matrix.json`](matrix.json) — see `node_versions` for the -current LTS majors and `default_node` for the version applied to short tags -like `latest` and `jdk`. +**Repository or enterprise runners.** The Compose file registers runners for +an organization (`RUNNER_SCOPE: org`). For other scopes, change the runner +environment to what the +[base image](https://github.com/myoung34/docker-github-actions-runner#environment-variables) +supports. -## Automated updates +**Your own image.** Set `RUNNER_IMAGE` to use a fork or a pinned tag. See +[Image tags](#image-tags). -Versions are kept up to date automatically: +## Caches -| Component | Mechanism | -|-----------|-----------| -| Base runner image | [Renovate](https://github.com/renovatebot/renovate) tracks `myoung34/github-runner` Docker tags | -| GitHub Actions versions | Renovate (`config:recommended`) | -| New JDK major versions | Weekly workflow queries the [Adoptium API](https://api.adoptium.net) and opens a PR | -| New Node LTS majors | Weekly workflow queries the [Node Release schedule](https://github.com/nodejs/Release/blob/main/schedule.json) and opens a PR | -| Android SDK platform level | Weekly workflow queries `sdkmanager` and opens a PR | -| Android build-tools | Weekly workflow queries `sdkmanager` and opens a PR | -| Android NDK | Weekly workflow queries `sdkmanager` and opens a PR | +Each runner gets its own volume for these caches: -Every pull request is validated by [`build.yml`](.github/workflows/build.yml), -which builds each JDK against the default Node, each Node against the default -JDK, and one `linux/arm64` build of the default pair. Its `All builds passed` -job is the single check to require in branch protection. +- bun, npm and pnpm +- Gradle +- Cargo and rustup +- Kotlin/Native +- mise and hk -Because this repository is a fork, `renovate.json` sets -`"forkProcessing": "enabled"`. The Mend-hosted Renovate App leaves fork -processing disabled for org-wide ("All repositories") installations, so without -that flag Renovate silently skips the repository and the base image is never -bumped. +The image sets `MISE_DATA_DIR`, `MISE_CACHE_DIR` and `HK_CACHE_DIR` so those +tools write there. -The weekly SDK/JDK/Node workflow opens its PR with `GITHUB_TOKEN`, which means -GitHub does not run `build.yml` on it. Upstream solves this with an -`AUTOMERGE_PAT` secret and `gh pr merge --auto`; this fork does not, so those -PRs are merged manually and the builds run on merge. +- **Never share a cache volume between runners.** Concurrent jobs would + corrupt it. +- **Keep the default cache locations in workflows.** If a job moves + `GRADLE_USER_HOME` or `BUN_INSTALL_CACHE_DIR` into `$RUNNER_TEMP`, the + volumes go unused. +- **Turn off the GitHub cache where a volume replaces it.** For example, use + `cache: false` on `jdx/mise-action`. +- **Don't put `/tmp` on tmpfs.** The job workspace is `/tmp/runner/work`. -## Runner registration preflight +## Action archive cache -The image wraps the base entrypoint with -[`preflight-entrypoint.sh`](preflight-entrypoint.sh). When -`CONFIGURED_ACTIONS_RUNNER_FILES_DIR` is set, the upstream entrypoint reuses -persisted registration credentials without checking them, so a runner that was -removed on GitHub's side crash-loops forever instead of re-registering. The -preflight asks GitHub whether the persisted runner still exists and wipes the -stored registration only on a definitive `404`, letting a fresh `ACCESS_TOKEN` -registration take over. Timeouts, missing variables, and unparseable state all -leave the configuration untouched. +Ephemeral runners download every `uses:` action on every job. Several runners +behind one IP address soon hit `429 Too Many Requests` in "Set up job". -## Building locally +The runner can read action archives from a local directory before it downloads +anything. Set `ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE` to that directory. Here it +is `/opt/actionarchivecache`, mounted read-only into every runner. The runner +never writes this directory itself, so the `pruner` fills it every hour. It +uses the GitHub App's token, so the downloads count against the App's rate +limit rather than your IP address. It fetches: +- every `uses:` found in the repositories listed in `PREFETCH_SOURCES`, such as + the repo that holds your reusable workflows; +- the actions listed in `PREFETCH_EXTRA`. + +With neither variable set, the cache stays empty and jobs download as usual. A +cache miss always falls back to a normal download. + +To fill the cache straight away: + +```bash +docker compose run --rm pruner prefetch ``` -docker build -f Dockerfile . -``` -Build args can be used to customize the image. See the top of the -[`Dockerfile`](Dockerfile) for the current defaults and available args -(`VERSION`, `JAVA_VERSION`, `NODE_VERSION`, `COMPILE_SDK`, `BUILD_TOOLS`, -`NDK_VERSION`, `SDK_TOOLS`, `RUNNER_IMAGE_TAG`, `RUNNER_IMAGE_REVISION`). +To confirm it works, re-run a job with debug logging. "Set up job" then shows +`Copied action archive '/opt/actionarchivecache/...'`. + +## Maintenance + +The `pruner` service ([`pruner.sh`](pruner.sh)) keeps slow work off the job +path: + +- **Nightly:** trims the Gradle, bun, mise and hk caches to `CACHE_CAP_BYTES` + each (default 20 GiB), dropping the least recently used files first. It then + runs `docker system prune --filter until=24h` and + `docker builder prune --filter until=72h`. Named volumes are never pruned. +- **Hourly:** fills the action archive cache, capped at 2 GiB. + +After each job, [`cleanup.sh`](cleanup.sh) does only a quick tidy-up: `/tmp`, +Gradle daemons and locks, and runner logs. + +To run the nightly pass once by hand: -Example: +```bash +docker compose run --rm -e PRUNE_RUN_ONCE=true pruner ``` -docker build --build-arg JAVA_VERSION=17 --build-arg NODE_VERSION=22 --build-arg COMPILE_SDK=35 -f Dockerfile . + +## Image tags + +Images are published to `ghcr.io/dodi-smart/github-runner`. + +| Tag | Pins | +|---|---| +| `latest` | Nothing | +| `jdk21` | JDK major | +| `node22` | Node major | +| `jdk21-node22` | JDK and Node majors | +| `2.338.0-jdk21-sdk36-node24` | Everything: runner, JDK, SDK and Node. Never overwritten | + +[`matrix.json`](matrix.json) lists the available JDK and Node versions and the +defaults. Each image sets `RUNNER_IMAGE_TAG` and `RUNNER_IMAGE_REVISION`, so a +job can tell which image it ran on. + +To use another Android platform, install it in the job: +`sdkmanager "platforms;android-28"`. + +## Notes + +- `EPHEMERAL` and `DISABLE_AUTO_UPDATE` count as set for any non-empty value, + `"false"` included. The upstream entrypoint only checks whether they are + empty. +- [`preflight-entrypoint.sh`](preflight-entrypoint.sh) wipes a stored runner + registration only after GitHub confirms that runner is gone (`404`). A + removed runner then registers again instead of crash-looping. + +## Development + +[`build.yml`](.github/workflows/build.yml) builds every pull request. Its +`All builds passed` check is the one to require. + +Renovate keeps the base image and the actions up to date; `forkProcessing` is +on because this repo is a fork. A weekly workflow opens pull requests for new +JDK, Node LTS, Android SDK, build-tools and NDK versions. + +To build locally: + +```bash +docker build --build-arg JAVA_VERSION=17 --build-arg NODE_VERSION=22 . ``` -## Inspiration -- https://github.com/kriskda/docker-github-action-android-container (comes with emulator which we don't need) -- https://github.com/jordond/docker-android-github-runner (desire more configurability - re:version of myoung34 container, etc) +The build args and their defaults are at the top of the +[`Dockerfile`](Dockerfile). diff --git a/cleanup.sh b/cleanup.sh index 5fbfef3..29dfb19 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -36,7 +36,7 @@ _ancestor_of() { return 1 } -# RUNNER_WORKDIR is /tmp/runner/work on the beelinks. A blanket +# RUNNER_WORKDIR is /tmp/runner/work in docker-compose.yml. A blanket # `find /tmp -delete` wipes the work volume. Spare the workdir (and # RUNNER_TEMP if it sits under /tmp), plus their ancestors under /tmp. # diff --git a/docker-compose.yml b/docker-compose.yml index 7991910..12a4542 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,11 @@ -# X64 runner layout (beelink by default; hetzner uses the same file with its own -# .env, see hosts/): three runners, each with its own work and cache volumes, -# plus a nightly `pruner`. Do not share bun/gradle/npm/cargo/mise volumes -# across runner services - concurrent jobs would corrupt the same tree. YAML -# merge of volumes: replaces the list, so docker.sock is repeated on each -# service. ARM64 / rpi5 hosts layer hosts/rpi5.compose.yml on top (no heavy -# cache volumes), see README. +# Three ephemeral runners, each with its own work and cache volumes, plus a +# `pruner` for maintenance. Never share a cache volume between runners: +# concurrent jobs would corrupt it. The action archive cache is the exception: +# read-only in every runner, filled by the pruner. `volumes:` lists replace on +# merge, so docker.sock is repeated per service. Small hosts can layer +# hosts/no-caches.compose.yml on top (see README). x-runner: &runner - image: ghcr.io/dodi-smart/github-runner + image: ${RUNNER_IMAGE:-ghcr.io/dodi-smart/github-runner} restart: unless-stopped environment: APP_ID: ${APP_ID} @@ -16,7 +15,7 @@ x-runner: &runner RUNNER_WORKDIR: /tmp/runner/work RUNNER_SCOPE: org ORG_NAME: ${APP_LOGIN} - LABELS: ${LABELS:-self-hosted,Linux,X64,docker,x86,large,light,beelink} + LABELS: ${LABELS:-docker} RUNNER_GROUP: ${RUNNER_GROUP:-Default} # The upstream entrypoint only tests these with `[ -n "$VAR" ]`, so ANY # non-empty value - including "false" - turns the flag on. @@ -31,6 +30,12 @@ x-runner: &runner # non-empty value to disable updates; `DISABLE_AUTO_UPDATE=false` disables # them too. See .env.example. DISABLE_AUTO_UPDATE: ${DISABLE_AUTO_UPDATE:-} + # Ephemeral runners re-download every action archive for every job, and the + # few public IPs of these hosts hit codeload.github.com's 429 limit. The + # runner (>= 2.319) reads /_/.tar.gz before it + # downloads, and only reads: the `pruner` service fills the shared + # runner-action-cache volume (see pruner.sh), mounted read-only below. + ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE: /opt/actionarchivecache # Runner.Listener is the long-lived process of a registered runner. Exec form # on purpose: under CMD-SHELL the wrapping `sh -c` carries the pattern in its # own command line and `pgrep -f` would match itself, healthy forever. The @@ -63,6 +68,7 @@ services: - runner-mise-data:/root/.local/share/mise - runner-mise-cache:/root/.cache/mise - runner-hk:/root/.cache/hk + - runner-action-cache:/opt/actionarchivecache:ro runner-2: <<: *runner volumes: @@ -78,6 +84,7 @@ services: - runner-mise-data-2:/root/.local/share/mise - runner-mise-cache-2:/root/.cache/mise - runner-hk-2:/root/.cache/hk + - runner-action-cache:/opt/actionarchivecache:ro runner-3: <<: *runner volumes: @@ -93,19 +100,22 @@ services: - runner-mise-data-3:/root/.local/share/mise - runner-mise-cache-3:/root/.cache/mise - runner-hk-3:/root/.cache/hk - # Nightly maintenance, off the job path (issue #16: the same scans inside the - # per-job cleanup.sh cost 21-39 s on every job). Same image, pruner.sh as the - # entrypoint. It needs only the caches it caps (not work/npm/pnpm/...), and - # docker.sock to prune the host daemon: `docker system prune -af --filter - # until=24h` and `docker builder prune -af --filter until=72h`. Named volumes - # are never pruned. Tunables: PRUNE_INTERVAL_SECONDS, CACHE_CAP_BYTES, ... - # (see pruner.sh). + - runner-action-cache:/opt/actionarchivecache:ro + # Maintenance off the job path (pruner.sh): nightly cache caps and docker + # prune, hourly action-cache fill. Mounts only what it caps. pruner: - image: ghcr.io/dodi-smart/github-runner + image: ${RUNNER_IMAGE:-ghcr.io/dodi-smart/github-runner} restart: unless-stopped entrypoint: ["/usr/local/bin/pruner.sh"] environment: CACHES_ROOT: /caches + APP_ID: ${APP_ID} # App token for authenticated action downloads + APP_PRIVATE_KEY: ${APP_PRIVATE_KEY} + APP_LOGIN: ${APP_LOGIN} + ACTION_CACHE_CAP_BYTES: ${ACTION_CACHE_CAP_BYTES:-2147483648} + PREFETCH_SOURCES: ${PREFETCH_SOURCES:-} + PREFETCH_EXTRA: ${PREFETCH_EXTRA:-} + PREFETCH_INTERVAL_SECONDS: ${PREFETCH_INTERVAL_SECONDS:-3600} CACHE_CAP_BYTES: ${CACHE_CAP_BYTES:-21474836480} PRUNE_INTERVAL_SECONDS: ${PRUNE_INTERVAL_SECONDS:-86400} volumes: @@ -125,10 +135,12 @@ services: - runner-mise-cache-3:/caches/3/mise-cache - runner-hk-3:/caches/3/hk - runner-mise-data-3:/caches/3/mise-data + - runner-action-cache:/opt/actionarchivecache security_opt: - label:disable volumes: + runner-action-cache: {} # read-only in runners, filled by the pruner runner-work: {} runner-work-2: {} runner-work-3: {} diff --git a/hosts/beelink.env.example b/hosts/beelink.env.example deleted file mode 100644 index 682fd4d..0000000 --- a/hosts/beelink.env.example +++ /dev/null @@ -1,18 +0,0 @@ -# Host .env for the beelink X64 pool (3 runners, the default docker-compose.yml). -# Copy to .env next to docker-compose.yml. Never commit .env. -# -# Target labels (CI redesign): -# large -> heavy jobs (Gradle/Android, Next.js builds) -# light -> overflow for light jobs, resolved by GitHub's own queue -# The runner binary adds self-hosted/Linux/X64 itself; they are listed to keep -# the whole set visible in one place. -APP_ID= -APP_PRIVATE_KEY= -APP_LOGIN= - -RUNNER_NAME_PREFIX=beelink -LABELS=self-hosted,Linux,X64,docker,x86,large,light,beelink -RUNNER_GROUP=Default - -# Runners are ephemeral (set in docker-compose.yml). Leave DISABLE_AUTO_UPDATE -# unset: ANY non-empty value, even "false", disables runner self-update. diff --git a/hosts/hetzner.env.example b/hosts/hetzner.env.example deleted file mode 100644 index 1101a97..0000000 --- a/hosts/hetzner.env.example +++ /dev/null @@ -1,20 +0,0 @@ -# Host .env for the hetzner X64 pool. If a host runs more than one stack, give -# each stack its own RUNNER_NAME_PREFIX. -# Copy to .env next to docker-compose.yml. Never commit .env. -# -# Target labels (CI redesign): capability labels the GitHub queue resolves. -# light -> default checks, release, deploy steps -# agent -> review, triage, assist -# No `large`: heavy jobs (Gradle/Android, Next.js builds) go to the beelinks. -# The runner binary adds self-hosted/Linux/X64 itself; they are listed to keep -# the whole set visible in one place. -APP_ID= -APP_PRIVATE_KEY= -APP_LOGIN= - -RUNNER_NAME_PREFIX=hetzner-a -LABELS=self-hosted,Linux,X64,docker,x86,light,agent,hetzner -RUNNER_GROUP=Default - -# Runners are ephemeral (set in docker-compose.yml). Leave DISABLE_AUTO_UPDATE -# unset: ANY non-empty value, even "false", disables runner self-update. diff --git a/hosts/no-caches.compose.yml b/hosts/no-caches.compose.yml new file mode 100644 index 0000000..4f7cc25 --- /dev/null +++ b/hosts/no-caches.compose.yml @@ -0,0 +1,24 @@ +# For small hosts (e.g. ARM64 boards with little disk): no per-runner cache +# volumes, only docker.sock, the work volume and the action archive cache. +# COMPOSE_FILE=docker-compose.yml:hosts/no-caches.compose.yml (in .env) +# `!override` replaces the base volume lists instead of merging into them. +services: + runner: + volumes: !override + - /var/run/docker.sock:/var/run/docker.sock + - runner-work:/tmp/runner/work + - runner-action-cache:/opt/actionarchivecache:ro + runner-2: + volumes: !override + - /var/run/docker.sock:/var/run/docker.sock + - runner-work-2:/tmp/runner/work + - runner-action-cache:/opt/actionarchivecache:ro + runner-3: + volumes: !override + - /var/run/docker.sock:/var/run/docker.sock + - runner-work-3:/tmp/runner/work + - runner-action-cache:/opt/actionarchivecache:ro + pruner: + volumes: !override + - /var/run/docker.sock:/var/run/docker.sock + - runner-action-cache:/opt/actionarchivecache diff --git a/hosts/rpi5.compose.yml b/hosts/rpi5.compose.yml deleted file mode 100644 index 4fa7fe7..0000000 --- a/hosts/rpi5.compose.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Layer on top of docker-compose.yml for the ARM64 rpi5 hosts: -# -# docker compose -f docker-compose.yml -f hosts/rpi5.compose.yml up -d -# -# hosts/rpi5.env.example sets COMPOSE_FILE so a plain `docker compose ...` -# does exactly that. The rpi5 pool is the agent/utility pool: it must not -# mount the heavy X64 cache volumes (bun, npm, pnpm, gradle, cargo, rustup, -# konan, mise, hk) - those runners isolate to RUNNER_TEMP and the SD/NVMe is -# small. Each runner keeps only docker.sock and its work volume, and the -# pruner (which only caps cache volumes) does docker pruning alone. -# `!override` replaces the base volume lists instead of merging into them. -services: - runner: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock - - runner-work:/tmp/runner/work - runner-2: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock - - runner-work-2:/tmp/runner/work - runner-3: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock - - runner-work-3:/tmp/runner/work - pruner: - volumes: !override - - /var/run/docker.sock:/var/run/docker.sock diff --git a/hosts/rpi5.env.example b/hosts/rpi5.env.example deleted file mode 100644 index 325e23b..0000000 --- a/hosts/rpi5.env.example +++ /dev/null @@ -1,22 +0,0 @@ -# Host .env for the rpi5 ARM64 pool (agent and utility runners). -# Copy to .env next to docker-compose.yml. Never commit .env. -# -# Target labels (CI redesign): agent + utility pool. NOT `light` (light jobs -# assume X64: Flutter, bun, zavet failed when ARM64 runners carried it) and -# NOT `large`. -# The runner binary adds self-hosted/Linux/ARM64 itself; they are listed to -# keep the whole set visible in one place. -APP_ID= -APP_PRIVATE_KEY= -APP_LOGIN= - -RUNNER_NAME_PREFIX=rpi5 -LABELS=self-hosted,Linux,ARM64,docker,agent,rpi5 -RUNNER_GROUP=Default - -# Do not mount the heavy cache volumes on the Pi: layer the override that -# keeps only docker.sock + the work volume (see hosts/rpi5.compose.yml). -COMPOSE_FILE=docker-compose.yml:hosts/rpi5.compose.yml - -# Runners are ephemeral (set in docker-compose.yml). Leave DISABLE_AUTO_UPDATE -# unset: ANY non-empty value, even "false", disables runner self-update. diff --git a/pruner.sh b/pruner.sh index 58f6998..8b701ae 100755 --- a/pruner.sh +++ b/pruner.sh @@ -8,6 +8,14 @@ # 1. LRU size caps on the per-runner cache volumes (mounted under # $CACHES_ROOT//, see docker-compose.yml). # 2. docker system prune / builder prune on the host daemon (docker.sock). +# 3. Hourly, in the background: fill the shared action archive cache. +# +# Action archive cache: the runner copies +# $ACTION_CACHE_DIR/_/.tar.gz instead of downloading when it +# exists, but never writes it. prefetch_actions() fills it as the GitHub App, so +# downloads count against the App rather than the host IP (codeload 429s). +# Refs: every `uses:` found in the PREFETCH_SOURCES repos, plus PREFETCH_EXTRA. +# `pruner.sh prefetch` runs it once (rollout warm-up). # # Env: # PRUNE_INTERVAL_SECONDS sleep between runs (default 86400) @@ -17,6 +25,9 @@ # CACHES_ROOT where the cache volumes are mounted (/caches) # CACHE_CAP_BYTES cap for each file-level cache dir (20 GiB) # MISE_DATA_CAP_BYTES cap for each mise installs tree (20 GiB) +# ACTION_CACHE_CAP_BYTES cap for the action archive cache (2 GiB) +# PREFETCH_SOURCES / PREFETCH_EXTRA "owner/repo@ref ..." to scan / fetch as-is +# PREFETCH_INTERVAL_SECONDS prefetch period (default 3600; 0 = off) # DOCKER_PRUNE_UNTIL / DOCKER_BUILDER_PRUNE_UNTIL (24h / 72h) # # Intentionally not using set -e: one failing step must not stop the others. @@ -24,6 +35,11 @@ CACHES_ROOT="${CACHES_ROOT:-/caches}" CACHE_CAP_BYTES="${CACHE_CAP_BYTES:-21474836480}" # 20 GiB MISE_DATA_CAP_BYTES="${MISE_DATA_CAP_BYTES:-21474836480}" # 20 GiB +ACTION_CACHE_DIR=/opt/actionarchivecache # mount point in docker-compose.yml +ACTION_CACHE_CAP_BYTES="${ACTION_CACHE_CAP_BYTES:-2147483648}" # 2 GiB +PREFETCH_SOURCES="${PREFETCH_SOURCES:-}" +PREFETCH_EXTRA="${PREFETCH_EXTRA:-}" +GITHUB_API="https://api.github.com" DOCKER_PRUNE_UNTIL="${DOCKER_PRUNE_UNTIL:-24h}" DOCKER_BUILDER_PRUNE_UNTIL="${DOCKER_BUILDER_PRUNE_UNTIL:-72h}" @@ -104,6 +120,89 @@ prune_caches() { lru_cap "$runner/hk" lru_cap_mise_installs "$runner/mise-data" done + lru_cap "$ACTION_CACHE_DIR" "$ACTION_CACHE_CAP_BYTES" +} + +gh_curl() { + curl -fsSL --retry 3 --connect-timeout 10 --max-time 180 \ + -H "Authorization: Bearer ${GH_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" "$@" +} + +# Print "owner/repo ref" for every remote action under tree $1. Skips local +# (./), docker:// and reusable-workflow (.github/workflows/) refs, which the +# runner never fetches as an action archive; subpath actions (owner/repo/sub) +# share the owner/repo archive. +action_refs_from_tree() { + grep -rhE --include='*.yml' --include='*.yaml' '^[[:space:]-]*uses:[[:space:]]' "$1" 2>/dev/null | + sed -E 's/^[[:space:]-]*uses:[[:space:]]*//; s/["'"'"']//g; s/[[:space:]#].*$//' | + awk -F@ 'NF == 2 && $1 !~ /^(\.|docker:)/ && $1 !~ /[$]/ && $1 !~ /\.github\/workflows\// { + split($1, p, "/"); if (p[2] != "") print p[1] "/" p[2], $2 }' +} + +# Fetch the archive of $1 (owner/repo) at $2 (tag, branch or sha) into the +# cache as _/.tar.gz, via temp file + rename so a runner never +# reads a partial file. Prints 1 when it downloaded something. +prefetch_one() { + local name="$1" ref="$2" sha="$2" dir file tmp + dir="$ACTION_CACHE_DIR/${name//\//_}" + if ! [[ "$sha" =~ ^[0-9a-f]{40}$ ]]; then + sha=$(gh_curl -H 'Accept: application/vnd.github.sha' "$GITHUB_API/repos/$name/commits/$ref") && + [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || { log "prefetch: cannot resolve $name@$ref" >&2; return 1; } + fi + file="$dir/$sha.tar.gz" + [ -s "$file" ] && return 0 + mkdir -p "$dir" && tmp=$(mktemp "$dir/.tmp.XXXXXX") || return 1 + if gh_curl -o "$tmp" "$GITHUB_API/repos/$name/tarball/$sha" && tar -tzf "$tmp" >/dev/null 2>&1; then + chmod 644 "$tmp" && mv -f "$tmp" "$file" && { log "prefetch: cached $name@$sha ($ref)" >&2; echo 1; return 0; } + fi + rm -f "$tmp" + log "prefetch: download failed for $name@$sha" >&2 + return 1 +} + +prefetch_actions() { + local work src refs name ref out n=0 fetched=0 failed=0 token + [ -n "$PREFETCH_SOURCES$PREFETCH_EXTRA" ] || return 0 # cache not configured + if [ -z "${APP_ID:-}" ] || [ -z "${APP_PRIVATE_KEY:-}" ] || [ -z "${APP_LOGIN:-}" ]; then + log "prefetch: APP_ID/APP_PRIVATE_KEY/APP_LOGIN not set; skipping" + return 0 + fi + mkdir -p "$ACTION_CACHE_DIR" || return 0 + # Same \n unescaping as the upstream entrypoint applies to APP_PRIVATE_KEY. + token=$(APP_PRIVATE_KEY="${APP_PRIVATE_KEY//\\n/$'\n'}" bash /app_token.sh 2>/dev/null) + if [ -z "$token" ] || [ "$token" = null ]; then + log "prefetch: could not mint an installation token; skipping" + return 0 + fi + GH_TOKEN="$token" + work=$(mktemp -d) || return 0 + : > "$work/refs" + for src in $PREFETCH_SOURCES; do + mkdir -p "$work/src" + if gh_curl -o "$work/src.tgz" "$GITHUB_API/repos/${src%@*}/tarball/${src#*@}" && + tar -xzf "$work/src.tgz" -C "$work/src"; then + action_refs_from_tree "$work/src" >> "$work/refs" + else + log "prefetch: cannot read source $src" + fi + rm -rf "$work/src" "$work/src.tgz" + done + for src in $PREFETCH_EXTRA; do echo "${src%@*} ${src#*@}" >> "$work/refs"; done + refs=$(sort -u "$work/refs") + rm -rf "$work" + while read -r name ref; do + [ -n "$name" ] && [ -n "$ref" ] || continue + n=$((n + 1)) + if out=$(prefetch_one "$name" "$ref"); then + [ "$out" = 1 ] && fetched=$((fetched + 1)) + else + failed=$((failed + 1)) + fi + done <<< "$refs" + # Stale temp files from an interrupted download. + find "$ACTION_CACHE_DIR" -name '.tmp.*' -mmin +60 -delete 2>/dev/null + log "prefetch: $n refs, $fetched downloaded, $failed failed" + GH_TOKEN="" } prune_docker() { @@ -126,11 +225,26 @@ run_once() { log "done. Disk free: ${before} -> ${after}" } +if [ "${1:-}" = prefetch ]; then + prefetch_actions + exit 0 +fi + if [ "${PRUNE_RUN_ONCE:-}" = "true" ]; then run_once exit 0 fi +if [ "${PREFETCH_INTERVAL_SECONDS:-3600}" -gt 0 ]; then + ( + sleep 30 + while true; do + prefetch_actions + sleep "${PREFETCH_INTERVAL_SECONDS:-3600}" + done + ) & +fi + sleep "${PRUNE_INITIAL_DELAY:-300}" while true; do run_once