Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/fork-ci-tests.txt
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ fork-ci/actions/pinned/pinned.test.py
fork-ci/actions/pr-queue/lens.test.py
fork-ci/actions/pr-queue/pr_queue.test.py
fork-ci/actions/pr-queue/tested-base/tested_base.test.py
fork-kit/common/tests/free_contract.test.py
fork-kit/common/tests/machine_file.test.py
fork-kit/desk/tests/install.test.py
fork-kit/desk/tests/windows.test.py
Expand Down
19 changes: 14 additions & 5 deletions .github/scripts/fork-fleet-launch.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
# TEMPLATE and swaps the two by one rename under the launch lock. Running jobs keep their copies; any
# failure leaves the old template. Below.
# active the NAME of every fork-fleet-* unit that is loaded (starting, running or stopping).
# fits MB exit 0 when MemAvailable minus RESERVE_MB leaves MB megabytes.
# fits MB exit 0 when MemAvailable minus RESERVE_MB leaves MB megabytes and, with FREE_COMMAND set, that free
# script answers free (free/v1, fork-kit/common/FREE.md: run by FREE_CHECK, its reader).
# stop NAME stops that unit (a job on it fails: its runner is gone).
# prune removes the copies and environment files whose unit is gone.
# refresh-net every dispatcher pass: looks this host's own public address up again and, when what is blocked would
Expand Down Expand Up @@ -61,7 +62,8 @@
# fork-fleet-job-paths.txt beside this launcher), ALLOW_PATHS (more paths jobs see read-only, comma-separated, say a
# tool cache: empty by default), IP_FORWARD, SYSTEMD_RUN, SYSTEMCTL, RUNNER_RELEASE (refresh's release record, an https
# URL), RUNNER_DOWNLOAD (the https base of its tarballs) and RUNNER_MIN (the oldest runner version refresh may install,
# empty for none) have defaults below.
# empty for none), FREE_COMMAND (a free/v1 script's absolute path, empty for none) and FREE_CHECK (its reader,
# free_contract.py beside this launcher) have defaults below.
# HOOK (and the two files beside it), WORK_ROOT and RESOLV_CONF are bound into every job too: never under a folder that
# holds data (/home, /root, /mnt, /media, /srv), also through a symlink (no_data). A bind mount of a data folder at
# one of them is not seen (it has no link to resolve): that is the operator's to avoid (.github/fork-fleet.md, step 4).
Expand Down Expand Up @@ -116,8 +118,8 @@ LINGER_DIR=/var/lib/systemd/linger RESERVE_MB=2048 MAX_SECONDS=3600 MEMINFO=/pro
RESOLV_CONF=/etc/fork-fleet/resolv.conf IP_FORWARD=/proc/sys/net/ipv4/ip_forward SYSTEMD_RUN=systemd-run
SYSTEMCTL=systemctl OWN_ADDRESS_URLS=https://api.ipify.org,https://checkip.amazonaws.com ALLOW_PATHS=''
JOB_PATHS='' RUNNER_RELEASE=https://api.github.com/repos/actions/runner/releases/latest
RUNNER_DOWNLOAD=https://github.com/actions/runner/releases/download RUNNER_MIN=''
setting='^(RUNNER_USERS|TEMPLATE|WORK_ROOT|RUN_DIR|HOOK|LINGER_DIR|RESERVE_MB|MAX_SECONDS|MEMINFO|NET_BASE|RESOLV_CONF|IP_FORWARD|SYSTEMD_RUN|SYSTEMCTL|OWN_ADDRESS_URLS|JOB_PATHS|ALLOW_PATHS|RUNNER_RELEASE|RUNNER_DOWNLOAD|RUNNER_MIN)=([^[:space:]]*)$'
RUNNER_DOWNLOAD=https://github.com/actions/runner/releases/download RUNNER_MIN='' FREE_COMMAND='' FREE_CHECK=''
setting='^(RUNNER_USERS|TEMPLATE|WORK_ROOT|RUN_DIR|HOOK|LINGER_DIR|RESERVE_MB|MAX_SECONDS|MEMINFO|NET_BASE|RESOLV_CONF|IP_FORWARD|SYSTEMD_RUN|SYSTEMCTL|OWN_ADDRESS_URLS|JOB_PATHS|ALLOW_PATHS|RUNNER_RELEASE|RUNNER_DOWNLOAD|RUNNER_MIN|FREE_COMMAND|FREE_CHECK)=([^[:space:]]*)$'
while IFS= read -r line || [[ -n $line ]]; do
[[ $line =~ ^[[:space:]]*(#|$) ]] && continue
[[ $line =~ $setting ]] || die "$conf: not a setting: ${line%%=*}"
Expand All @@ -135,6 +137,9 @@ PLAIN_PATH='(/[A-Za-z0-9._@+-]+)+'
if [[ -n $JOB_PATHS ]]; then JOB_PATHS=$ROOT$JOB_PATHS; else
JOB_PATHS=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)/fork-fleet-job-paths.txt
fi
[[ $FREE_COMMAND =~ ^($PLAIN_PATH)?$ && $FREE_CHECK =~ ^($PLAIN_PATH)?$ ]] ||
die "$conf: FREE_COMMAND and FREE_CHECK are plain absolute paths"
[[ -n $FREE_CHECK ]] || FREE_CHECK=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)/free_contract.py

NAME='^[a-z0-9][a-z0-9-]{0,62}-[0-9]{1,20}$'
name_ok() { [[ $1 =~ $NAME ]] || die "not a runner name: $1"; }
Expand Down Expand Up @@ -682,7 +687,11 @@ fits() {
[[ $1 =~ ^[0-9]{1,7}$ ]] || die "fits: not a number of megabytes: $1"
kb=$(awk '$1 == "MemAvailable:" { print $2 }' "$MEMINFO")
[[ $kb =~ ^[0-9]+$ ]] || die "no MemAvailable in $MEMINFO"
((kb / 1024 - RESERVE_MB >= $1))
((kb / 1024 - RESERVE_MB >= $1)) || return 1
[[ -n $FREE_COMMAND ]] || return 0
owned_safe "$FREE_CHECK"
# Its one line (free, busy: why, not available: why) for the dispatcher's log; any exit but 0 is not free.
/usr/bin/python3 -I "$FREE_CHECK" ask --need-mb "$1" "$FREE_COMMAND" >&2
}

# The ranges no job reaches: the host itself is refused on its own (the input chain), these on the way out: RFC 1918,
Expand Down
2 changes: 1 addition & 1 deletion .github/scripts/fork-fleet-walls/wsl2
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ ROOT = os.environ.get('FORK_FLEET_WALLS_ROOT', '')
ME = os.path.realpath(__file__)
# The sha256 of the launcher (fork-fleet-launch.sh) whose job units unit_props reads as safe: the one this file ships
# with. A changed launcher must be read again and this pin changed with it (fork-fleet-walls-wsl2.test.py checks it).
LAUNCHER_SHA256 = 'a2f56e1fe49b485359d0bc1ecb6ee0ff78365afaddb715b3f7e717d6a542f38a'
LAUNCHER_SHA256 = 'aabfec798b7d0fda3986a2686bddc425a06c8b0246a016266d944bac748ed869'
WSL_CONF = '/etc/wsl.conf'
BINFMT = '/proc/sys/fs/binfmt_misc'
MASK = '/etc/binfmt.d/WSLInterop.conf' # same name as WSL's /usr/lib/binfmt.d file: systemd-binfmt reads this one
Expand Down
35 changes: 34 additions & 1 deletion .github/scripts/fork-fleet.test.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
"""Tests of fork-fleet.py, fork-fleet-launch.sh and fork-fleet-hook.sh, and of their wiring in fork-tests.yml,
fork-fleet.yml and fork-release.yml. Run: python3 <this>. Linux only (the gate and launcher tests run real processes)."""
import copy, http.client, importlib.util, io, json, os, pwd, re, shutil, signal, socket, stat, subprocess, sys
import copy, datetime, http.client, importlib.util, io, json, os, pwd, re, shutil, signal, socket, stat, subprocess, sys
import tempfile
import unittest, urllib.error, urllib.parse
from contextlib import redirect_stdout
Expand Down Expand Up @@ -2169,6 +2169,39 @@ def test_fits(self):
self.write('meminfo', 'MemTotal: 1 kB\n', 0o644)
self.assertIn('no MemAvailable', self.launch('fits', '1').stderr)

def test_fits_asks_the_free_script_too(self):
"""FREE_COMMAND (free/v1, fork-kit/common/FREE.md): after the memory rule, its answer through FREE_CHECK."""
with open(os.path.join(HERE, '..', '..', 'fork-kit', 'common', 'free_contract.py')) as f:
reader = self.write('free_contract.py', f.read(), 0o644)
script = self.path('free')
self.conf(FREE_COMMAND=script, FREE_CHECK=reader)
free = 8388608 // 1024 - 2048

def answer(code, **facts):
at = datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
body = json.dumps({'v': 1, 'free': code == 0, 'at': at, **facts})
self.write('free', f"#!/bin/sh\necho '{body}'\n: > {self.rec}/free-ran\nexit {code}\n", 0o755)
answer(0)
self.assertEqual(self.launch('fits', str(free)).returncode, 0)
answer(75, reason='a game is running')
got = self.launch('fits', '1')
self.assertEqual(got.returncode, 75)
self.assertIn('busy: a game is running', got.stderr)
answer(0, memory={'freeMb': 1000})
self.assertNotEqual(self.launch('fits', '1001').returncode, 0)
self.assertEqual(self.launch('fits', '1000').returncode, 0)
answer(3)
self.assertIn('not available: exit 3', self.launch('fits', '1').stderr)
answer(0)
os.remove(self.path('rec/free-ran'))
self.assertEqual(self.launch('fits', str(free + 1)).returncode, 1)
self.assertFalse(os.path.exists(self.path('rec/free-ran')), 'the memory rule says no first')
os.chmod(reader, 0o666)
self.assertIn('writable by its owner only', self.launch('fits', '1').stderr)
os.chmod(reader, 0o644)
self.conf(FREE_COMMAND='free')
self.assertIn('plain absolute paths', self.launch('fits', '1').stderr)

def test_stop(self):
self.assertEqual(self.launch('stop', self.NAME).returncode, 0)
self.assertEqual(self.rec_text('stopped'), f'stop -- fork-fleet-{self.NAME}.service\n')
Expand Down
77 changes: 77 additions & 0 deletions fork-kit/common/FREE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
# free/v1: "is this machine free?"

One small contract for every tool that asks whether a machine may take more work: a fleet desktop's `fits`
(the desk kit), the placement `factsCommand`, and `heavy facts` once it exists. A machine answers with one
**free script**; every consumer reads it the same way, so CI and agents never disagree about one machine.

## The script

- Called with **no arguments**, by its **absolute path**, stdin empty, a plain environment (`PATH`, `LANG`),
from `/`. It answers within **3 seconds** or counts as not available; its whole process group is then ended,
and so is any process that still holds its stdout. It leaves nothing running: a child that leaves the group
(`setsid`) and closes stdout is not seen and not ended.
- It and every folder above it belong to root (or to the user the consumer runs as) and no one else can change
them (a sticky folder such as `/tmp` is fine). Otherwise it is not run. The path is followed one part at a
time: the folders of the path as written are judged too, and every link on the way must belong to root or to
that user (another user's link in `/tmp` is refused, as the kernel's `fs.protected_symlinks` would). The script
is judged and run by the file the path leads to, resolved once. Only the mode bits are read: an ACL that lets
others write is not seen, so do not set one. A script that could not be ended (a group the consumer may not
signal) is waited for at most one more second, then left to finish on its own.
- **The exit code is the verdict:** `0` free, `75` busy. Anything else, a timeout, output that is not one JSON
object (`NaN` and `Infinity` are not JSON), a higher `v`, a `free` that disagrees with the exit code, or facts
more than 60 seconds from now = **not available**. Busy and not available both count as **not free**.
- **stdout is the detail:** one JSON object of at most 4096 bytes. Unknown fields are ignored.

| Field | Required | What |
| ----------- | -------------------- | ------------------------------------------------------------------------ |
| `v` | yes | `1` |
| `free` | yes | `true` or `false`, the same as the exit code |
| `at` | yes | when the facts were taken, ISO 8601 with a zone (`2026-11-15T05:00:00Z`) |
| `reason` | when `free` is false | short words for a person ("memory pressure 62%", "a game is running") |
| `labels` | no | names a placement rule can match (`["ci-runner"]`) |
| `memory` | no | `budgetMb`, `reservedMb`, `freeMb`, `pressureFull10`, `swapUsedPercent` |
| `queue` | no | `running`, `waiting` |
| `leftovers` | no | `[{"comm", "count", "cpuPercent"}]`, busiest first |

```json
{
"v": 1,
"free": false,
"reason": "memory pressure 62%",
"at": "2026-11-15T05:00:00Z",
"labels": ["ci-runner"],
"memory": {
"budgetMb": 4608,
"reservedMb": 3072,
"freeMb": 1536,
"pressureFull10": 62,
"swapUsedPercent": 80
},
"queue": { "running": 2, "waiting": 1 },
"leftovers": [{ "comm": "python3", "count": 73, "cpuPercent": 380 }]
}
```

A consumer that needs a size (the desk `fits MB`) also treats a free answer whose `memory.freeMb` is below that
size as busy. An answer without `memory.freeMb` makes no memory claim; the consumer's own memory rule still holds.

## Pieces

- [free_contract.py](free_contract.py): the one Python reader (`judge` for an answer, `ask` to run a script, and
`python3 -I free_contract.py ask [--need-mb MB] /path` for shell callers: exit 0 free, 75 busy, 1 not available).
- [free-fixtures.json](free-fixtures.json): the cases every reader, in any language, must agree on.
- Examples in [free/](free/):
- [memory](free/memory): busy under memory pressure or with swap nearly full. The desk kit installs it as
`/opt/fork-fleet/free-memory`. Pressure is "all tasks stalled" over 10 seconds, which can stay low while the
machine swaps hard; the swap rule is there for that.
- [busy-while-running](free/busy-while-running): busy while a named program runs (a game, say). Copy it and put
your programs in it. It sees the processes of its own system only: in WSL, Windows programs are not visible.

Several conditions in one script: call the others from it and answer busy when one is busy.

## Using it on a desk pool

Default: no free script; `fits` uses its memory rule alone (MemAvailable minus `RESERVE_MB`). To add one, put
`"freeCommand": "/opt/fork-fleet/free-memory"` (or your own script's path) in the machine file's `desk` section
and install again: the kit writes `FREE_COMMAND` to `launch.conf`, `fits` then also needs a free answer, and
`desk-install check` shows what the script answers now.
4 changes: 4 additions & 0 deletions fork-kit/common/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ install -d -m 755 /etc/fork-kit && install -m 644 /dev/stdin /etc/fork-kit/machi
EOF
```

## free/v1

The "is this machine free?" contract, its one reader and example scripts: [FREE.md](FREE.md).

## Loader

```python
Expand Down
173 changes: 173 additions & 0 deletions fork-kit/common/free-fixtures.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
{
"about": "free/v1 (FREE.md): the cases every parser must agree on. exit null = no answer in time; with padTo, the PAD in stdout is filled with x until stdout is padTo bytes long; now is the reader's clock; needMb is the size the reader asks for.",
"now": "2026-11-15T05:00:30Z",
"cases": [
{
"name": "free",
"exit": 0,
"stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\"}",
"verdict": "free"
},
{
"name": "busy with a reason",
"exit": 75,
"stdout": "{\"v\": 1, \"free\": false, \"reason\": \"memory pressure 62%\", \"at\": \"2026-11-15T05:00:00Z\"}",
"verdict": "busy"
},
{
"name": "the full example",
"exit": 75,
"stdout": "{\"v\": 1, \"free\": false, \"reason\": \"memory pressure 62%\", \"at\": \"2026-11-15T05:00:00Z\", \"labels\": [\"ci-runner\"], \"memory\": {\"budgetMb\": 4608, \"reservedMb\": 3072, \"freeMb\": 1536, \"pressureFull10\": 62, \"swapUsedPercent\": 80}, \"queue\": {\"running\": 2, \"waiting\": 1}, \"leftovers\": [{\"comm\": \"python3\", \"count\": 73, \"cpuPercent\": 380}]}",
"verdict": "busy"
},
{
"name": "unknown fields are ignored",
"exit": 0,
"stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\", \"later\": {\"x\": 1}}",
"verdict": "free"
},
{
"name": "an offset instead of Z",
"exit": 0,
"stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T06:00:00.250+01:00\"}",
"verdict": "free"
},
{
"name": "enough memory for the size asked",
"exit": 0,
"needMb": 2048,
"stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\", \"memory\": {\"freeMb\": 2048}}",
"verdict": "free"
},
{
"name": "too little memory for the size asked",
"exit": 0,
"needMb": 2049,
"stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\", \"memory\": {\"freeMb\": 2048}}",
"verdict": "busy"
},
{
"name": "no memory figure: a size asked is not judged",
"exit": 0,
"needMb": 99999,
"stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\"}",
"verdict": "free"
},
{
"name": "NaN is not JSON",
"exit": 0,
"needMb": 100,
"stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\", \"memory\": {\"freeMb\": NaN}}",
"verdict": "unavailable"
},
{
"name": "Infinity is not JSON",
"exit": 0,
"needMb": 100,
"stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\", \"memory\": {\"freeMb\": -Infinity}}",
"verdict": "unavailable"
},
{ "name": "no answer in time", "exit": null, "stdout": "", "verdict": "unavailable" },
{
"name": "another exit code",
"exit": 1,
"stdout": "{\"v\": 1, \"free\": false, \"reason\": \"x\", \"at\": \"2026-11-15T05:00:00Z\"}",
"verdict": "unavailable"
},
{
"name": "exit 0 but free is false",
"exit": 0,
"stdout": "{\"v\": 1, \"free\": false, \"reason\": \"x\", \"at\": \"2026-11-15T05:00:00Z\"}",
"verdict": "unavailable"
},
{
"name": "exit 75 but free is true",
"exit": 75,
"stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\"}",
"verdict": "unavailable"
},
{
"name": "free is a string",
"exit": 0,
"stdout": "{\"v\": 1, \"free\": \"true\", \"at\": \"2026-11-15T05:00:00Z\"}",
"verdict": "unavailable"
},
{
"name": "busy without a reason",
"exit": 75,
"stdout": "{\"v\": 1, \"free\": false, \"at\": \"2026-11-15T05:00:00Z\"}",
"verdict": "unavailable"
},
{
"name": "busy with a blank reason",
"exit": 75,
"stdout": "{\"v\": 1, \"free\": false, \"reason\": \" \", \"at\": \"2026-11-15T05:00:00Z\"}",
"verdict": "unavailable"
},
{
"name": "a newer version",
"exit": 0,
"stdout": "{\"v\": 2, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\"}",
"verdict": "unavailable"
},
{
"name": "no version",
"exit": 0,
"stdout": "{\"free\": true, \"at\": \"2026-11-15T05:00:00Z\"}",
"verdict": "unavailable"
},
{
"name": "version true",
"exit": 0,
"stdout": "{\"v\": true, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\"}",
"verdict": "unavailable"
},
{
"name": "no time",
"exit": 0,
"stdout": "{\"v\": 1, \"free\": true}",
"verdict": "unavailable"
},
{
"name": "a time without a zone",
"exit": 0,
"stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00\"}",
"verdict": "unavailable"
},
{
"name": "facts older than 60 seconds",
"exit": 0,
"stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T04:59:29Z\"}",
"verdict": "unavailable"
},
{
"name": "facts from 60 seconds ago",
"exit": 0,
"stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T04:59:30Z\"}",
"verdict": "free"
},
{
"name": "facts from the future",
"exit": 0,
"stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:01:31Z\"}",
"verdict": "unavailable"
},
{ "name": "not JSON", "exit": 0, "stdout": "free", "verdict": "unavailable" },
{ "name": "a JSON list", "exit": 0, "stdout": "[1]", "verdict": "unavailable" },
{ "name": "nothing on stdout", "exit": 0, "stdout": "", "verdict": "unavailable" },
{
"name": "4096 bytes is the most",
"exit": 0,
"stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\", \"pad\": \"PAD\"}",
"padTo": 4096,
"verdict": "free"
},
{
"name": "4097 bytes is too many",
"exit": 0,
"stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\", \"pad\": \"PAD\"}",
"padTo": 4097,
"verdict": "unavailable"
}
]
}
Loading
Loading