diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index 55c2903cd3e5e..ccedb1f29c7de 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -41,6 +41,7 @@ fork-ci/actions/pinned/pinned.test.py fork-ci/actions/pr-queue/lens.test.py fork-ci/actions/pr-queue/pr_queue.test.py fork-ci/actions/pr-queue/tested-base/tested_base.test.py +fork-kit/common/tests/free_contract.test.py fork-kit/common/tests/machine_file.test.py fork-kit/desk/tests/install.test.py fork-kit/desk/tests/windows.test.py diff --git a/.github/scripts/fork-fleet-launch.sh b/.github/scripts/fork-fleet-launch.sh index e3250885ba769..7ac6b6e6dd608 100755 --- a/.github/scripts/fork-fleet-launch.sh +++ b/.github/scripts/fork-fleet-launch.sh @@ -20,7 +20,8 @@ # TEMPLATE and swaps the two by one rename under the launch lock. Running jobs keep their copies; any # failure leaves the old template. Below. # active the NAME of every fork-fleet-* unit that is loaded (starting, running or stopping). -# fits MB exit 0 when MemAvailable minus RESERVE_MB leaves MB megabytes. +# fits MB exit 0 when MemAvailable minus RESERVE_MB leaves MB megabytes and, with FREE_COMMAND set, that free +# script answers free (free/v1, fork-kit/common/FREE.md: run by FREE_CHECK, its reader). # stop NAME stops that unit (a job on it fails: its runner is gone). # prune removes the copies and environment files whose unit is gone. # refresh-net every dispatcher pass: looks this host's own public address up again and, when what is blocked would @@ -61,7 +62,8 @@ # fork-fleet-job-paths.txt beside this launcher), ALLOW_PATHS (more paths jobs see read-only, comma-separated, say a # tool cache: empty by default), IP_FORWARD, SYSTEMD_RUN, SYSTEMCTL, RUNNER_RELEASE (refresh's release record, an https # URL), RUNNER_DOWNLOAD (the https base of its tarballs) and RUNNER_MIN (the oldest runner version refresh may install, -# empty for none) have defaults below. +# empty for none), FREE_COMMAND (a free/v1 script's absolute path, empty for none) and FREE_CHECK (its reader, +# free_contract.py beside this launcher) have defaults below. # HOOK (and the two files beside it), WORK_ROOT and RESOLV_CONF are bound into every job too: never under a folder that # holds data (/home, /root, /mnt, /media, /srv), also through a symlink (no_data). A bind mount of a data folder at # one of them is not seen (it has no link to resolve): that is the operator's to avoid (.github/fork-fleet.md, step 4). @@ -116,8 +118,8 @@ LINGER_DIR=/var/lib/systemd/linger RESERVE_MB=2048 MAX_SECONDS=3600 MEMINFO=/pro RESOLV_CONF=/etc/fork-fleet/resolv.conf IP_FORWARD=/proc/sys/net/ipv4/ip_forward SYSTEMD_RUN=systemd-run SYSTEMCTL=systemctl OWN_ADDRESS_URLS=https://api.ipify.org,https://checkip.amazonaws.com ALLOW_PATHS='' JOB_PATHS='' RUNNER_RELEASE=https://api.github.com/repos/actions/runner/releases/latest -RUNNER_DOWNLOAD=https://github.com/actions/runner/releases/download RUNNER_MIN='' -setting='^(RUNNER_USERS|TEMPLATE|WORK_ROOT|RUN_DIR|HOOK|LINGER_DIR|RESERVE_MB|MAX_SECONDS|MEMINFO|NET_BASE|RESOLV_CONF|IP_FORWARD|SYSTEMD_RUN|SYSTEMCTL|OWN_ADDRESS_URLS|JOB_PATHS|ALLOW_PATHS|RUNNER_RELEASE|RUNNER_DOWNLOAD|RUNNER_MIN)=([^[:space:]]*)$' +RUNNER_DOWNLOAD=https://github.com/actions/runner/releases/download RUNNER_MIN='' FREE_COMMAND='' FREE_CHECK='' +setting='^(RUNNER_USERS|TEMPLATE|WORK_ROOT|RUN_DIR|HOOK|LINGER_DIR|RESERVE_MB|MAX_SECONDS|MEMINFO|NET_BASE|RESOLV_CONF|IP_FORWARD|SYSTEMD_RUN|SYSTEMCTL|OWN_ADDRESS_URLS|JOB_PATHS|ALLOW_PATHS|RUNNER_RELEASE|RUNNER_DOWNLOAD|RUNNER_MIN|FREE_COMMAND|FREE_CHECK)=([^[:space:]]*)$' while IFS= read -r line || [[ -n $line ]]; do [[ $line =~ ^[[:space:]]*(#|$) ]] && continue [[ $line =~ $setting ]] || die "$conf: not a setting: ${line%%=*}" @@ -135,6 +137,9 @@ PLAIN_PATH='(/[A-Za-z0-9._@+-]+)+' if [[ -n $JOB_PATHS ]]; then JOB_PATHS=$ROOT$JOB_PATHS; else JOB_PATHS=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)/fork-fleet-job-paths.txt fi +[[ $FREE_COMMAND =~ ^($PLAIN_PATH)?$ && $FREE_CHECK =~ ^($PLAIN_PATH)?$ ]] || + die "$conf: FREE_COMMAND and FREE_CHECK are plain absolute paths" +[[ -n $FREE_CHECK ]] || FREE_CHECK=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)/free_contract.py NAME='^[a-z0-9][a-z0-9-]{0,62}-[0-9]{1,20}$' name_ok() { [[ $1 =~ $NAME ]] || die "not a runner name: $1"; } @@ -682,7 +687,11 @@ fits() { [[ $1 =~ ^[0-9]{1,7}$ ]] || die "fits: not a number of megabytes: $1" kb=$(awk '$1 == "MemAvailable:" { print $2 }' "$MEMINFO") [[ $kb =~ ^[0-9]+$ ]] || die "no MemAvailable in $MEMINFO" - ((kb / 1024 - RESERVE_MB >= $1)) + ((kb / 1024 - RESERVE_MB >= $1)) || return 1 + [[ -n $FREE_COMMAND ]] || return 0 + owned_safe "$FREE_CHECK" + # Its one line (free, busy: why, not available: why) for the dispatcher's log; any exit but 0 is not free. + /usr/bin/python3 -I "$FREE_CHECK" ask --need-mb "$1" "$FREE_COMMAND" >&2 } # The ranges no job reaches: the host itself is refused on its own (the input chain), these on the way out: RFC 1918, diff --git a/.github/scripts/fork-fleet-walls/wsl2 b/.github/scripts/fork-fleet-walls/wsl2 index 055c103ae4f15..e6bffaed078cb 100755 --- a/.github/scripts/fork-fleet-walls/wsl2 +++ b/.github/scripts/fork-fleet-walls/wsl2 @@ -36,7 +36,7 @@ ROOT = os.environ.get('FORK_FLEET_WALLS_ROOT', '') ME = os.path.realpath(__file__) # The sha256 of the launcher (fork-fleet-launch.sh) whose job units unit_props reads as safe: the one this file ships # with. A changed launcher must be read again and this pin changed with it (fork-fleet-walls-wsl2.test.py checks it). -LAUNCHER_SHA256 = 'a2f56e1fe49b485359d0bc1ecb6ee0ff78365afaddb715b3f7e717d6a542f38a' +LAUNCHER_SHA256 = 'aabfec798b7d0fda3986a2686bddc425a06c8b0246a016266d944bac748ed869' WSL_CONF = '/etc/wsl.conf' BINFMT = '/proc/sys/fs/binfmt_misc' MASK = '/etc/binfmt.d/WSLInterop.conf' # same name as WSL's /usr/lib/binfmt.d file: systemd-binfmt reads this one diff --git a/.github/scripts/fork-fleet.test.py b/.github/scripts/fork-fleet.test.py index 0494947aee1b5..6f0434f637854 100644 --- a/.github/scripts/fork-fleet.test.py +++ b/.github/scripts/fork-fleet.test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Tests of fork-fleet.py, fork-fleet-launch.sh and fork-fleet-hook.sh, and of their wiring in fork-tests.yml, fork-fleet.yml and fork-release.yml. Run: python3 . Linux only (the gate and launcher tests run real processes).""" -import copy, http.client, importlib.util, io, json, os, pwd, re, shutil, signal, socket, stat, subprocess, sys +import copy, datetime, http.client, importlib.util, io, json, os, pwd, re, shutil, signal, socket, stat, subprocess, sys import tempfile import unittest, urllib.error, urllib.parse from contextlib import redirect_stdout @@ -2169,6 +2169,39 @@ def test_fits(self): self.write('meminfo', 'MemTotal: 1 kB\n', 0o644) self.assertIn('no MemAvailable', self.launch('fits', '1').stderr) + def test_fits_asks_the_free_script_too(self): + """FREE_COMMAND (free/v1, fork-kit/common/FREE.md): after the memory rule, its answer through FREE_CHECK.""" + with open(os.path.join(HERE, '..', '..', 'fork-kit', 'common', 'free_contract.py')) as f: + reader = self.write('free_contract.py', f.read(), 0o644) + script = self.path('free') + self.conf(FREE_COMMAND=script, FREE_CHECK=reader) + free = 8388608 // 1024 - 2048 + + def answer(code, **facts): + at = datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + body = json.dumps({'v': 1, 'free': code == 0, 'at': at, **facts}) + self.write('free', f"#!/bin/sh\necho '{body}'\n: > {self.rec}/free-ran\nexit {code}\n", 0o755) + answer(0) + self.assertEqual(self.launch('fits', str(free)).returncode, 0) + answer(75, reason='a game is running') + got = self.launch('fits', '1') + self.assertEqual(got.returncode, 75) + self.assertIn('busy: a game is running', got.stderr) + answer(0, memory={'freeMb': 1000}) + self.assertNotEqual(self.launch('fits', '1001').returncode, 0) + self.assertEqual(self.launch('fits', '1000').returncode, 0) + answer(3) + self.assertIn('not available: exit 3', self.launch('fits', '1').stderr) + answer(0) + os.remove(self.path('rec/free-ran')) + self.assertEqual(self.launch('fits', str(free + 1)).returncode, 1) + self.assertFalse(os.path.exists(self.path('rec/free-ran')), 'the memory rule says no first') + os.chmod(reader, 0o666) + self.assertIn('writable by its owner only', self.launch('fits', '1').stderr) + os.chmod(reader, 0o644) + self.conf(FREE_COMMAND='free') + self.assertIn('plain absolute paths', self.launch('fits', '1').stderr) + def test_stop(self): self.assertEqual(self.launch('stop', self.NAME).returncode, 0) self.assertEqual(self.rec_text('stopped'), f'stop -- fork-fleet-{self.NAME}.service\n') diff --git a/fork-kit/common/FREE.md b/fork-kit/common/FREE.md new file mode 100644 index 0000000000000..95eebbe60cedc --- /dev/null +++ b/fork-kit/common/FREE.md @@ -0,0 +1,77 @@ +# free/v1: "is this machine free?" + +One small contract for every tool that asks whether a machine may take more work: a fleet desktop's `fits` +(the desk kit), the placement `factsCommand`, and `heavy facts` once it exists. A machine answers with one +**free script**; every consumer reads it the same way, so CI and agents never disagree about one machine. + +## The script + +- Called with **no arguments**, by its **absolute path**, stdin empty, a plain environment (`PATH`, `LANG`), + from `/`. It answers within **3 seconds** or counts as not available; its whole process group is then ended, + and so is any process that still holds its stdout. It leaves nothing running: a child that leaves the group + (`setsid`) and closes stdout is not seen and not ended. +- It and every folder above it belong to root (or to the user the consumer runs as) and no one else can change + them (a sticky folder such as `/tmp` is fine). Otherwise it is not run. The path is followed one part at a + time: the folders of the path as written are judged too, and every link on the way must belong to root or to + that user (another user's link in `/tmp` is refused, as the kernel's `fs.protected_symlinks` would). The script + is judged and run by the file the path leads to, resolved once. Only the mode bits are read: an ACL that lets + others write is not seen, so do not set one. A script that could not be ended (a group the consumer may not + signal) is waited for at most one more second, then left to finish on its own. +- **The exit code is the verdict:** `0` free, `75` busy. Anything else, a timeout, output that is not one JSON + object (`NaN` and `Infinity` are not JSON), a higher `v`, a `free` that disagrees with the exit code, or facts + more than 60 seconds from now = **not available**. Busy and not available both count as **not free**. +- **stdout is the detail:** one JSON object of at most 4096 bytes. Unknown fields are ignored. + +| Field | Required | What | +| ----------- | -------------------- | ------------------------------------------------------------------------ | +| `v` | yes | `1` | +| `free` | yes | `true` or `false`, the same as the exit code | +| `at` | yes | when the facts were taken, ISO 8601 with a zone (`2026-11-15T05:00:00Z`) | +| `reason` | when `free` is false | short words for a person ("memory pressure 62%", "a game is running") | +| `labels` | no | names a placement rule can match (`["ci-runner"]`) | +| `memory` | no | `budgetMb`, `reservedMb`, `freeMb`, `pressureFull10`, `swapUsedPercent` | +| `queue` | no | `running`, `waiting` | +| `leftovers` | no | `[{"comm", "count", "cpuPercent"}]`, busiest first | + +```json +{ + "v": 1, + "free": false, + "reason": "memory pressure 62%", + "at": "2026-11-15T05:00:00Z", + "labels": ["ci-runner"], + "memory": { + "budgetMb": 4608, + "reservedMb": 3072, + "freeMb": 1536, + "pressureFull10": 62, + "swapUsedPercent": 80 + }, + "queue": { "running": 2, "waiting": 1 }, + "leftovers": [{ "comm": "python3", "count": 73, "cpuPercent": 380 }] +} +``` + +A consumer that needs a size (the desk `fits MB`) also treats a free answer whose `memory.freeMb` is below that +size as busy. An answer without `memory.freeMb` makes no memory claim; the consumer's own memory rule still holds. + +## Pieces + +- [free_contract.py](free_contract.py): the one Python reader (`judge` for an answer, `ask` to run a script, and + `python3 -I free_contract.py ask [--need-mb MB] /path` for shell callers: exit 0 free, 75 busy, 1 not available). +- [free-fixtures.json](free-fixtures.json): the cases every reader, in any language, must agree on. +- Examples in [free/](free/): + - [memory](free/memory): busy under memory pressure or with swap nearly full. The desk kit installs it as + `/opt/fork-fleet/free-memory`. Pressure is "all tasks stalled" over 10 seconds, which can stay low while the + machine swaps hard; the swap rule is there for that. + - [busy-while-running](free/busy-while-running): busy while a named program runs (a game, say). Copy it and put + your programs in it. It sees the processes of its own system only: in WSL, Windows programs are not visible. + +Several conditions in one script: call the others from it and answer busy when one is busy. + +## Using it on a desk pool + +Default: no free script; `fits` uses its memory rule alone (MemAvailable minus `RESERVE_MB`). To add one, put +`"freeCommand": "/opt/fork-fleet/free-memory"` (or your own script's path) in the machine file's `desk` section +and install again: the kit writes `FREE_COMMAND` to `launch.conf`, `fits` then also needs a free answer, and +`desk-install check` shows what the script answers now. diff --git a/fork-kit/common/README.md b/fork-kit/common/README.md index 364979c395b6b..58217a014049c 100644 --- a/fork-kit/common/README.md +++ b/fork-kit/common/README.md @@ -48,6 +48,10 @@ install -d -m 755 /etc/fork-kit && install -m 644 /dev/stdin /etc/fork-kit/machi EOF ``` +## free/v1 + +The "is this machine free?" contract, its one reader and example scripts: [FREE.md](FREE.md). + ## Loader ```python diff --git a/fork-kit/common/free-fixtures.json b/fork-kit/common/free-fixtures.json new file mode 100644 index 0000000000000..062358fc48486 --- /dev/null +++ b/fork-kit/common/free-fixtures.json @@ -0,0 +1,173 @@ +{ + "about": "free/v1 (FREE.md): the cases every parser must agree on. exit null = no answer in time; with padTo, the PAD in stdout is filled with x until stdout is padTo bytes long; now is the reader's clock; needMb is the size the reader asks for.", + "now": "2026-11-15T05:00:30Z", + "cases": [ + { + "name": "free", + "exit": 0, + "stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\"}", + "verdict": "free" + }, + { + "name": "busy with a reason", + "exit": 75, + "stdout": "{\"v\": 1, \"free\": false, \"reason\": \"memory pressure 62%\", \"at\": \"2026-11-15T05:00:00Z\"}", + "verdict": "busy" + }, + { + "name": "the full example", + "exit": 75, + "stdout": "{\"v\": 1, \"free\": false, \"reason\": \"memory pressure 62%\", \"at\": \"2026-11-15T05:00:00Z\", \"labels\": [\"ci-runner\"], \"memory\": {\"budgetMb\": 4608, \"reservedMb\": 3072, \"freeMb\": 1536, \"pressureFull10\": 62, \"swapUsedPercent\": 80}, \"queue\": {\"running\": 2, \"waiting\": 1}, \"leftovers\": [{\"comm\": \"python3\", \"count\": 73, \"cpuPercent\": 380}]}", + "verdict": "busy" + }, + { + "name": "unknown fields are ignored", + "exit": 0, + "stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\", \"later\": {\"x\": 1}}", + "verdict": "free" + }, + { + "name": "an offset instead of Z", + "exit": 0, + "stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T06:00:00.250+01:00\"}", + "verdict": "free" + }, + { + "name": "enough memory for the size asked", + "exit": 0, + "needMb": 2048, + "stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\", \"memory\": {\"freeMb\": 2048}}", + "verdict": "free" + }, + { + "name": "too little memory for the size asked", + "exit": 0, + "needMb": 2049, + "stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\", \"memory\": {\"freeMb\": 2048}}", + "verdict": "busy" + }, + { + "name": "no memory figure: a size asked is not judged", + "exit": 0, + "needMb": 99999, + "stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\"}", + "verdict": "free" + }, + { + "name": "NaN is not JSON", + "exit": 0, + "needMb": 100, + "stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\", \"memory\": {\"freeMb\": NaN}}", + "verdict": "unavailable" + }, + { + "name": "Infinity is not JSON", + "exit": 0, + "needMb": 100, + "stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\", \"memory\": {\"freeMb\": -Infinity}}", + "verdict": "unavailable" + }, + { "name": "no answer in time", "exit": null, "stdout": "", "verdict": "unavailable" }, + { + "name": "another exit code", + "exit": 1, + "stdout": "{\"v\": 1, \"free\": false, \"reason\": \"x\", \"at\": \"2026-11-15T05:00:00Z\"}", + "verdict": "unavailable" + }, + { + "name": "exit 0 but free is false", + "exit": 0, + "stdout": "{\"v\": 1, \"free\": false, \"reason\": \"x\", \"at\": \"2026-11-15T05:00:00Z\"}", + "verdict": "unavailable" + }, + { + "name": "exit 75 but free is true", + "exit": 75, + "stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\"}", + "verdict": "unavailable" + }, + { + "name": "free is a string", + "exit": 0, + "stdout": "{\"v\": 1, \"free\": \"true\", \"at\": \"2026-11-15T05:00:00Z\"}", + "verdict": "unavailable" + }, + { + "name": "busy without a reason", + "exit": 75, + "stdout": "{\"v\": 1, \"free\": false, \"at\": \"2026-11-15T05:00:00Z\"}", + "verdict": "unavailable" + }, + { + "name": "busy with a blank reason", + "exit": 75, + "stdout": "{\"v\": 1, \"free\": false, \"reason\": \" \", \"at\": \"2026-11-15T05:00:00Z\"}", + "verdict": "unavailable" + }, + { + "name": "a newer version", + "exit": 0, + "stdout": "{\"v\": 2, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\"}", + "verdict": "unavailable" + }, + { + "name": "no version", + "exit": 0, + "stdout": "{\"free\": true, \"at\": \"2026-11-15T05:00:00Z\"}", + "verdict": "unavailable" + }, + { + "name": "version true", + "exit": 0, + "stdout": "{\"v\": true, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\"}", + "verdict": "unavailable" + }, + { + "name": "no time", + "exit": 0, + "stdout": "{\"v\": 1, \"free\": true}", + "verdict": "unavailable" + }, + { + "name": "a time without a zone", + "exit": 0, + "stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00\"}", + "verdict": "unavailable" + }, + { + "name": "facts older than 60 seconds", + "exit": 0, + "stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T04:59:29Z\"}", + "verdict": "unavailable" + }, + { + "name": "facts from 60 seconds ago", + "exit": 0, + "stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T04:59:30Z\"}", + "verdict": "free" + }, + { + "name": "facts from the future", + "exit": 0, + "stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:01:31Z\"}", + "verdict": "unavailable" + }, + { "name": "not JSON", "exit": 0, "stdout": "free", "verdict": "unavailable" }, + { "name": "a JSON list", "exit": 0, "stdout": "[1]", "verdict": "unavailable" }, + { "name": "nothing on stdout", "exit": 0, "stdout": "", "verdict": "unavailable" }, + { + "name": "4096 bytes is the most", + "exit": 0, + "stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\", \"pad\": \"PAD\"}", + "padTo": 4096, + "verdict": "free" + }, + { + "name": "4097 bytes is too many", + "exit": 0, + "stdout": "{\"v\": 1, \"free\": true, \"at\": \"2026-11-15T05:00:00Z\", \"pad\": \"PAD\"}", + "padTo": 4097, + "verdict": "unavailable" + } + ] +} diff --git a/fork-kit/common/free/busy-while-running b/fork-kit/common/free/busy-while-running new file mode 100755 index 0000000000000..f5b5cd40723bd --- /dev/null +++ b/fork-kit/common/free/busy-while-running @@ -0,0 +1,17 @@ +#!/bin/sh +# A free/v1 script (FREE.md): busy while one of the programs in NAMES runs, say a game or a video editor, so the +# machine is left to its person. Copy it, put your programs' process names (as `ps -e -o comm=` shows them) in +# NAMES, make the copy root's and not writable by others, and name the copy as the free command. +# It sees the processes of this system only: in WSL, Windows programs are not visible (interop is off, on purpose). +NAMES='example-game example-editor' + +at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +running=$(ps -e -o comm=) || { echo '{"v": 1, "free": false, "reason": "ps failed", "at": "'"$at"'"}'; exit 1; } +for name in $NAMES; do + short=$(printf '%.15s' "$name") # the kernel keeps only the first 15 characters of a process name + if printf '%s\n' "$running" | grep -qxF -- "$short"; then + echo '{"v": 1, "free": false, "reason": "'"$name"' is running", "at": "'"$at"'"}' + exit 75 + fi +done +echo '{"v": 1, "free": true, "at": "'"$at"'"}' diff --git a/fork-kit/common/free/memory b/fork-kit/common/free/memory new file mode 100755 index 0000000000000..b94727ba8219c --- /dev/null +++ b/fork-kit/common/free/memory @@ -0,0 +1,40 @@ +#!/usr/bin/python3 -I +"""A free/v1 script (FREE.md): free while memory is not under pressure and swap is not nearly full. + +Busy when, over the last 10 seconds, all tasks waited on memory more than PRESSURE_FULL10 percent of the time +(/proc/pressure/memory, "full avg10"), or when more than SWAP_USED_PERCENT of swap is in use. Change the two +numbers in a copy of this file; the desk kit installs this one as /opt/fork-fleet/free-memory. +""" +import datetime, json, sys + +PRESSURE_FULL10, SWAP_USED_PERCENT = 10, 50 + + +def meminfo(): + with open('/proc/meminfo', encoding='ascii') as f: + return {k: int(v.split()[0]) for k, v in (line.split(':', 1) for line in f)} + + +def pressure_full10(): + try: + with open('/proc/pressure/memory', encoding='ascii') as f: + for line in f: + if line.startswith('full '): + return float(dict(x.split('=') for x in line.split()[1:])['avg10']) + except OSError: # a kernel without PSI: no pressure figure, the swap rule alone + return None + return None + + +m, full10 = meminfo(), pressure_full10() +swap = round(100 * (m['SwapTotal'] - m['SwapFree']) / m['SwapTotal']) if m.get('SwapTotal') else 0 +reason = (f'memory pressure {full10:g}%' if full10 is not None and full10 > PRESSURE_FULL10 else + f'swap {swap}% used' if swap > SWAP_USED_PERCENT else '') +memory = {'freeMb': m['MemAvailable'] // 1024, 'swapUsedPercent': swap} +if full10 is not None: + memory['pressureFull10'] = full10 +answer = {'v': 1, 'free': not reason, 'at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'memory': memory} +if reason: + answer['reason'] = reason +print(json.dumps(answer)) +sys.exit(75 if reason else 0) diff --git a/fork-kit/common/free_contract.py b/fork-kit/common/free_contract.py new file mode 100644 index 0000000000000..f0216467c9073 --- /dev/null +++ b/fork-kit/common/free_contract.py @@ -0,0 +1,230 @@ +"""free/v1, "is this machine free?" (FREE.md beside this file): runs a free script and judges its answer. + +One parser for every Python consumer (the desk launcher's `fits`, the desk kit's check); free-fixtures.json beside +it holds the cases every parser, in any language, must agree on. + + python3 -I free_contract.py ask [--need-mb MB] /absolute/script + one line on stdout (free / busy: REASON / not available: REASON); + exit 0 free, 75 busy, 1 not available, 2 a wrong call +""" +import argparse, datetime, json, os, select, signal, stat, subprocess, sys, time + +VERSION, LIMIT_SECONDS, MAX_BYTES, MAX_AGE_SECONDS = 1, 3, 4096, 60 +FREE, BUSY = 0, 75 +ENV = {'PATH': '/usr/sbin:/usr/bin:/sbin:/bin', 'LANG': 'C.UTF-8'} + + +def number(x): + return isinstance(x, (int, float)) and not isinstance(x, bool) + + +def parse_at(at): + if not isinstance(at, str): + return None + try: + when = datetime.datetime.fromisoformat(at[:-1] + '+00:00' if at.endswith('Z') else at) + except ValueError: + return None + return when if when.tzinfo is not None else None + + +def no_constant(name): + raise ValueError(f'{name} is not JSON') + + +def judge(code, out, now, need_mb=None): + """(verdict, reason, facts) for one answer: verdict 'free', 'busy' or 'unavailable' (only 'free' is free). + + code: the exit code, None when the script gave no answer in time. out: its stdout (bytes). now: an aware + datetime. need_mb: when given, a `memory.freeMb` below it makes a free answer busy.""" + def no(why): + return 'unavailable', why, None + if len(out) > MAX_BYTES: + return no(f'more than {MAX_BYTES} bytes on stdout') + if code is None: + return no(f'no answer within {LIMIT_SECONDS} seconds') + if code not in (FREE, BUSY): + return no(f'exit {code}') + try: + facts = json.loads(out.decode('utf-8'), parse_constant=no_constant) + except ValueError: # UnicodeDecodeError is one, and so is NaN or Infinity (not JSON, and NaN < size is False) + return no('stdout is not JSON') + if not isinstance(facts, dict): + return no('stdout is not a JSON object') + v, free, reason = facts.get('v'), facts.get('free'), facts.get('reason') + if type(v) is not int or v < 1: + return no('no version "v"') + if v > VERSION: + return no(f'version {v}, this reader knows {VERSION}') + if type(free) is not bool or free != (code == FREE): + return no(f'"free" is {json.dumps(free)} but the exit code is {code}') + if not free and not (isinstance(reason, str) and reason.strip()): + return no('busy without a "reason"') + when = parse_at(facts.get('at')) + if when is None: + return no('no time "at" (ISO 8601 with a zone)') + if abs((now - when).total_seconds()) > MAX_AGE_SECONDS: + return no(f'"at" is {facts["at"]}, more than {MAX_AGE_SECONDS} seconds from now') + if not free: + return 'busy', ' '.join(reason.split())[:200], facts + memory = facts.get('memory') + free_mb = memory.get('freeMb') if isinstance(memory, dict) else None + if need_mb is not None and number(free_mb) and free_mb < need_mb: + return 'busy', f'{free_mb:g} MB free, {need_mb} MB needed', facts + return 'free', '', facts + + +def kept(path, st, uid): + """Why the folder PATH (its stat ST) may not be trusted to hold what we run, or None.""" + if st.st_uid not in (0, uid): + return f'{path} belongs to uid {st.st_uid}' + if st.st_mode & 0o022 and not (stat.S_ISDIR(st.st_mode) and st.st_mode & stat.S_ISVTX): + return f'{path} may be changed by others' + return None + + +def resolve(command, uid=None): + """(the file COMMAND leads to, None), or (None, why not). Follows the path one part at a time, as the kernel + does, so the folders of the path as written are judged, not only those of the file it leads to: every folder + on the way belongs to root or to this user and no one else may change it (a sticky folder like /tmp is fine), + and every link on the way belongs to root or to this user. The kernel's fs.protected_symlinks would refuse + another user's link in a sticky folder; resolving here must not quietly follow one.""" + uid = os.geteuid() if uid is None else uid + if not os.path.isabs(command) or os.path.normpath(command) != command: + return None, f'{command!r} is not an absolute plain path' + parts, path, links = command.split('/')[1:], '/', 0 + try: + while parts: + name = parts.pop(0) + if name in ('', '.'): + continue + if name == '..': + path = os.path.dirname(path) + continue + why = kept(path, os.stat(path), uid) + if why: + return None, why + here = os.path.join(path, name) + st = os.lstat(here) + if not stat.S_ISLNK(st.st_mode): + path = here + continue + if st.st_uid not in (0, uid): + return None, f'{here} is a link of uid {st.st_uid}' + links += 1 + if links > 40: + return None, f'{command}: too many levels of links' + target = os.readlink(here) + parts = target.split('/') + parts + if target.startswith('/'): + path = '/' + except OSError as e: + return None, f'{os.path.join(path, name)}: {e.strerror}' + return path, None + + +def unsafe(command, uid=None): + """Why COMMAND must not run, or None: an absolute plain path that resolve() accepts, to an executable file + that it and every folder above it belong to root or to this user and that no one else may change (a sticky + folder like /tmp is fine). ask() resolves a link once and both judges and runs the file it leads to. + Plain mode bits only: an ACL that lets others write is not seen.""" + uid = os.geteuid() if uid is None else uid + path, why = resolve(command, uid) + if why: + return why + try: + if not stat.S_ISREG(os.stat(path).st_mode) or not os.access(path, os.X_OK): + return f'{path} is not an executable file' + while True: + why = kept(path, os.stat(path), uid) + if why or path == '/': + return why + path = os.path.dirname(path) + except OSError as e: + return f'{path}: {e.strerror}' + + +def holders(pipe): + """The other processes that hold the pipe PIPE (a file object) open: a script's child that left its process + group (setsid) and kept stdout. Only processes this user may look at are seen (all of them for root).""" + name, found = f'pipe:[{os.fstat(pipe.fileno()).st_ino}]', [] + for pid in filter(str.isdigit, os.listdir('/proc')): + if int(pid) == os.getpid(): + continue + try: + if any(os.readlink(f'/proc/{pid}/fd/{fd}') == name for fd in os.listdir(f'/proc/{pid}/fd')): + found.append(int(pid)) + except OSError: # gone, or not ours to look at + pass + return found + + +def ask(command, need_mb=None, timeout=LIMIT_SECONDS): + """Runs the free script (no arguments, stdin empty, a plain environment) and judges its answer: (verdict, + reason, facts). After it, its whole process group is ended, and so is any other process that still holds its + stdout (a child that left the group with setsid). A child that leaves the group and closes stdout is not seen: + FREE.md asks scripts to leave nothing running.""" + path, why = resolve(command) # resolved once: a link repointed later changes nothing + why = why or unsafe(path) + if why: + return 'unavailable', why, None + try: + p = subprocess.Popen([path], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + env=ENV, cwd='/', start_new_session=True) + except OSError as e: + return 'unavailable', f'cannot run {path}: {e.strerror}', None + out, code, closed, deadline = b'', None, False, time.monotonic() + timeout + poll = select.poll() # not select(): a long-lived consumer may hold descriptors above 1024 + poll.register(p.stdout, select.POLLIN) + try: + while len(out) <= MAX_BYTES: + left = deadline - time.monotonic() + if left <= 0 or not poll.poll(left * 1000): + break + chunk = os.read(p.stdout.fileno(), 65536) + if not chunk: + closed = True + try: + code = p.wait(max(0.0, deadline - time.monotonic())) + except subprocess.TimeoutExpired: + pass + break + out += chunk + finally: + try: + os.killpg(p.pid, signal.SIGKILL) + except ProcessLookupError: + pass + except PermissionError: # a group we may not signal (a set-uid script): end at least the script + try: + p.kill() + except OSError: + pass + if not closed: # someone still holds stdout open + for pid in holders(p.stdout): + try: + os.kill(pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + pass + p.stdout.close() + try: # never a wait without a limit: a script we could not end is left to finish on its own + p.wait(1) + except subprocess.TimeoutExpired: + pass + return judge(code, out, datetime.datetime.now(datetime.timezone.utc), need_mb) + + +def main(argv): + parser = argparse.ArgumentParser(prog='free_contract.py') + sub = parser.add_subparsers(dest='verb', required=True) + a = sub.add_parser('ask', help='run a free script and say what it answered') + a.add_argument('--need-mb', type=int) + a.add_argument('command') + args = parser.parse_args(argv) + verdict, reason, _ = ask(args.command, args.need_mb) + print({'free': 'free', 'busy': f'busy: {reason}'}.get(verdict, f'not available: {reason}')) + return {'free': 0, 'busy': BUSY}.get(verdict, 1) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/fork-kit/common/machine.example.json b/fork-kit/common/machine.example.json index aa2cb5e9e7a1f..0f1821df39b37 100644 --- a/fork-kit/common/machine.example.json +++ b/fork-kit/common/machine.example.json @@ -31,6 +31,7 @@ "host": "fleet-host.example", "hostKey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "slots": 2, - "reserveMb": 4096 + "reserveMb": 4096, + "freeCommand": "/opt/fork-fleet/free-memory" } } diff --git a/fork-kit/common/tests/free_contract.test.py b/fork-kit/common/tests/free_contract.test.py new file mode 100644 index 0000000000000..8423a0ffd31be --- /dev/null +++ b/fork-kit/common/tests/free_contract.test.py @@ -0,0 +1,299 @@ +"""Contract tests for free/v1 (fork-kit/common/FREE.md): free_contract.py against free-fixtures.json (the cases +every reader shares), running real scripts (time limit, output limit, process group, safety), its command line, +and the example scripts in free/. + + python3 -I -B fork-kit/common/tests/free_contract.test.py +""" +import datetime, importlib.machinery, importlib.util, json, os, shutil, subprocess, sys, tempfile, time, unittest + +COMMON = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +READER = os.path.join(COMMON, 'free_contract.py') + + +def load(): + loader = importlib.machinery.SourceFileLoader('free_contract', READER) + mod = importlib.util.module_from_spec(importlib.util.spec_from_loader('free_contract', loader)) + loader.exec_module(mod) + return mod + + +fc = load() + + +def now_text(delta=0): + t = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=delta) + return t.strftime('%Y-%m-%dT%H:%M:%SZ') + + +class Fixtures(unittest.TestCase): + def test_every_shared_case(self): + with open(os.path.join(COMMON, 'free-fixtures.json'), encoding='utf-8') as f: + data = json.load(f) + now = fc.parse_at(data['now']) + self.assertGreater(len(data['cases']), 20) + for case in data['cases']: + with self.subTest(case['name']): + out = case['stdout'] + if 'padTo' in case: + out = out.replace('PAD', 'x' * (case['padTo'] - len(out) + 3)) + self.assertEqual(len(out), case['padTo']) + verdict, reason, facts = fc.judge(case['exit'], out.encode(), now, case.get('needMb')) + self.assertEqual(verdict, case['verdict'], reason) + self.assertEqual(bool(reason), verdict != 'free') + self.assertEqual(facts is None, verdict == 'unavailable') + + +class Ask(unittest.TestCase): + def setUp(self): + self.d = os.path.realpath(tempfile.mkdtemp(prefix='free-contract-test-')) + self.addCleanup(shutil.rmtree, self.d, True) + + def script(self, body, name='free', mode=0o755): + path = os.path.join(self.d, name) + with open(path, 'w') as f: + f.write('#!/bin/sh\n' + body) + os.chmod(path, mode) + return path + + def test_free_busy_and_the_environment_it_gets(self): + s = self.script(f'env > {self.d}/env; pwd > {self.d}/pwd; echo "$#" > {self.d}/argc\n' + f'echo \'{{"v": 1, "free": true, "at": "{now_text()}"}}\'\n') + self.assertEqual(fc.ask(s)[0], 'free') + with open(f'{self.d}/env') as f: + names = sorted(line.split('=', 1)[0] for line in f.read().splitlines()) + self.assertEqual([n for n in names if n not in ('PWD', 'SHLVL', '_', 'OLDPWD')], ['LANG', 'PATH']) + with open(f'{self.d}/pwd') as f, open(f'{self.d}/argc') as g: + self.assertEqual((f.read(), g.read()), ('/\n', '0\n')) + busy = f'{{"v": 1, "free": false, "reason": "a game\\nis on", "at": "{now_text()}"}}' + s = self.script(f'printf "%s\\n" \'{busy}\'\nexit 75\n') # not echo: in dash it turns \\n into a newline + self.assertEqual(fc.ask(s)[:2], ('busy', 'a game is on')) + + def test_no_answer_in_time_ends_its_whole_group(self): + s = self.script(f'sleep 30 & echo $! > {self.d}/child\nsleep 30\n') + t = time.monotonic() + verdict, reason, _ = fc.ask(s, timeout=1) + self.assertLess(time.monotonic() - t, 10) + self.assertEqual(verdict, 'unavailable') + self.assertIn('no answer', reason) + with open(f'{self.d}/child') as f: + child = int(f.read()) + for _ in range(50): + try: + with open(f'/proc/{child}/stat') as f: + if f.read().rsplit(')', 1)[1].split()[0] == 'Z': + break + except (FileNotFoundError, ProcessLookupError): # gone between the open and the read + break + time.sleep(0.1) + else: + self.fail('the script\'s background child still runs') + + def test_a_child_that_keeps_stdout_open_is_no_answer(self): + s = self.script(f'(sleep 30) &\necho \'{{"v": 1, "free": true, "at": "{now_text()}"}}\'\n') + self.assertEqual(fc.ask(s, timeout=1)[0], 'unavailable') + + def test_endless_output_is_cut_off(self): + s = self.script('yes\n') + verdict, reason, _ = fc.ask(s, timeout=5) + self.assertEqual(verdict, 'unavailable') + self.assertIn('4096 bytes', reason) + + def test_unsafe_scripts_never_run(self): + mark = os.path.join(self.d, 'ran') + body = f': > {mark}\necho \'{{"v": 1, "free": true, "at": "{now_text()}"}}\'\n' + cases = {'relative': lambda: 'free', + 'dot-dot': lambda: self.script(body).replace('/free', '/../' + os.path.basename(self.d) + '/free'), + 'writable by others': lambda: self.script(body, mode=0o757), + 'not executable': lambda: self.script(body, mode=0o644), + 'a folder': lambda: self.d, + 'missing': lambda: os.path.join(self.d, 'nothing')} + for name, make in cases.items(): + with self.subTest(name): + verdict, reason, _ = fc.ask(make()) + self.assertEqual(verdict, 'unavailable') + self.assertFalse(os.path.exists(mark), reason) + os.chmod(self.d, 0o777) # a folder others may change, no sticky bit + try: + self.assertIn('may be changed by others', fc.ask(self.script(body))[1]) + finally: + os.chmod(self.d, 0o700) + self.assertFalse(os.path.exists(mark)) + self.assertIn('belongs to uid', fc.unsafe(self.script(body), uid=os.geteuid() + 12345) or '') + os.chmod(self.d, 0o1777) # sticky, like /tmp: fine + self.assertEqual(fc.ask(self.script(body))[0], 'free') + + def test_a_link_is_judged_by_where_it_leads(self): + s = self.script(f'echo \'{{"v": 1, "free": true, "at": "{now_text()}"}}\'\n') + os.symlink(s, os.path.join(self.d, 'link')) + self.assertEqual(fc.ask(os.path.join(self.d, 'link'))[0], 'free') + + def test_a_link_another_user_owns_in_a_sticky_folder_is_not_followed(self): + # what fs.protected_symlinks refuses; resolving in Python must not quietly follow it to a root program + real, geteuid = os.geteuid(), os.geteuid + with tempfile.TemporaryDirectory() as d: + os.chmod(d, 0o1777) + link = os.path.join(d, 'free') + os.symlink(os.path.realpath(shutil.which('true')), link) + os.geteuid = lambda: real + 4242 + try: + verdict, reason, _ = fc.ask(link) + finally: + os.geteuid = geteuid + self.assertEqual(verdict, 'unavailable') + self.assertNotEqual(reason, 'stdout is not JSON') # true ran: the link was followed + self.assertIn(f'uid {real}', reason) + + def test_the_folders_of_the_path_as_written_are_judged_too(self): + body = f': > {self.d}/ran\necho \'{{"v": 1, "free": true, "at": "{now_text()}"}}\'\n' + good = self.script(body) + open_ = os.path.join(self.d, 'open') + os.mkdir(open_) + os.symlink(good, os.path.join(open_, 'free')) + os.chmod(open_, 0o777) # others may swap the link for one to any program they like + verdict, reason, _ = fc.ask(os.path.join(open_, 'free')) + self.assertEqual((verdict, reason), ('unavailable', f'{open_} may be changed by others')) + self.assertFalse(os.path.exists(f'{self.d}/ran')) + os.chmod(open_, 0o755) + self.assertEqual(fc.ask(os.path.join(open_, 'free'))[0], 'free') + + def test_relative_links_chains_and_loops(self): + good = self.script(f'echo \'{{"v": 1, "free": true, "at": "{now_text()}"}}\'\n') + os.mkdir(os.path.join(self.d, 'sub'), 0o755) + os.chmod(os.path.join(self.d, 'sub'), 0o755) # whatever the umask + os.symlink('../free', os.path.join(self.d, 'sub', 'one')) + os.symlink('one', os.path.join(self.d, 'sub', 'two')) + os.symlink(os.path.join(self.d, 'sub'), os.path.join(self.d, 'dir')) + self.assertEqual(fc.resolve(os.path.join(self.d, 'dir', 'two')), (good, None)) + self.assertEqual(fc.ask(os.path.join(self.d, 'dir', 'two'))[0], 'free') + os.symlink('b', os.path.join(self.d, 'a')) + os.symlink('a', os.path.join(self.d, 'b')) + verdict, reason, _ = fc.ask(os.path.join(self.d, 'a')) + self.assertEqual(verdict, 'unavailable') + self.assertIn('too many levels of links', reason) + + def test_a_script_it_may_not_end_is_not_waited_for_without_a_limit(self): + s = self.script(f'echo $$ > {self.d}/pid\necho \'{{"v": 1, "free": true, "at": "{now_text()}"}}\'\n' + 'exec sleep 30 >&-\n') + def refused(*_): + raise PermissionError(1, 'Operation not permitted') + killpg, kill = os.killpg, os.kill + os.killpg = os.kill = refused # a set-uid script: its group is not ours to signal + t = time.monotonic() + try: + verdict = fc.ask(s, timeout=1)[0] + finally: + os.killpg, os.kill = killpg, kill + with open(f'{self.d}/pid') as f: + pid = int(f.read()) + try: + os.kill(pid, 9) + except ProcessLookupError: + pass + self.assertLess(time.monotonic() - t, 10) + self.assertEqual(verdict, 'unavailable') + + def test_a_link_repointed_after_the_check_does_not_change_what_runs(self): + good = self.script(f': > {self.d}/good-ran\necho \'{{"v": 1, "free": true, "at": "{now_text()}"}}\'\n', + name='good') + bad = self.script(f': > {self.d}/bad-ran\necho \'{{"v": 1, "free": true, "at": "{now_text()}"}}\'\n', + name='bad') + link = os.path.join(self.d, 'link') + os.symlink(good, link) + checked = fc.unsafe + + def check_then_repoint(command, uid=None): + why = checked(command, uid) + os.unlink(link) + os.symlink(bad, link) # what another user could do in a folder they may change + return why + fc.unsafe = check_then_repoint + try: + self.assertEqual(fc.ask(link)[0], 'free') + finally: + fc.unsafe = checked + self.assertEqual((os.path.exists(f'{self.d}/good-ran'), os.path.exists(f'{self.d}/bad-ran')), (True, False)) + + def test_a_child_that_left_the_group_and_holds_stdout_is_ended(self): + s = self.script(f'setsid sh -c \'echo $$ > {self.d}/child; exec sleep 30\' &\n' + f'echo \'{{"v": 1, "free": true, "at": "{now_text()}"}}\'\n') + self.assertEqual(fc.ask(s, timeout=1)[0], 'unavailable') + with open(f'{self.d}/child') as f: + child = int(f.read()) + for _ in range(50): + try: + with open(f'/proc/{child}/stat') as f: + if f.read().rsplit(')', 1)[1].split()[0] == 'Z': + break + except (FileNotFoundError, ProcessLookupError): + break + time.sleep(0.1) + else: + os.kill(child, 9) + self.fail('the child that left the group with setsid still runs') + + def test_the_command_line(self): + def cli(*args): + return subprocess.run([sys.executable, '-I', '-B', READER, *args], capture_output=True, text=True, + timeout=30) + s = self.script(f'echo \'{{"v": 1, "free": true, "at": "{now_text()}", "memory": {{"freeMb": 500}}}}\'\n') + got = cli('ask', s) + self.assertEqual((got.returncode, got.stdout), (0, 'free\n')) + got = cli('ask', '--need-mb', '501', s) + self.assertEqual((got.returncode, got.stdout), (75, 'busy: 500 MB free, 501 MB needed\n')) + got = cli('ask', 'relative') + self.assertEqual(got.returncode, 1) + self.assertTrue(got.stdout.startswith('not available: ')) + self.assertEqual(cli('ask').returncode, 2) + + +class Examples(unittest.TestCase): + """Each example answers by the contract on this machine (free or busy, never not available). Run from a copy in + a folder of its own: a checkout's folders may be group-writable, and then the reader rightly refuses them.""" + + def copy(self, name, text=None): + d = os.path.realpath(tempfile.mkdtemp(prefix='free-contract-test-')) + self.addCleanup(shutil.rmtree, d, True) + if text is None: + with open(os.path.join(COMMON, 'free', name)) as f: + text = f.read() + path = os.path.join(d, name) + with open(path, 'w') as f: + f.write(text) + os.chmod(path, 0o755) + return path + + def test_memory(self): + verdict, reason, facts = fc.ask(self.copy('memory')) + self.assertIn(verdict, ('free', 'busy'), reason) + self.assertIsInstance(facts['memory']['freeMb'], int) + + def test_busy_while_running(self): + with open(os.path.join(COMMON, 'free', 'busy-while-running')) as f: + text = f.read() + self.assertEqual(fc.ask(self.copy('busy-while-running'))[0], 'free') + names = "NAMES='example-game example-editor'" + self.assertIn(names, text) + copy = self.copy('busy', text.replace(names, "NAMES='not-running sleep'")) + p = subprocess.Popen(['sleep', '30']) + try: + self.assertEqual(fc.ask(copy)[:2], ('busy', 'sleep is running')) + finally: + p.kill() + p.wait() + + def test_busy_while_running_knows_names_longer_than_15_characters(self): + with open(os.path.join(COMMON, 'free', 'busy-while-running')) as f: + text = f.read() + copy = self.copy('busy', text.replace("NAMES='example-game example-editor'", "NAMES='sleeps-a-long-while'")) + d = os.path.dirname(copy) + shutil.copy(shutil.which('sleep'), os.path.join(d, 'sleeps-a-long-while')) + p = subprocess.Popen([os.path.join(d, 'sleeps-a-long-while'), '30']) + try: + self.assertEqual(fc.ask(copy)[:2], ('busy', 'sleeps-a-long-while is running')) + finally: + p.kill() + p.wait() + + +if __name__ == '__main__': + unittest.main() diff --git a/fork-kit/desk/README.md b/fork-kit/desk/README.md index 7da0fa4d9985d..b94cb3ae4cf0f 100644 --- a/fork-kit/desk/README.md +++ b/fork-kit/desk/README.md @@ -27,7 +27,9 @@ The facts of this machine go in the `desk` section of `/etc/fork-kit/machine.jso `--fleet-config` has its own pools); `host` and `hostKey` are the always-on host and its ssh host key (on that host: `cat /etc/ssh/ssh_host_ed25519_key.pub`, without the comment). Optional: `port` (22), `user` (`fleet-desk`), `slots` (1, at most 8; one runner user each), `reserveMb` (memory the launcher keeps free for the owner), `netBase` (`10.N`, the runners' network, -default `10.204`: pick another when this machine's network already uses it). +default `10.204`: pick another when this machine's network already uses it), `freeCommand` (a free/v1 script, +[FREE.md](../common/FREE.md): `fits` then also needs it to answer free; the kit installs an example as +`/opt/fork-fleet/free-memory`; default none, the memory rule alone). ## Install, check, uninstall diff --git a/fork-kit/desk/sbin/desk-install b/fork-kit/desk/sbin/desk-install index 5ff5e9be200af..4dccb54862cd7 100755 --- a/fork-kit/desk/sbin/desk-install +++ b/fork-kit/desk/sbin/desk-install @@ -36,7 +36,8 @@ FILES = {'fork-fleet-launch.sh': '.github/scripts/fork-fleet-launch.sh', 'fork-fleet-desk-wire.py': '.github/scripts/fork-fleet-desk-wire.py', 'hook.sh': '.github/scripts/fork-fleet-hook.sh', 'fork-fleet.py': '.github/scripts/fork-fleet.py', 'fork-fleet.json': '.github/fork-fleet.json', 'desk-install': 'fork-kit/desk/sbin/desk-install', - 'machine_file.py': 'fork-kit/common/machine_file.py', 'README.md': 'fork-kit/desk/README.md'} + 'machine_file.py': 'fork-kit/common/machine_file.py', 'README.md': 'fork-kit/desk/README.md', + 'free_contract.py': 'fork-kit/common/free_contract.py', 'free-memory': 'fork-kit/common/free/memory'} WALLS = '.github/scripts/fork-fleet-walls' # copied whole: every OS's walls file and fixture UNITS = {'fork-fleet-desk-agent.service': '.github/scripts/fork-fleet-desk-agent.service', 't3code-fleet-refresh.service': '.github/scripts/fork-fleet-refresh.service', @@ -51,6 +52,7 @@ ENV = {'PATH': '/usr/sbin:/usr/bin:/sbin:/bin', 'LC_ALL': 'C.UTF-8'} SHA = re.compile(r'[0-9a-f]{40}') SHAPES = {'pool': r'[a-z0-9][a-z0-9_-]{0,39}', 'host': r'[A-Za-z0-9][A-Za-z0-9.-]{0,252}', 'user': r'[a-z_][a-z0-9_-]{0,31}', 'netBase': r'10\.(0|[1-9][0-9]?|1[0-9]{2}|2[0-4][0-9]|25[0-5])', + 'freeCommand': r'(/[A-Za-z0-9._@+-]+)+', 'hostKey': r'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI[A-Za-z0-9+/]{43}'} RUNNER_UNIT = re.compile(r'fork-fleet-[a-z0-9][a-z0-9-]{0,62}-[0-9]{1,20}\.service') # the launcher's runner units @@ -133,7 +135,8 @@ def machine_section(): 'port': (int, 22, within(1, 65535)), 'user': (str, 'fleet-desk', shape('user')), 'hostKey': (str, mf.REQUIRED, shape('hostKey')), 'slots': (int, 1, within(1, 8)), 'reserveMb': (int, None, within(0, 1000000)), - 'netBase': (str, None, shape('netBase'))}, 'desk') + 'netBase': (str, None, shape('netBase')), + 'freeCommand': (str, None, shape('freeCommand'))}, 'desk') except mf.MachineFileError as e: raise Refused(f'machine file: {e}') from None @@ -235,7 +238,7 @@ def stage(source, sha, fleet_config, pool): dst = os.path.join(tmp, rel) os.makedirs(os.path.dirname(dst), mode=0o755, exist_ok=True) shutil.copyfile(src, dst) - executable = rel in ('fork-fleet-launch.sh', 'hook.sh', 'desk-install') or ( + executable = rel in ('fork-fleet-launch.sh', 'hook.sh', 'desk-install', 'free-memory') or ( rel.startswith('fork-fleet-walls/') and not rel.endswith('.json')) os.chmod(dst, 0o755 if executable else 0o644) manifest[rel] = sha256(dst) @@ -337,7 +340,8 @@ def settings(m, users): known = (m['host'] if m['port'] == 22 else f'[{m["host"]}]:{m["port"]}') + ' ' + m['hostKey'] files = {ETC + '/launch.conf': (f'RUNNER_USERS={",".join(users)}\n' + (f'RESERVE_MB={m["reserveMb"]}\n' if m['reserveMb'] is not None else '') - + (f'NET_BASE={m["netBase"]}\n' if m['netBase'] else ''), 0o600), + + (f'NET_BASE={m["netBase"]}\n' if m['netBase'] else '') + + (f'FREE_COMMAND={m["freeCommand"]}\n' if m['freeCommand'] else ''), 0o600), ETC + '/desk.conf': (f'POOL={m["pool"]}\nHOST={m["host"]}\nPORT={m["port"]}\nUSER={m["user"]}\n', 0o600), ETC + '/desk-known_hosts': (known + '\n', 0o644), ETC + '/resolv.conf': ('nameserver 1.1.1.1\nnameserver 9.9.9.9\n', 0o644), @@ -504,6 +508,9 @@ def check(): m = machine_section() users = [f'fork-runner-{i}' for i in range(1, m['slots'] + 1)] row(True, 'machine-file', f'desk section: pool {m["pool"]}, {m["slots"]} slot(s), always-on host {m["host"]}') + if m['freeCommand']: # busy is fine: the script answers by the contract (FREE.md) + r = py(p(OPT + '/free_contract.py'), 'ask', m['freeCommand']) + row(r.returncode in (0, 75), 'free-script', f'{m["freeCommand"]}: {(r.stdout or r.stderr).strip()[:200]}') stale = [path for path, (text, mode) in settings(m, users).items() if not same_text(path, text, mode)] row(not stale, 'settings', f'{stale[0]} differs from the machine file: install again' if stale else 'the written settings match the machine file') diff --git a/fork-kit/desk/tests/install.test.py b/fork-kit/desk/tests/install.test.py index e00bef21edfe1..1432d38cb3fa3 100644 --- a/fork-kit/desk/tests/install.test.py +++ b/fork-kit/desk/tests/install.test.py @@ -4,7 +4,8 @@ python3 -I -B fork-kit/desk/tests/install.test.py """ -import contextlib, importlib.machinery, importlib.util, io, json, os, shutil, stat, subprocess, sys, tempfile, unittest +import contextlib, datetime, importlib.machinery, importlib.util, io, json, os, shutil, stat, subprocess, sys +import tempfile, unittest KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) REPO = os.path.dirname(os.path.dirname(KIT)) @@ -140,7 +141,8 @@ def release(self, name='release', extra=None): if text is not None: self.write(os.path.join(top, rel), text.replace('{log}', os.path.join(self.fakes, 'calls.log')) .replace('{here}', self.fakes), 0o755) - for rel in ('fork-kit/desk/sbin/desk-install', 'fork-kit/desk/README.md', 'fork-kit/common/machine_file.py'): + for rel in ('fork-kit/desk/sbin/desk-install', 'fork-kit/desk/README.md', 'fork-kit/common/machine_file.py', + 'fork-kit/common/free_contract.py', 'fork-kit/common/free/memory'): self.write(os.path.join(top, rel), slurp(os.path.join(REPO, rel)), 0o755) return top @@ -458,6 +460,25 @@ def test_reserve_and_port_reach_the_settings(self): self.assertTrue(self.read('/etc/fork-fleet/desk-known_hosts').endswith( f'[fleet-host.example]:2222 {HOST_KEY}\n')) + def test_a_free_script_reaches_the_settings_and_the_check(self): + script = os.path.join(self.tmp, 'free') + self.machine({'version': 1, 'desk': {**MACHINE['desk'], 'freeCommand': script}}) + self.assertEqual(self.install()[0], 0) + self.assertIn(f'FREE_COMMAND={script}\n', self.read('/etc/fork-fleet/launch.conf')) + self.assertEqual(stat.S_IMODE(os.stat(self.path('/opt/fork-fleet/free-memory')).st_mode), 0o755) + self.template() + now = datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + for code, body, want in ((0, '"free": true', 'PASS free-script ' + script + ': free\n'), + (75, '"free": false, "reason": "a game"', 'busy: a game'), + (75, '"free": true', 'FAIL free-script')): + self.write(script, f"#!/bin/sh\necho '{{\"v\": 1, {body}, \"at\": \"{now}\"}}'\nexit {code}\n", 0o755) + out = self.run_main('check')[1] + self.assertIn(want, out) + self.machine({'version': 1, 'desk': {**MACHINE['desk'], 'freeCommand': 'free'}}) + code, _, err = self.install() + self.assertEqual(code, 1) + self.assertIn('freeCommand', err) + def test_existing_user_is_used_not_recorded(self): self.write(os.path.join(self.fakes, 'users'), 'fork-runner-1\n') self.assertEqual(self.install()[0], 0)