diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index 58e58018f94aa..495cdceec99f6 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -42,6 +42,7 @@ fork-ci/actions/pr-queue/lens.test.py fork-ci/actions/pr-queue/pr_queue.test.py fork-ci/actions/pr-queue/tested-base/tested_base.test.py fork-kit/common/tests/machine_file.test.py +fork-kit/desk/tests/install.test.py fork-kit/golive/tests/check.test.py fork-kit/golive/tests/contract.test.py fork-kit/golive/tests/journal.test.py diff --git a/.github/fork-fleet.md b/.github/fork-fleet.md index a7be89e2d3339..a42c9379baa9f 100644 --- a/.github/fork-fleet.md +++ b/.github/fork-fleet.md @@ -642,8 +642,9 @@ beside the walls (one kit: the one those walls read), so `desk.conf` needs no `LAUNCH` line; an old one that names another launcher makes the agent refuse to run (exit 2) until the line is gone. The agent fixes the kit when it starts, in its real path: a link above it that an upgrade repoints changes nothing -until the agent restarts, so restart it after an upgrade. Then, once per -desktop: +until the agent restarts, so restart it after an upgrade. The desk kit +([fork-kit/desk](../fork-kit/desk/README.md)) does all of this, with a PASS/FAIL +check and an uninstall; by hand, once per desktop: ```sh install -d -m 755 /etc/fork-fleet diff --git a/fork-kit/README.md b/fork-kit/README.md index eac02830f821d..0f3680cd3d5c3 100644 --- a/fork-kit/README.md +++ b/fork-kit/README.md @@ -15,6 +15,7 @@ Every kit follows the same pattern: | Kit | What | | ---------------------------- | ------------------------------------------------------------------------------------------------------- | +| [desk](desk/README.md) | A desktop's side of a fork fleet desk pool: runner users, walls, desk agent; one folder (Linux, WSL2) | | [gh-apps](gh-apps/README.md) | GitHub App tokens for agents through `gh-as`, without giving agents the keys (Linux) | | [golive](golive/README.md) | Move a T3 boot service to a new build with a cold copy first; core and `check` so far (Linux) | | [heavy](heavy/README.md) | `heavy `: run builds and tests in a capped slice, stop what they leave behind (Linux, systemd) | diff --git a/fork-kit/common/machine.example.json b/fork-kit/common/machine.example.json index f2173a8f51d48..aa2cb5e9e7a1f 100644 --- a/fork-kit/common/machine.example.json +++ b/fork-kit/common/machine.example.json @@ -25,5 +25,12 @@ "backupDir": "/srv/example/golive/backups", "stateDir": "/srv/example/golive/state", "freeFloorMb": 2048 + }, + "desk": { + "pool": "pc", + "host": "fleet-host.example", + "hostKey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "slots": 2, + "reserveMb": 4096 } } diff --git a/fork-kit/desk/README.md b/fork-kit/desk/README.md new file mode 100644 index 0000000000000..703577dcd3c4c --- /dev/null +++ b/fork-kit/desk/README.md @@ -0,0 +1,102 @@ +# desk: a desktop's side of a fork fleet desk pool + +Turns a Linux machine (or WSL2 on Windows) into one pool of the fork fleet: the desk agent dials out to the +always-on host over ssh, and the dispatcher there starts one-job runners on this machine through it. Everything +the fleet runbook ([.github/fork-fleet.md](../../.github/fork-fleet.md), "The desktop's side") lists by hand, +done by one installer, with a PASS/FAIL check and an uninstall. + +## The machine file + +The facts of this machine go in the `desk` section of `/etc/fork-kit/machine.json` (root-owned; see +[common](../common/README.md)). Made-up example: + +```json +{ + "version": 1, + "desk": { + "pool": "pc", + "host": "fleet-host.example", + "hostKey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI...", + "slots": 2, + "reserveMb": 4096 + } +} +``` + +`pool` must be a pool in the fleet config that is installed (the release's own has `pc`; a project's +`--fleet-config` has its own pools); `host` and `hostKey` are the always-on host and its ssh host key +(on that host: `cat /etc/ssh/ssh_host_ed25519_key.pub`, without the comment). Optional: `port` (22), `user` (`fleet-desk`), `slots` (1, at most 8; one +runner user each), `reserveMb` (memory the launcher keeps free for the owner), `netBase` (`10.N`, the runners' network, +default `10.204`: pick another when this machine's network already uses it). + +## Install, check, uninstall + +As root, from the fork's protected release branch: + +```sh +sh bootstrap.sh --repo OWNER/REPO --sha RELEASE_SHA [--fleet-config /path/to/project/fork-fleet.json] +sh bootstrap.sh --repo OWNER/REPO --sha RELEASE_SHA --release-fleet-config # back to the release's own +/opt/fork-fleet/desk-install check # every line PASS, then ALL PASS +/opt/fork-fleet/desk-install uninstall +``` + +`--fleet-config` is the project's fleet config (its pools); without it the release's own is used. It must be in a +folder only root (or you) can write; a link is followed, and the folders of both the link and the file it names are +checked. The project's is kept in `/etc/fork-fleet/fleet-config.json` and the choice is +recorded before any file is swapped, so a later install of a new release keeps it, even after a stopped run; +`--release-fleet-config` goes back to the release's own. Install prints the desk key's public half: add it on +the always-on host with `fork-fleet-desk-key.py add POOL`. The first runner template is made in the background by +the refresh timer; `check` says FAIL for `template` until it is there. + +`check` only reads: the installed files against their manifest (text, root owner and mode), the folders' and the +desk key's modes and root owner (not a link), the project's fleet config against the installed one, each unit against +the release's (header plus `/opt/fork-fleet/units/UNIT`, mode and root owner), a restart a stopped install still owes, systemd drop-ins that change the kit's units, +every wall of this OS (`fork-fleet-walls.py describe`), the agent's settings, the units, the template, the settings +files against the machine file (text, mode and root owner), the runner users (each must exist, have no login shell, +a locked password, a uid other than 0, its own group only, no home that is there (`/nonexistent`) and no line in +`cron.allow`/`at.allow`; and it names a user the kit made that a smaller `slots` no longer uses) and IPv4 +forwarding. + +A `fork-runner-N` user made by hand before the install is used, but only when it passes the same checks (no login +shell, locked password, uid not 0, its own group only, no home that is there, not in an allow file; the launcher +refuses the same uid and group cases at each job start): otherwise install refuses before changing anything and +names the fix (`usermod -s /usr/sbin/nologin`, `passwd -l`, `gpasswd -d`, `usermod -d /nonexistent`). A +`/opt/fork-fleet`, `/opt/fork-fleet/template`, `/etc/fork-fleet` or desk key that is a link or not owned by root (a +hand setup) is refused the same way, not followed or fixed. One install or uninstall runs at a time (a lock on +`/etc/fork-fleet`, which only root can open); a second one is refused. + +## Runs again, stops half-way, old or empty state + +- **Again:** files, settings and units that did not change are not rewritten (a file in `/opt/fork-fleet`, a + setting or a unit whose mode or owner was changed, or that is a link, is written again as root's; the folders' + modes are set back; a linked or foreign-owned folder or key is refused, above), and the agent is not stopped or + restarted (a restart would end a job it is starting). After a change it is restarted. The agent reads its settings + only when it starts, so before the first change the record gets a restart-pending mark, cleared after the + restart: a run stopped in between restarts the agent on the next run. The mark is also set before a new fleet + config choice is saved, so a run stopped between saving the choice and the swap fails `check` until the next run. +- **Stopped half-way:** the next install ends it. A new release is staged in `/opt/fork-fleet/.kit-stage-*` and + checked first (gate loads with the fleet config, the pool is in it, `bash -n`, the walls know this OS); a release + that fails is never installed. Files are renamed in one by one while the agent and the refresh timer are stopped; + `SOURCE` is written + last, so `check` reports a half-done swap. Users are recorded in `/etc/fork-fleet/desk-kit.json` before they are + made, so uninstall removes them even after a stop. +- **Old record:** a record that names a project's fleet config while neither `/etc/fork-fleet/fleet-config.json` + nor the installed copy is there is refused with the fix (`--fleet-config FILE` or `--release-fleet-config`). +- **Old install:** a launcher, `launch.conf`, `desk.conf`, desk key, unit, or a systemd drop-in for one of the kit's + units (`UNIT.d/*.conf`, also the dash-prefix forms) that this kit did not write is refused before anything changes. + The type-wide `service.d/` and `timer.d/` folders are not checked: they hold the distribution's defaults for every + unit (refusing them would refuse a stock system), and only root can write them. Remove the older (by hand) install first, the runbook's steps in reverse. +- **Empty:** `check` before an install says FAIL and writes nothing; `uninstall` with no record does nothing. + +Uninstall stops every runner (through the launcher, then `systemctl stop` for any left). While one still runs it +stops there and changes nothing else: the `fork_fleet` nft table, users and record stay; run it again. Then it +removes every `fork-fleet-*` namespace and `ffh-*` link, the nft table, the kit's units, the files of its manifest, +the template and runner copies, its settings and key, the users it made, and `cron.deny`/`at.deny` when the kit +made them (otherwise the lines it added, for every runner user, also one that was there before). The record goes last. Anything else in its folders (a file the kit +did not make, a drop-in) is kept and named. IPv4 forwarding stays on until the next reboot. + +## Why one flat folder + +The other kits keep one folder per version behind one link. The desk agent refuses a launcher or walls file whose +folder path has a link in it (so nobody can point it elsewhere), and every fleet default names `/opt/fork-fleet`, so +this kit installs flat there: changed files are renamed in, a manifest records each hash. diff --git a/fork-kit/desk/bootstrap.sh b/fork-kit/desk/bootstrap.sh new file mode 100755 index 0000000000000..7465eba5c60ae --- /dev/null +++ b/fork-kit/desk/bootstrap.sh @@ -0,0 +1,71 @@ +#!/bin/sh +# desk bootstrap: installs the desk kit (a desktop's side of a fork fleet desk pool) of one release of this fork. +# Run as root on the desktop (on WSL2: inside the distro), after the owner wrote the machine file's `desk` section: +# +# sh desk-bootstrap.sh --repo FORK_OWNER/FORK --sha RELEASE_SHA [--fleet-config FILE | --release-fleet-config] +# +# --fleet-config: the fleet config of the project whose jobs this desktop runs (that repository's file, at the commit +# its always-on host uses); without it, the release's own .github/fork-fleet.json (the fork's CI), or the project's +# from the last install. --release-fleet-config: back to the release's own after a project's was used. +# It refuses a sha that is not a commit on the fork's protected release branch (ci-release), downloads exactly that +# commit, prints the sha and the hash of what it downloaded, and runs its installer (safe to run again). +# Needs: curl, tar, python3. Linux with systemd only (WSL2 with systemd on). +set -eu +umask 022 +PATH=/usr/sbin:/usr/bin:/sbin:/bin +export PATH +unset TMPDIR CURL_HOME SSL_CERT_FILE SSL_CERT_DIR || true + +die() { echo "desk-bootstrap: $*" >&2; exit 1; } + +repo='' sha='' branch='ci-release' fleet='' release_fleet='' api='https://api.github.com' +codeload='https://codeload.github.com' +while [ $# -gt 0 ]; do + case "$1" in + --repo | --sha | --release-branch | --fleet-config) [ $# -ge 2 ] || die "$1 needs a value" ;; + esac + case "$1" in + --repo) repo="$2"; shift 2 ;; + --sha) sha="$2"; shift 2 ;; + --release-branch) branch="$2"; shift 2 ;; + --fleet-config) fleet="$2"; shift 2 ;; + --release-fleet-config) release_fleet=1; shift ;; + *) die "unknown option $1 (see the top of this file)" ;; + esac +done +printf '%s\n' "$repo" | grep -Eqx '[A-Za-z0-9][A-Za-z0-9-]{0,38}/[A-Za-z0-9._-]{1,100}' || die "--repo must be OWNER/NAME" +printf '%s\n' "$sha" | grep -Eqx '[0-9a-f]{40}' || die "--sha must be the full 40-character release commit" +printf '%s\n' "$branch" | grep -Eqx '[A-Za-z0-9._/-]{1,100}' || die "bad --release-branch" +[ -z "$fleet" ] || [ -f "$fleet" ] || die "--fleet-config $fleet: no such file" +[ -z "$fleet" ] || [ -z "$release_fleet" ] || die "--fleet-config and --release-fleet-config: one or the other" +[ "$(id -u)" = 0 ] || die "run this as root" + +work="$(mktemp -d /tmp/desk-bootstrap.XXXXXXXX)" +trap 'rm -rf "${work:?}"' EXIT +trap 'exit 1' INT TERM HUP +fetch() { curl -fsSL --proto '=https' --tlsv1.2 --max-redirs 0 -H 'Accept: application/vnd.github+json' -o "$2" "$1"; } + +fetch "$api/repos/$repo/branches/$branch" "$work/branch.json" || die "cannot read branch $branch of $repo" +fetch "$api/repos/$repo/compare/$sha...$branch" "$work/compare.json" || die "cannot compare $sha with $branch" +python3 -I -B - "$work/branch.json" "$work/compare.json" "$sha" <<'PY' || die "$sha is not a release: it must be a commit on the protected branch $branch" +import json, sys +branch, compare = (json.load(open(p, encoding='utf-8')) for p in sys.argv[1:3]) +# base_commit is always the sha asked for; merge_base_commit is the newest commit both share: it is the sha itself +# only when the sha is on the branch +ok = (branch.get('protected') is True and compare.get('status') in ('ahead', 'identical') + and (compare.get('merge_base_commit') or {}).get('sha') == sys.argv[3]) +sys.exit(0 if ok else 1) +PY + +# codeload.github.com answers a commit tarball directly (no redirect) +fetch "$codeload/$repo/tar.gz/$sha" "$work/kit.tar.gz" || die "cannot download $repo at $sha" +mkdir "$work/src" +tar -xzf "$work/kit.tar.gz" -C "$work/src" --no-same-owner --no-same-permissions +top="$(find "$work/src" -mindepth 1 -maxdepth 1 -type d)" +[ "$(printf '%s\n' "$top" | wc -l)" = 1 ] && [ -n "$top" ] || die "the download does not have one top folder" +[ -f "$top/fork-kit/desk/sbin/desk-install" ] || die "release $sha has no fork-kit/desk (is it a release with the kit?)" +echo "desk-bootstrap: release $sha of $repo, download sha256 $(sha256sum "$work/kit.tar.gz" | cut -d' ' -f1)" +set -- install --source "$top" --sha "$sha" +[ -z "$fleet" ] || set -- "$@" --fleet-config "$fleet" +[ -z "$release_fleet" ] || set -- "$@" --release-fleet-config +python3 -I -B "$top/fork-kit/desk/sbin/desk-install" "$@" diff --git a/fork-kit/desk/sbin/desk-install b/fork-kit/desk/sbin/desk-install new file mode 100755 index 0000000000000..8a24cd7cbe214 --- /dev/null +++ b/fork-kit/desk/sbin/desk-install @@ -0,0 +1,819 @@ +#!/usr/bin/python3 -IB +"""desk-install: a desktop's side of a fork fleet desk pool, as root (fork-kit/desk/README.md). + + desk-install install --source RELEASE_FOLDER --sha RELEASE_SHA [--fleet-config FILE | --release-fleet-config] + desk-install check every part and wall, PASS or FAIL; reads only (safe while a job runs) + desk-install uninstall + +install reads the `desk` section of the machine file, makes the runner users, stages the fleet files of one release +(launcher, walls, desk agent, gate hook and the fleet config), checks the staged copy, stops the agent, moves each +file into /opt/fork-fleet, writes MANIFEST.sha256 and then SOURCE (last), writes the settings in /etc/fork-fleet and +the units, makes the desk key once, and starts the agent and the refresh timer (restarting the agent only when +something changed). Safe to run again and to stop at any step: a stopped run leaves SOURCE naming the old release +(check says what differs) and the next run ends it. uninstall removes only what the kit made, its record last. +""" +import argparse, contextlib, fcntl, hashlib, importlib.util, json, os, pwd, re, shutil, stat, subprocess, sys, tempfile + +PREFIX = '' # tests only: every fixed path below is read under this folder +ROOT_UID = 0 # tests only: who must run this and own the installed files +MACHINE = None # tests only: the machine file (None: the fixed path) +OPT, ETC, WORK = '/opt/fork-fleet', '/etc/fork-fleet', '/var/lib/fork-fleet' +STATE = ETC + '/desk-kit.json' +KEY = ETC + '/desk-key' +UNIT_DIR = '/etc/systemd/system' +DROPIN_ROOTS = ('/etc/systemd/system', '/run/systemd/system', '/usr/local/lib/systemd/system', '/usr/lib/systemd/system', + '/lib/systemd/system') +DENY = ('/etc/cron.deny', '/etc/at.deny') +PROJECT = ETC + '/fleet-config.json' # the project's fleet config (--fleet-config), kept for later installs +SETTINGS = ('launch.conf', 'desk.conf', 'desk-known_hosts', 'resolv.conf', 'desk-key', 'desk-key.pub', + 'fleet-config.json') +SYSCTL = '/etc/sysctl.d/60-fork-fleet.conf' +HEADER = '# desk kit: written by desk-install. Do not edit: change the machine file and install again.\n' +FILES = {'fork-fleet-launch.sh': '.github/scripts/fork-fleet-launch.sh', + 'fork-fleet-job-paths.txt': '.github/scripts/fork-fleet-job-paths.txt', + 'fork-fleet-walls.py': '.github/scripts/fork-fleet-walls.py', + 'fork-fleet-desk-agent.py': '.github/scripts/fork-fleet-desk-agent.py', + 'fork-fleet-desk-wire.py': '.github/scripts/fork-fleet-desk-wire.py', + 'hook.sh': '.github/scripts/fork-fleet-hook.sh', 'fork-fleet.py': '.github/scripts/fork-fleet.py', + 'fork-fleet.json': '.github/fork-fleet.json', 'desk-install': 'fork-kit/desk/sbin/desk-install', + 'machine_file.py': 'fork-kit/common/machine_file.py', 'README.md': 'fork-kit/desk/README.md'} +WALLS = '.github/scripts/fork-fleet-walls' # copied whole: every OS's walls file and fixture +UNITS = {'fork-fleet-desk-agent.service': '.github/scripts/fork-fleet-desk-agent.service', + 't3code-fleet-refresh.service': '.github/scripts/fork-fleet-refresh.service', + 't3code-fleet-refresh.timer': '.github/scripts/fork-fleet-refresh.timer'} +AGENT, TIMER, REFRESH = 'fork-fleet-desk-agent.service', 't3code-fleet-refresh.timer', 't3code-fleet-refresh.service' +FOREIGN = (OPT + '/fork-fleet-launch.sh', ETC + '/launch.conf', ETC + '/desk.conf', KEY, + *(f'{UNIT_DIR}/{u}' for u in UNITS)) +CMD = {k: [k] for k in ('useradd', 'userdel', 'loginctl', 'systemctl', 'sysctl', 'ssh-keygen', 'ip', 'nft', 'bash', + 'id', 'getent')} +NOLOGIN = ('/usr/sbin/nologin', '/sbin/nologin', '/bin/false', '/usr/bin/false') +CMD['python3'] = ['/usr/bin/python3'] +ENV = {'PATH': '/usr/sbin:/usr/bin:/sbin:/bin', 'LC_ALL': 'C.UTF-8'} +SHA = re.compile(r'[0-9a-f]{40}') +SHAPES = {'pool': r'[a-z0-9][a-z0-9_-]{0,39}', 'host': r'[A-Za-z0-9][A-Za-z0-9.-]{0,252}', + 'user': r'[a-z_][a-z0-9_-]{0,31}', 'netBase': r'10\.(0|[1-9][0-9]?|1[0-9]{2}|2[0-4][0-9]|25[0-5])', + 'hostKey': r'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI[A-Za-z0-9+/]{43}'} +RUNNER_UNIT = re.compile(r'fork-fleet-[a-z0-9][a-z0-9-]{0,62}-[0-9]{1,20}\.service') # the launcher's runner units + + +class Refused(Exception): + pass + + +def p(path): + return PREFIX + path + + +def say(msg): + print(f'desk-install: {msg}', flush=True) + + +def run(name, *args, check=True, **kw): + r = subprocess.run([*CMD.get(name, [name]), *args], env=ENV, stdin=subprocess.DEVNULL, capture_output=True, + text=True, timeout=kw.pop('timeout', 120), **kw) + if check and r.returncode: + raise Refused(f'{name} {" ".join(args)} failed: {(r.stderr or r.stdout).strip()[:300]}') + return r + + +def py(script, *args): + return run('python3', '-I', '-B', script, *args, check=False) + + +def sha256(path): + with open(path, 'rb') as f: + return hashlib.sha256(f.read()).hexdigest() + + +def user_exists(name): + return run('id', '-u', name, check=False).returncode == 0 + + +def user_problem(u): + """Why runner user u (made by the kit or by hand) could be used for more than one-job runners: a login shell, a + usable password (or none), uid 0, a group besides its own, a home that is there, a line in cron.allow/at.allow; + None when it is fine. The launcher's check_users refuses the same users at each job start (uid, groups). Read + through getent and id (as root).""" + r = run('getent', 'passwd', u, check=False) + fields = r.stdout.strip().split(':') + if r.returncode or len(fields) < 7: + return f'{u}: its passwd entry cannot be read' + if fields[6] not in NOLOGIN: + return f'{u} has the login shell {fields[6]!r} (usermod -s /usr/sbin/nologin {u})' + if fields[2] == '0' or not fields[2].isdigit(): + return f'{u} has uid {fields[2]}: a runner user is a user of its own, not root (userdel {u}; install makes it)' + gid, groups = run('id', '-g', u, check=False), run('id', '-G', u, check=False) + if gid.returncode or groups.returncode or groups.stdout.split() != gid.stdout.split(): + return f'{u} is in other groups ({groups.stdout.strip()}): a runner user is in its own group only (gpasswd -d)' + if fields[5] not in ('', '/nonexistent') and os.path.lexists(p(fields[5])): + return f'{u} has the home {fields[5]} (usermod -d /nonexistent {u}): a home could hold its ssh keys' + r = run('getent', 'shadow', u, check=False) + fields = r.stdout.strip().split(':') + if r.returncode or len(fields) < 2: + return f'{u}: its shadow entry cannot be read' + if not fields[1].startswith(('!', '*')): + return f'{u} has a usable password or none (passwd -l {u})' + for f in DENY: + allow = f.replace('.deny', '.allow') + if u in lines_of(allow): + return f'{u} is in {allow} (remove that line)' + return None + + +def not_root_owned(path, kind): + """Why a folder or file the root-run kit trusts is not safe (a link, another kind, not owned by root); None when it + is, or absent. Install refuses such a path rather than follow it or fix it (chown would follow a link).""" + try: + st = os.lstat(p(path)) + except FileNotFoundError: + return None + if stat.S_ISLNK(st.st_mode): + return f'{path} is a link (a hand setup?): remove the link and run again' + if not kind(st.st_mode): + return f'{path} is not a {"folder" if kind is stat.S_ISDIR else "plain file"}: remove it and run again' + if st.st_uid != ROOT_UID: + return f'{path} is not owned by root (uid {st.st_uid}): chown root {path} (or remove it) and run again' + return None + + +def same_file(real, text, mode): + """The file (not a link) holds text, has mode and is owned by root.""" + try: + st = os.lstat(real) + with open(real, encoding='utf-8') as f: + return (stat.S_ISREG(st.st_mode) and st.st_uid == ROOT_UID and stat.S_IMODE(st.st_mode) == mode + and f.read() == text) + except (OSError, UnicodeDecodeError): + return False + + +def job_writable(path): + """Why a job could change this file (it, or a folder above it, writable by others or by a runner user's group, or + owned by a runner user); None when only root or the caller can. A sticky folder (/tmp) is fine: nobody else may + rename or remove what is in it. A link is followed: the folders of the path as given and of the file it names are + both checked (the install reads the file it names).""" + uids, gids = set(), set() + for i in range(1, 9): + with contextlib.suppress(KeyError): + pw = pwd.getpwnam(f'fork-runner-{i}') + uids.add(pw.pw_uid) + gids.add(pw.pw_gid) + for path in dict.fromkeys((os.path.abspath(path), os.path.realpath(path))): + why = writable_by_job(path, uids, gids) + if why: + return why + return None + + +def writable_by_job(path, uids, gids): + while True: + try: + st = os.stat(path) + except OSError as e: + return f'{path} cannot be read ({e.strerror})' + if st.st_uid in uids: + return f'{path} belongs to a runner user' + if (st.st_mode & 0o002 or st.st_mode & 0o020 and st.st_gid in gids) and not ( + stat.S_ISDIR(st.st_mode) and st.st_mode & stat.S_ISVTX): + return f'{path} can be written by other users' + if path == os.path.dirname(path): + return None + path = os.path.dirname(path) + + +def machine_section(): + here = os.path.dirname(os.path.realpath(__file__)) + found = [c for c in (os.path.join(here, 'machine_file.py'), os.path.join(here, '..', '..', 'common', + 'machine_file.py')) if os.path.isfile(c)] + spec = importlib.util.spec_from_file_location('machine_file', found[0]) + mf = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mf) + + def shape(key): + return lambda v: None if re.fullmatch(SHAPES[key], v) else f'not a {key} ({SHAPES[key]})' + + def within(lo, hi): + return lambda v: None if lo <= v <= hi else f'{lo} to {hi}' + try: + section = mf.read_section('desk', MACHINE, ROOT_UID) + return mf.fields(section, {'pool': (str, mf.REQUIRED, shape('pool')), 'host': (str, mf.REQUIRED, shape('host')), + 'port': (int, 22, within(1, 65535)), 'user': (str, 'fleet-desk', shape('user')), + 'hostKey': (str, mf.REQUIRED, shape('hostKey')), 'slots': (int, 1, within(1, 8)), + 'reserveMb': (int, None, within(0, 1000000)), + 'netBase': (str, None, shape('netBase'))}, 'desk') + except mf.MachineFileError as e: + raise Refused(f'machine file: {e}') from None + + +def read_state(): + try: + with open(p(STATE), encoding='utf-8') as f: + state = json.load(f) + except FileNotFoundError: + return None + except (OSError, ValueError) as e: + raise Refused(f'{STATE} cannot be read ({e}): fix or remove it and run again') from None + state = state if isinstance(state, dict) else {} + state.setdefault('createdDenyFiles', []) + state.setdefault('addedDenyLines', []) # [deny file, user]: lines the kit added (also for users it did not make) + deny = '|'.join(map(re.escape, DENY)) + + def names(key, shape): + return isinstance(state.get(key), list) and all(isinstance(x, str) and re.fullmatch(shape, x) + for x in state[key]) + if not (state.get('version') == 1 and names('createdUsers', SHAPES['user']) + and names('createdDenyFiles', deny) + and isinstance(state['addedDenyLines'], list) + and all(isinstance(x, list) and len(x) == 2 and all(isinstance(y, str) for y in x) + and re.fullmatch(deny, x[0]) and re.fullmatch(SHAPES['user'], x[1]) + for x in state['addedDenyLines']) + and isinstance(state.get('restartPending', False), bool) + and state.get('fleetConfig') in ('release', 'project') + and (state.get('sha') is None or SHA.fullmatch(str(state['sha'])))): + raise Refused(f'{STATE} is not an install record of this kit (version 1): fix or remove it and run again') + return state + + +def write_file(path, text, mode=0o644): + """Atomic (a temp file in the same folder, renamed over the old one); unchanged text is not rewritten (a link, a + changed mode or an owner other than root is: the new file is root's).""" + real = p(path) + if same_file(real, text, mode): + return False + os.makedirs(os.path.dirname(real), mode=0o755, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(real), prefix='.' + os.path.basename(real) + '.') + try: + with os.fdopen(fd, 'w', encoding='utf-8') as f: + f.write(text) + f.flush() + os.fsync(f.fileno()) + os.chmod(tmp, mode) + os.replace(tmp, real) + except BaseException: + with contextlib.suppress(OSError): + os.unlink(tmp) + raise + return True + + +def save_state(state): + write_file(STATE, json.dumps(state, indent=2, sort_keys=True) + '\n', 0o600) + + +def ours(path): + try: + with open(p(path), encoding='utf-8') as f: + return f.readline() == HEADER + except (OSError, UnicodeDecodeError): + return False + + +def drop_ins(unit): + """Drop-ins that change one of the kit's units: `UNIT.d/`, and the dash-prefix forms (`fork-fleet-.service.d/`) + in every unit folder, and whatever systemd itself lists. The type-wide `service.d/` (distribution defaults) is left + out: it is not aimed at the kit's units.""" + stem, kind = unit.rsplit('.', 1) + names = [unit] + [stem[:i + 1] + '.' + kind for i, c in enumerate(stem) if c == '-'] + found = set() + for root in DROPIN_ROOTS: + for n in names: + with contextlib.suppress(OSError): + found.update(f'{root}/{n}.d/{f}' for f in os.listdir(p(f'{root}/{n}.d')) if f.endswith('.conf')) + r = run('systemctl', 'show', '-p', 'DropInPaths', '--value', unit, check=False) + if r.returncode == 0: + found.update(x for x in r.stdout.split() if os.path.basename(os.path.dirname(x)) != kind + '.d') + return sorted(found) + + +def stage(source, sha, fleet_config, pool): + """The release's files in /opt/fork-fleet/.kit-stage-*, checked; returns (folder, {path: sha256}).""" + for name in os.listdir(p(OPT)): # a run that stopped while staging + if name.startswith('.kit-stage-'): + shutil.rmtree(os.path.join(p(OPT), name), ignore_errors=True) + tmp = tempfile.mkdtemp(prefix='.kit-stage-', dir=p(OPT)) + pairs = [(dst, os.path.join(source, src)) for dst, src in FILES.items()] + pairs += [(f'units/{u}', os.path.join(source, src)) for u, src in UNITS.items()] + pairs += [(f'fork-fleet-walls/{n}', os.path.join(source, WALLS, n)) for n in sorted(os.listdir(os.path.join( + source, WALLS)))] + manifest = {} + for rel, src in pairs: + src = fleet_config if rel == 'fork-fleet.json' and fleet_config else src + dst = os.path.join(tmp, rel) + os.makedirs(os.path.dirname(dst), mode=0o755, exist_ok=True) + shutil.copyfile(src, dst) + os.chmod(dst, file_mode(rel)) + manifest[rel] = sha256(dst) + for d, _, _ in os.walk(tmp): + os.chmod(d, 0o755) + problems = [] + r = py(os.path.join(tmp, 'fork-fleet-walls.py'), 'os') + if r.returncode: + problems.append(f'fork-fleet-walls.py os: exit {r.returncode} {r.stderr.strip()[:200]}') + r = py(os.path.join(tmp, 'fork-fleet.py'), '--config', os.path.join(tmp, 'fork-fleet.json'), 'gate') + if 'no Runner.Worker above this hook' not in r.stdout + r.stderr: + problems.append(f'the gate does not load with this fleet config: {(r.stderr or r.stdout).strip()[:300]}') + try: + with open(os.path.join(tmp, 'fork-fleet.json'), encoding='utf-8') as f: + pools = json.load(f).get('pools') + if not isinstance(pools, list) or not all(isinstance(x, dict) for x in pools): + problems.append('the fleet config\'s "pools" is not a list of pools') + elif pool not in [x.get('name') for x in pools]: + problems.append(f'the fleet config has no pool {pool!r} (the machine file\'s desk.pool)') + except (OSError, ValueError, AttributeError) as e: + problems.append(f'the fleet config cannot be read: {e}') + for rel in ('fork-fleet-launch.sh', 'hook.sh'): + if run('bash', '-n', os.path.join(tmp, rel), check=False).returncode: + problems.append(f'{rel}: bash -n fails') + if py(os.path.join(tmp, 'fork-fleet-desk-agent.py'), '--help').returncode: + problems.append('fork-fleet-desk-agent.py --help fails') + if problems: + shutil.rmtree(tmp, ignore_errors=True) + raise Refused(f'the release failed its self-test ({OPT} is unchanged):\n ' + '\n '.join(problems)) + return tmp, manifest + + +def file_mode(rel): + """The mode of an installed file: programs 0755, the rest 0644.""" + executable = rel in ('fork-fleet-launch.sh', 'hook.sh', 'desk-install') or ( + rel.startswith('fork-fleet-walls/') and not rel.endswith('.json')) + return 0o755 if executable else 0o644 + + +def read_manifest(): + with contextlib.suppress(OSError): + with open(p(OPT + '/MANIFEST.sha256'), encoding='utf-8') as f: + return dict(reversed(line.rstrip('\n').split(' ', 1)) for line in f if ' ' in line) + return {} + + +def swap(tmp, manifest, sha, before_change): + """Each changed file renamed into /opt/fork-fleet (the agent stopped meanwhile, so no verb runs from half-old, + half-new files, and the refresh timer and service stopped too: refresh runs the launcher); files of the old + release that are gone, removed. Unchanged files (same text, mode and owner root) are left as they are; one whose + mode or owner was changed is replaced by the staged copy. True when anything changed.""" + def same(rel): + with contextlib.suppress(OSError): + st = os.lstat(p(f'{OPT}/{rel}')) + return (stat.S_ISREG(st.st_mode) and st.st_uid == ROOT_UID and stat.S_IMODE(st.st_mode) == file_mode(rel) + and sha256(p(f'{OPT}/{rel}')) == manifest[rel]) + changed = [rel for rel in sorted(manifest) if not same(rel)] + gone = sorted(set(read_manifest()) - set(manifest)) + if changed or gone: + before_change() + run('systemctl', 'stop', TIMER, REFRESH, AGENT, check=False) + for rel in changed: + os.makedirs(os.path.dirname(p(f'{OPT}/{rel}')), mode=0o755, exist_ok=True) + os.replace(os.path.join(tmp, rel), p(f'{OPT}/{rel}')) + for d in sorted({os.path.dirname(rel) for rel in manifest} - {''}): + os.chmod(p(f'{OPT}/{d}'), 0o755) + for rel in gone: + with contextlib.suppress(FileNotFoundError): + os.unlink(p(f'{OPT}/{rel}')) + write_file(OPT + '/MANIFEST.sha256', ''.join(f'{d} {r}\n' for r, d in sorted(manifest.items()))) + write_file(OPT + '/SOURCE', sha + '\n') + shutil.rmtree(tmp, ignore_errors=True) + return bool(changed or gone) + + +def lines_of(path): + with contextlib.suppress(FileNotFoundError), open(p(path), encoding='utf-8') as f: + return f.read().splitlines() + return [] + + +def make_users(users, state): + for u in users: + if user_exists(u): + continue + if u not in state['createdUsers']: # recorded first: a run stopped here still removes it on uninstall + state['createdUsers'].append(u) + save_state(state) + run('useradd', '--no-create-home', '--home-dir', '/nonexistent', '--user-group', '--shell', + '/usr/sbin/nologin', u) + say(f'made runner user {u}') + for u in users: + run('loginctl', 'disable-linger', u, check=False) + for f in DENY: + if os.path.exists(p(f.replace('.deny', '.allow'))): + continue + # A deny file that was not there opens cron or at to every other user (with neither file, only root may + # use at): recorded first, so uninstall removes the file again. + if not os.path.lexists(p(f)) and f not in state['createdDenyFiles']: + state['createdDenyFiles'].append(f) + save_state(state) + lines = lines_of(f) + if u not in lines: + if [f, u] not in state['addedDenyLines']: # recorded first: uninstall removes the line again + state['addedDenyLines'].append([f, u]) + save_state(state) + write_file(f, '\n'.join(lines + [u]) + '\n') + + +def settings(m, users): + known = (m['host'] if m['port'] == 22 else f'[{m["host"]}]:{m["port"]}') + ' ' + m['hostKey'] + files = {ETC + '/launch.conf': (f'RUNNER_USERS={",".join(users)}\n' + + (f'RESERVE_MB={m["reserveMb"]}\n' if m['reserveMb'] is not None else '') + + (f'NET_BASE={m["netBase"]}\n' if m['netBase'] else ''), 0o600), + ETC + '/desk.conf': (f'POOL={m["pool"]}\nHOST={m["host"]}\nPORT={m["port"]}\nUSER={m["user"]}\n', 0o600), + ETC + '/desk-known_hosts': (known + '\n', 0o644), + ETC + '/resolv.conf': ('nameserver 1.1.1.1\nnameserver 9.9.9.9\n', 0o644), + SYSCTL: ('net.ipv4.ip_forward = 1\n', 0o644)} + return {path: (HEADER + text, mode) for path, (text, mode) in files.items()} + + +def make_key(pool, before_change): + """The desk key, once: made under a temp name, then renamed (the public half first).""" + if os.path.exists(p(KEY)) and os.path.exists(p(KEY + '.pub')): + return False + before_change() + new = p(ETC + '/.desk-key.new') + for f in (new, new + '.pub'): + with contextlib.suppress(FileNotFoundError): + os.unlink(f) + if os.path.exists(p(KEY)): + raise Refused(f'{KEY} is there without {KEY}.pub: remove both and run again (a new key)') + run('ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-C', f'desk-{pool}', '-f', new) + os.replace(new + '.pub', p(KEY + '.pub')) + os.replace(new, p(KEY)) + return True + + +def lock(stack): + """One install or uninstall at a time: a lock on the /etc/fork-fleet folder itself (0700, so only root can open it + and hold it; nothing to clean up), held until the run ends. A second run is refused, not queued: it would sweep the + first one's staging folder.""" + try: + fd = os.open(p(ETC), os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC) + except OSError as e: + raise Refused(f'{ETC} cannot be locked ({e.strerror}): it must be a folder, not a link') from None + stack.callback(os.close, fd) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise Refused('another desk-install install or uninstall is running: wait for it and run again') from None + + +def install(a, stop_after=None, stack=None): + def done(step): + if step == stop_after: + raise SystemExit(f'stopped after {step} (test)') + + if os.geteuid() != ROOT_UID: + raise Refused('run this as root') + if a.fleet_config and a.release_fleet_config: + raise Refused('--fleet-config and --release-fleet-config: one or the other') + if not SHA.fullmatch(a.sha or ''): + raise Refused('--sha must be the full 40-character release commit') + missing = [s for s in [*FILES.values(), *UNITS.values(), WALLS] if not os.path.exists(os.path.join(a.source, s))] + if missing: + raise Refused(f'--source {a.source}: {missing[0]} is missing (point it at the release\'s top folder)') + if a.fleet_config and not os.path.isfile(a.fleet_config): + raise Refused(f'--fleet-config {a.fleet_config}: no such file') + if a.fleet_config and job_writable(a.fleet_config): # the file it names and every folder above both + raise Refused(f'--fleet-config {a.fleet_config}: {job_writable(a.fleet_config)}; copy it to a folder only root ' + 'can write (for example /root) and run again') + m = machine_section() + r = py(os.path.join(a.source, FILES['fork-fleet-walls.py']), 'os') + if r.returncode == 3: + raise Refused(f'this OS ({r.stdout.strip() or "unknown"}) is not supported yet: it has no walls file') + state = read_state() + if state is None: + there = [f for f in FOREIGN if os.path.lexists(p(f))] + if there: + raise Refused(f'{there[0]} is from an install this kit did not make: uninstall that first (README)') + state = {'version': 1, 'createdUsers': [], 'createdDenyFiles': [], 'addedDenyLines': [], + 'fleetConfig': 'release'} + for u in UNITS: + if os.path.lexists(p(f'{UNIT_DIR}/{u}')) and not ours(f'{UNIT_DIR}/{u}'): + raise Refused(f'{UNIT_DIR}/{u} was not written by this kit: remove it and run again') + if drop_ins(u): + raise Refused(f'{drop_ins(u)[0]} changes {u} and was not written by this kit: remove it and run again') + config = os.path.realpath(a.fleet_config) if a.fleet_config else None + if config is None and state['fleetConfig'] == 'project' and not a.release_fleet_config: + # the project's config from the last install (older records: the installed copy), checked again below + config = p(PROJECT) if os.path.isfile(p(PROJECT)) else p(OPT + '/fork-fleet.json') + if not os.path.isfile(config): + raise Refused(f'the install record says a project\'s fleet config, but neither {PROJECT} nor ' + f'{OPT}/fork-fleet.json is there: run again with --fleet-config FILE or ' + '--release-fleet-config') + for path, kind in ((OPT, stat.S_ISDIR), (OPT + '/template', stat.S_ISDIR), (ETC, stat.S_ISDIR), + (KEY, stat.S_ISREG), (KEY + '.pub', stat.S_ISREG)): + if not_root_owned(path, kind): # a hand setup: the root-run launcher must not sit where a user can write + raise Refused(not_root_owned(path, kind)) + users = [f'fork-runner-{i}' for i in range(1, m['slots'] + 1)] + for u in users: # a runner user made by hand must be as locked as one the kit makes + if user_exists(u) and user_problem(u): + raise Refused(f'runner user {user_problem(u)}: fix it and run again') + done('check') + for d, mode in ((OPT, 0o755), (OPT + '/template', 0o755), (ETC, 0o700)): + os.makedirs(p(d), mode=mode, exist_ok=True) + os.chmod(p(d), mode) + lock(stack if stack is not None else contextlib.ExitStack()) + save_state(state) + make_users(users, state) + done('users') + tmp, manifest = stage(a.source, a.sha, config, m['pool']) + done('stage') + + def pending(): + """Recorded before anything the agent reads changes: the agent reads its settings only when it starts, so a + run stopped after a write (and before the restart) still restarts it on the next run.""" + if not state.get('restartPending'): + state['restartPending'] = True + save_state(state) + + def put(path, text, mode=0o644): + if not same_text(path, text, mode): + pending() + write_file(path, text, mode) + # the fleet config choice before the swap: a run stopped after it keeps what was asked for. The restart mark goes + # first when the swap will change the installed config, so `check` fails until a run ends the swap. + if manifest['fork-fleet.json'] != read_manifest().get('fork-fleet.json'): + pending() + if config: + with open(os.path.join(tmp, 'fork-fleet.json'), encoding='utf-8') as f: + write_file(PROJECT, f.read()) + state['fleetConfig'] = 'project' if config else 'release' + save_state(state) + if not config: + remove(PROJECT) + swap(tmp, manifest, a.sha, pending) + state['sha'] = a.sha + save_state(state) + done('files') + for path, (text, mode) in settings(m, users).items(): + put(path, text, mode) + run('sysctl', '-q', '-p', p(SYSCTL)) + make_key(m['pool'], pending) + os.chmod(p(KEY), 0o600) # `check` wants it so; an older key made with another mode is set right + done('settings') + for u in UNITS: + with open(p(f'{OPT}/units/{u}'), encoding='utf-8') as f: + put(f'{UNIT_DIR}/{u}', HEADER + f.read()) + r = py(p(OPT + '/fork-fleet-desk-agent.py'), 'check') + if r.returncode: + raise Refused(f'the desk agent refuses its settings (not started): {r.stderr.strip()[:300]}') + run('systemctl', 'daemon-reload') + run('systemctl', 'enable', '--now', TIMER) + if not os.path.exists(p(OPT + '/template/runner-template/.fork-fleet-version')): + run(p(OPT + '/fork-fleet-launch.sh'), 'prune', check=False) # makes the launch lock refresh opens + run('systemctl', 'start', '--no-block', REFRESH) # the first runner template, in the background + run('systemctl', 'enable', AGENT) + # restart only after a change (this run's or a stopped run's): a restart ends the agent's whole control group, a + # `start` it is running with it (the always-on host would see that job fail). Unchanged: start is a no-op. + run('systemctl', 'restart' if state.get('restartPending') else 'start', AGENT) + if state.pop('restartPending', None) is not None: + save_state(state) + done('units') + with open(p(KEY + '.pub'), encoding='utf-8') as f: + pub = f.read().strip() + say(f'installed release {a.sha} for pool {m["pool"]} ({len(users)} slot(s)); the agent dials {m["host"]}') + say(f'its public key, for the always-on host (fork-fleet-desk-key.py add {m["pool"]}):\n{pub}') + say('then: desk-install check (every line PASS)') + return 0 + + +def same_text(path, text, mode): + return same_file(p(path), text, mode) + + +def check(): + rows = [] + + def row(ok, name, what): + rows.append(ok) + print(f'{"PASS" if ok else "FAIL"} {name} {what}', flush=True) + problems = installed_problems() + for x in problems: + row(False, 'kit', x) + if not problems: + row(True, 'kit', f'the installed files are release {read_state()["sha"]}\'s') + walls = py(p(OPT + '/fork-fleet-walls.py'), 'describe') + lines = [x for x in walls.stdout.splitlines() if x.startswith(('PASS ', 'FAIL '))] + for line in lines: + print(line) + rows.append(line.startswith('PASS')) + if walls.returncode and not any(x.startswith('FAIL') for x in lines): + row(False, 'walls', f'exit {walls.returncode}: {walls.stderr.strip()[:200]}') + r = py(p(OPT + '/fork-fleet-desk-agent.py'), 'check') + row(r.returncode == 0, 'agent-settings', (r.stdout or r.stderr).strip()[:200]) + for unit in (AGENT, TIMER): + on = run('systemctl', 'is-active', '--quiet', unit, check=False).returncode == 0 + row(on and run('systemctl', 'is-enabled', '--quiet', unit, check=False).returncode == 0, 'unit', + f'{unit} enabled and active') + row(os.path.exists(p(OPT + '/template/runner-template/.fork-fleet-version')), 'template', + 'the runner template is there (the refresh timer makes it)') + try: + m = machine_section() + users = [f'fork-runner-{i}' for i in range(1, m['slots'] + 1)] + row(True, 'machine-file', f'desk section: pool {m["pool"]}, {m["slots"]} slot(s), always-on host {m["host"]}') + stale = [path for path, (text, mode) in settings(m, users).items() if not same_text(path, text, mode)] + row(not stale, 'settings', f'{stale[0]} differs from the machine file: install again' if stale else + 'the written settings match the machine file') + except Refused as e: + users = [] + row(False, 'machine-file', str(e)[:300]) + for u in users: + why = f'{u} does not exist' if not user_exists(u) else user_problem(u) + row(not why, 'runner-user', why or f'{u} exists: no login shell, no password, own group only, no home, ' + 'not in cron.allow/at.allow') + with contextlib.suppress(Refused): # a smaller `slots` leaves the kit's extra users: named, not lost + for u in (read_state() or {}).get('createdUsers', []) if users else []: + if u not in users and user_exists(u): + row(True, 'runner-user', f'{u} is no longer a slot (no login, unused; uninstall removes it)') + try: + with open(p('/proc/sys/net/ipv4/ip_forward'), encoding='ascii') as f: + forward = f.read().strip() == '1' + except OSError: + forward = False + row(forward, 'ip-forward', 'IPv4 forwarding is on (the runners\' namespaces need it)') + fails = rows.count(False) + print('ALL PASS' if not fails else f'{fails} FAIL') + return 0 if not fails else 1 + + +def installed_problems(): + """What differs from the install record: /opt/fork-fleet against its manifest, SOURCE, the units.""" + try: + problems, state = [], read_state() or {} + except Refused as e: + return [str(e)] + manifest = read_manifest() + if not manifest or not state.get('sha'): + return [f'{OPT}/MANIFEST.sha256 or {STATE} is missing: not installed (or a first install stopped)'] + for rel, digest in manifest.items(): + try: + st = os.lstat(p(f'{OPT}/{rel}')) + ok = stat.S_ISREG(st.st_mode) and sha256(p(f'{OPT}/{rel}')) == digest and st.st_uid == ROOT_UID + mode_ok = stat.S_IMODE(st.st_mode) == file_mode(rel) + except OSError: + ok = mode_ok = False + if not ok: + problems.append(f'{OPT}/{rel}: differs from what was installed') + elif not mode_ok: + problems.append(f'{OPT}/{rel}: mode {stat.S_IMODE(st.st_mode):o}, not {file_mode(rel):o}: install again') + folders = {OPT: 0o755, OPT + '/template': 0o755, ETC: 0o700, KEY: 0o600} + folders.update({f'{OPT}/{os.path.dirname(rel)}': 0o755 for rel in manifest if os.path.dirname(rel)}) + for path, mode in sorted(folders.items()): + try: + st = os.lstat(p(path)) + except OSError: + continue # a missing key or folder is named by its own check (agent settings, template) + if stat.S_ISLNK(st.st_mode) or st.st_uid != ROOT_UID: + problems.append(f'{path}: a link or not owned by root: remove the link (or chown root), then install again') + elif stat.S_IMODE(st.st_mode) != mode: + problems.append(f'{path}: mode {stat.S_IMODE(st.st_mode):o}, not {mode:o}: install again') + if state.get('fleetConfig') == 'project': + try: + ok = sha256(p(PROJECT)) == manifest.get('fork-fleet.json') + except OSError: + ok = False + if not ok: + problems.append(f'{PROJECT} (the project\'s fleet config) is not the installed one: install again') + try: + with open(p(OPT + '/SOURCE'), encoding='utf-8') as f: + source = f.read().strip() + except OSError: + source = None + if source != state['sha']: + problems.append(f'{OPT}/SOURCE names {source}, the record {state["sha"]}: install again') + if state.get('restartPending'): + problems.append('a stopped install changed what the agent reads but did not restart it: install again') + for u in UNITS: + try: + with open(p(f'{OPT}/units/{u}'), encoding='utf-8') as f: + want = HEADER + f.read() + except OSError: + want = None + if not ours(f'{UNIT_DIR}/{u}'): + problems.append(f'{UNIT_DIR}/{u}: missing or not the kit\'s') + elif want is None or not same_text(f'{UNIT_DIR}/{u}', want, 0o644): + problems.append(f'{UNIT_DIR}/{u}: not the unit of release {state["sha"]} ({OPT}/units/{u}), or its mode or ' + 'owner changed: install again') + problems += [f'{d}: a drop-in the kit did not write changes {u}' for u in UNITS for d in drop_ins(u)] + return problems + + +def running(): + """The launcher's runner units still running (or starting or stopping).""" + r = run('systemctl', 'list-units', '--plain', '--no-legend', '--type=service', + '--state=active,activating,deactivating,reloading', 'fork-fleet-*', check=False) + return [x.split()[0] for x in r.stdout.splitlines() if x.split() and RUNNER_UNIT.fullmatch(x.split()[0])] + + +def remove(path): + """One path the kit made (a folder with all in it); absent is fine.""" + real = p(path) + if os.path.isdir(real) and not os.path.islink(real): + shutil.rmtree(real) + else: + with contextlib.suppress(FileNotFoundError): + os.unlink(real) + + +def rmdir_left(path): + """The folder if empty; otherwise what is left in it (not the kit's, so kept).""" + try: + os.rmdir(p(path)) + except FileNotFoundError: + pass + except OSError: + return [f'{path}/{n}' for n in sorted(os.listdir(p(path)))] + return [] + + +def uninstall(stack=None): + """Removes only what the kit made: its units, the files of its manifest, its settings, the runner copies and + template its programs made, the users and deny files it made. Every runner is stopped first; while one still runs, + the network table, users and record stay (run it again). The record goes last, so a stopped uninstall runs again.""" + if os.geteuid() != ROOT_UID: + raise Refused('run this as root') + state = read_state() + if state is None: + say(f'no {STATE}: nothing here was installed by this kit (nothing changed)') + return 0 + lock(stack if stack is not None else contextlib.ExitStack()) # the record is in /etc/fork-fleet: it is there + for unit in (AGENT, TIMER, REFRESH): + run('systemctl', 'disable', '--now', unit, check=False) + launcher = p(OPT + '/fork-fleet-launch.sh') + if os.path.exists(launcher): + for name in run(launcher, 'active', check=False).stdout.split(): + run(launcher, 'stop', name, check=False) + for unit in running(): # what the launcher could not stop (or no launcher is left to ask) + run('systemctl', 'stop', unit, check=False) + left = running() + if left: + raise Refused(f'runner {left[0]} is still running: the network table, users and record stay (its namespace ' + 'keeps routing through them). Stop it (systemctl stop) and run uninstall again') + if os.path.exists(launcher): + run(launcher, 'prune', check=False) + for ns in run('ip', 'netns', 'list', check=False).stdout.splitlines(): + if ns.split() and ns.split()[0].startswith('fork-fleet-'): # every runner's, made by the kit or by hand + run('ip', 'netns', 'delete', ns.split()[0], check=False) + for line in run('ip', '-o', 'link', 'show', check=False).stdout.splitlines(): + link = re.match(r'[0-9]+: (ffh-[0-9]+)[@:]', line) + if link: + run('ip', 'link', 'delete', link.group(1), check=False) + run('nft', 'delete', 'table', 'inet', 'fork_fleet', check=False) + for u in UNITS: + if ours(f'{UNIT_DIR}/{u}'): + os.unlink(p(f'{UNIT_DIR}/{u}')) + run('systemctl', 'daemon-reload', check=False) + if ours(SYSCTL): + os.unlink(p(SYSCTL)) + for rel in [*read_manifest(), 'MANIFEST.sha256', 'SOURCE', 'template', + *(n for n in os.listdir(p(OPT)) if n.startswith('.kit-stage-'))] if os.path.isdir(p(OPT)) else []: + remove(f'{OPT}/{rel}') + remove(WORK + '/runners') + for u in state['createdUsers']: + if user_exists(u): + run('userdel', u) + for f in DENY: + lines = lines_of(f) + # the lines the kit added (for every runner user, made by it or not; older records: the users it made) + added = set(state['createdUsers']) | {u for g, u in state['addedDenyLines'] if g == f} + if any(x in added for x in lines): + write_file(f, ''.join(x + '\n' for x in lines if x not in added)) + if f in state['createdDenyFiles'] and not [x for x in lines_of(f) if x.strip()]: + remove(f) + for name in SETTINGS: + remove(f'{ETC}/{name}') + for name in os.listdir(p(ETC)) if os.path.isdir(p(ETC)) else []: + if name.startswith(('.desk-key.new', '.desk-kit.json.', *(f'.{n}.' for n in SETTINGS))): # a stopped write or keygen + remove(f'{ETC}/{name}') + remove(STATE) # the record last + left = [] + for d in ([f'{OPT}/{x}' for x in ('units', 'fork-fleet-walls')] + [OPT, WORK, ETC]): + left += rmdir_left(d) + left += [f'{d} (a drop-in, not the kit\'s)' for u in UNITS for d in drop_ins(u)] + for x in left: + say(f'left {x}: the kit did not make it') + say('uninstalled; on the always-on host remove its key: fork-fleet-desk-key.py remove POOL') + return 0 + + +def main(argv, stop_after=None): + ap = argparse.ArgumentParser(prog='desk-install', description=__doc__.split('\n\n')[0]) + sub = ap.add_subparsers(dest='cmd', required=True) + i = sub.add_parser('install') + i.add_argument('--source', required=True) + i.add_argument('--sha', required=True) + i.add_argument('--fleet-config') + i.add_argument('--release-fleet-config', action='store_true', + help='back to the release\'s own fleet config after a project\'s was used') + for name in ('check', 'uninstall'): + sub.add_parser(name) + a = ap.parse_args(argv) + os.umask(0o022) + try: + with contextlib.ExitStack() as stack: + if a.cmd == 'install': + return install(a, stop_after, stack) + return check() if a.cmd == 'check' else uninstall(stack) + except Refused as e: + print(f'desk-install: {e}', file=sys.stderr) + return 1 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/fork-kit/desk/tests/install.test.py b/fork-kit/desk/tests/install.test.py new file mode 100644 index 0000000000000..a5b813f994482 --- /dev/null +++ b/fork-kit/desk/tests/install.test.py @@ -0,0 +1,973 @@ +"""desk-install on a fake host in a temporary folder: every fixed path under PREFIX, the current user as root, and +every system command (useradd, systemctl, ssh-keygen, ...) a fake that records its arguments. A fake release folder +holds stubs of the fleet files. No root, no systemd, no network, no real user. + + python3 -I -B fork-kit/desk/tests/install.test.py +""" +import contextlib, fcntl, importlib.machinery, importlib.util, io, json, os, shutil, stat, subprocess, sys, tempfile +import types, unittest, unittest.mock + +KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +REPO = os.path.dirname(os.path.dirname(KIT)) +SHA1, SHA2 = 'a' * 40, 'b' * 40 +HOST_KEY = 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI' + 'A' * 43 +MACHINE = {'version': 1, 'desk': {'pool': 'desktop', 'host': 'fleet-host.example', 'hostKey': HOST_KEY, 'slots': 2}} + +# One fake for every system command: argv[1] is the command's name. Its state lives in files beside it. +FAKE = r''' +import os, sys +here = os.path.dirname(os.path.abspath(__file__)) +name, args = sys.argv[1], sys.argv[2:] +with open(os.path.join(here, 'calls.log'), 'a') as f: + f.write(' '.join([name, *args]) + '\n') +def lines(n): + try: + return open(os.path.join(here, n)).read().split() + except FileNotFoundError: + return [] +def put(n, items): + open(os.path.join(here, n), 'w').write('\n'.join(items) + '\n') +fail = lines('fail') +if name in fail: + put('fail', [x for x in fail if x != name]) # fails once + sys.exit(9) +if name in lines('fail-after'): # does its work, then dies (a run stopped right after it) + put('fail-after', [x for x in lines('fail-after') if x != name]) + put('users', lines('users') + [args[-1]]) + sys.exit(9) +def one(n, default): # a per-user setting: the file N-USER + try: + return open(os.path.join(here, n + '-' + args[-1])).read().strip() + except FileNotFoundError: + return default +if name == 'id': # its groups: the file groups-USER (its own group only) + if args[-1] not in lines('users'): + sys.exit(1) + print({'-u': one('uid', '1001'), '-g': '1001', '-G': one('groups', '1001')}.get(args[0], '')) +if name == 'getent': # login shell: shell-USER (nologin); password field: pw-USER (!); uid-USER; home-USER + u = args[-1] + if u not in lines('users'): + sys.exit(2) + if args[0] == 'passwd': + print(f'{u}:x:' + one('uid', '1001') + ':1001::' + one('home', '/nonexistent') + ':' + + one('shell', '/usr/sbin/nologin')) + else: + print(f'{u}:' + one('pw', '!') + ':19000:0:99999:7:::') +if name == 'useradd': + put('users', lines('users') + [args[-1]]) +if name == 'userdel': + put('users', [u for u in lines('users') if u != args[-1]]) +if name == 'ssh-keygen': + key = args[args.index('-f') + 1] + open(key, 'w').write('PRIVATE\n'); os.chmod(key, 0o600) + open(key + '.pub', 'w').write('ssh-ed25519 AAAAfake desk-x\n') +if name == 'systemctl' and args[0] in ('is-active', 'is-enabled'): + sys.exit(1 if args[-1] in lines('down') else 0) +if name == 'systemctl' and args[0] == 'list-units': # runner units: the file `units`; `stuck` ones never stop + print(''.join(f'{u} loaded active running runner\n' for u in lines('units')), end='') +if name == 'systemctl' and args[0] == 'stop': + put('units', [u for u in lines('units') if u not in args[1:] or u in lines('stuck')]) +if name == 'ip' and args[:2] == ['netns', 'list']: + print(''.join(f'{n} (id: 0)\n' for n in lines('netns')), end='') +if name == 'ip' and args[:2] == ['netns', 'delete']: + put('netns', [n for n in lines('netns') if n != args[2]]) + put('links', [n for n in lines('links') if n != 'ffh-' + args[2].rsplit('-', 1)[-1]]) +if name == 'ip' and args[:2] == ['-o', 'link']: + print(''.join(f'{i}: {n}@if2: mtu 1500\n' for i, n in enumerate(lines('links'), 5)), end='') +if name == 'ip' and args[:2] == ['link', 'delete']: + put('links', [n for n in lines('links') if n != args[2]]) +''' + +STUBS = { + '.github/scripts/fork-fleet-launch.sh': '#!/bin/bash\necho "launch $*" >> {log}\n' + '[ "$1" = active ] && cat {here}/active 2>/dev/null\n' + 'if [ "$1" = stop ]; then grep -vx "$2" {here}/active > {here}/a.new;' + ' mv {here}/a.new {here}/active; fi\nexit 0\n', + '.github/scripts/fork-fleet-walls.py': '#!/usr/bin/python3 -I\nimport sys, os\n' + 'v = sys.argv[1]\nif v == "os":\n' + ' print("linux"); sys.exit(int(open("{here}/os-exit").read()))\n' + 'bad = os.path.exists("{here}/wall-fails")\n' + 'print("PASS os linux")\nprint(("FAIL vsock closed: open" if bad else' + ' "PASS vsock closed"))\nsys.exit(1 if bad else 0)\n', + '.github/scripts/fork-fleet.py': 'import sys\nprint("::error::no Runner.Worker above this hook")\nsys.exit(1)\n', + '.github/scripts/fork-fleet-desk-agent.py': 'import os, sys\nif sys.argv[-1] == "check" and ' + 'os.path.exists("{here}/agent-bad"):\n' + ' print("bad settings", file=sys.stderr); sys.exit(2)\n' + 'print("settings ok")\n', + '.github/scripts/fork-fleet-desk-wire.py': '# wire\n', + '.github/scripts/fork-fleet-hook.sh': '#!/bin/bash\nexit 0\n', + '.github/scripts/fork-fleet-job-paths.txt': '/usr\n', + '.github/fork-fleet.json': json.dumps({'pools': [{'name': 'desktop'}, {'name': 'vm'}]}), + '.github/scripts/fork-fleet-walls/linux': '#!/bin/sh\n', + '.github/scripts/fork-fleet-walls/linux.fixture.json': '{}\n', + '.github/scripts/fork-fleet-walls/wsl2': '#!/bin/sh\n', + '.github/scripts/fork-fleet-walls/wsl2.fixture.json': '{}\n', + '.github/scripts/fork-fleet-desk-agent.service': '# agent unit\n[Service]\n', + '.github/scripts/fork-fleet-refresh.service': '# refresh unit\n[Service]\n', + '.github/scripts/fork-fleet-refresh.timer': '# refresh timer\n[Timer]\n', +} + + +def slurp(path, mode='r'): + with open(path, mode) as f: + return f.read() + + +def load(): + path = os.path.join(KIT, 'sbin', 'desk-install') + loader = importlib.machinery.SourceFileLoader('desk_install', path) + mod = importlib.util.module_from_spec(importlib.util.spec_from_loader('desk_install', loader)) + loader.exec_module(mod) + return mod + + +class Desk(unittest.TestCase): + def setUp(self): + self.umask = os.umask(0o022) + self.tmp = os.path.realpath(tempfile.mkdtemp(prefix='desk-install-test-')) + self.addCleanup(self.close) + self.root, self.fakes = os.path.join(self.tmp, 'root'), os.path.join(self.tmp, 'fakes') + os.makedirs(self.fakes) + with open(os.path.join(self.fakes, 'fake.py'), 'w') as f: + f.write(FAKE) + self.write(os.path.join(self.fakes, 'os-exit'), '0') + self.source = self.release() + self.m = load() + self.m.PREFIX, self.m.ROOT_UID = self.root, os.geteuid() + self.m.MACHINE = os.path.join(self.tmp, 'machine', 'machine.json') + fake = os.path.join(self.fakes, 'fake.py') + self.m.CMD = {k: [sys.executable, '-I', fake, k] for k in self.m.CMD} + self.m.CMD.update(python3=[sys.executable, '-I', '-B'], bash=['bash']) + self.machine(MACHINE) + self.write(self.path('/proc/sys/net/ipv4/ip_forward'), '1\n') + + def close(self): + os.umask(self.umask) + shutil.rmtree(self.tmp, ignore_errors=True) + + def write(self, path, text, mode=0o644): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as f: + f.write(text) + os.chmod(path, mode) + + def release(self, name='release', extra=None): + top = os.path.join(self.tmp, name) + for rel, text in {**STUBS, **(extra or {})}.items(): + if text is not None: + self.write(os.path.join(top, rel), text.replace('{log}', os.path.join(self.fakes, 'calls.log')) + .replace('{here}', self.fakes), 0o755) + for rel in ('fork-kit/desk/sbin/desk-install', 'fork-kit/desk/README.md', 'fork-kit/common/machine_file.py'): + self.write(os.path.join(top, rel), slurp(os.path.join(REPO, rel)), 0o755) + return top + + def machine(self, data): + path = self.m.MACHINE + os.makedirs(os.path.dirname(path), mode=0o755, exist_ok=True) + self.write(path, json.dumps(data) if not isinstance(data, str) else data) + + def path(self, p): + return self.root + p + + def calls(self): + with contextlib.suppress(FileNotFoundError): + with open(os.path.join(self.fakes, 'calls.log')) as f: + return f.read().splitlines() + return [] + + def clear_calls(self): + with contextlib.suppress(FileNotFoundError): + os.unlink(os.path.join(self.fakes, 'calls.log')) + + def run_main(self, *argv, stop_after=None): + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + try: + code = self.m.main(list(argv), stop_after=stop_after) + except SystemExit as e: + code = str(e) + return code, out.getvalue(), err.getvalue() + + def install(self, *extra, sha=SHA1, stop_after=None, source=None): + return self.run_main('install', '--source', source or self.source, '--sha', sha, *extra, + stop_after=stop_after) + + def read(self, p): + with open(self.path(p)) as f: + return f.read() + + def tree(self): + """Every path under the fake root with its kind, mode and contents (no times).""" + out = {} + for dirpath, dirnames, filenames in os.walk(self.root): + for name in dirnames + filenames: + full = os.path.join(dirpath, name) + st = os.lstat(full) + body = slurp(full, 'rb') if stat.S_ISREG(st.st_mode) else None + out[full[len(self.root):]] = (stat.S_IFMT(st.st_mode), st.st_mode & 0o777, body) + return out + + def template(self): + self.write(self.path('/opt/fork-fleet/template/runner-template/.fork-fleet-version'), '2.330.0\n') + + # ------------------------------------------------------------------------------------------------- install + + def test_fresh_install_files_settings_users_key_units(self): + code, out, err = self.install() + self.assertEqual(code, 0, err) + self.assertEqual(self.read('/opt/fork-fleet/SOURCE'), SHA1 + '\n') + for rel in ('fork-fleet-launch.sh', 'hook.sh', 'fork-fleet.py', 'fork-fleet.json', 'fork-fleet-walls.py', + 'fork-fleet-walls/linux', 'fork-fleet-walls/wsl2.fixture.json', 'desk-install', 'machine_file.py'): + self.assertTrue(os.path.isfile(self.path(f'/opt/fork-fleet/{rel}')), rel) + self.assertEqual(stat.S_IMODE(os.stat(self.path('/opt/fork-fleet/hook.sh')).st_mode), 0o755) + self.assertEqual(stat.S_IMODE(os.stat(self.path('/opt/fork-fleet/fork-fleet.json')).st_mode), 0o644) + self.assertFalse([n for n in os.listdir(self.path('/opt/fork-fleet')) if n.startswith('.kit-stage')]) + launch = self.read('/etc/fork-fleet/launch.conf') + self.assertTrue(launch.startswith(self.m.HEADER)) + self.assertIn('RUNNER_USERS=fork-runner-1,fork-runner-2\n', launch) + self.assertNotIn('RESERVE_MB', launch) + self.assertIn('POOL=desktop\nHOST=fleet-host.example\nPORT=22\nUSER=fleet-desk\n', + self.read('/etc/fork-fleet/desk.conf')) + self.assertTrue(self.read('/etc/fork-fleet/desk-known_hosts').endswith(f'fleet-host.example {HOST_KEY}\n')) + for f, mode in (('launch.conf', 0o600), ('desk.conf', 0o600), ('desk-key', 0o600), ('desk-kit.json', 0o600)): + self.assertEqual(stat.S_IMODE(os.stat(self.path(f'/etc/fork-fleet/{f}')).st_mode), mode, f) + self.assertEqual(stat.S_IMODE(os.stat(self.path('/etc/fork-fleet')).st_mode), 0o700) + self.assertIn('ssh-ed25519 AAAAfake desk-x', out) + self.assertEqual(self.read('/etc/sysctl.d/60-fork-fleet.conf'), self.m.HEADER + 'net.ipv4.ip_forward = 1\n') + for u in self.m.UNITS: + self.assertTrue(self.read(f'/etc/systemd/system/{u}').startswith(self.m.HEADER), u) + calls = self.calls() + self.assertEqual([c for c in calls if c.startswith('useradd')], [ + f'useradd --no-create-home --home-dir /nonexistent --user-group --shell /usr/sbin/nologin fork-runner-{i}' + for i in (1, 2)]) + self.assertEqual(self.read('/etc/cron.deny'), 'fork-runner-1\nfork-runner-2\n') + self.assertEqual(json.loads(self.read('/etc/fork-fleet/desk-kit.json'))['createdUsers'], + ['fork-runner-1', 'fork-runner-2']) + order = [c for c in calls if c.startswith(('systemctl', 'launch')) and not c.startswith('systemctl show')] + self.assertEqual(order, ['systemctl stop t3code-fleet-refresh.timer t3code-fleet-refresh.service ' + 'fork-fleet-desk-agent.service', 'systemctl daemon-reload', + 'systemctl enable --now t3code-fleet-refresh.timer', 'launch prune', + 'systemctl start --no-block t3code-fleet-refresh.service', + 'systemctl enable fork-fleet-desk-agent.service', + 'systemctl restart fork-fleet-desk-agent.service']) + self.template() + code, out, _ = self.run_main('check') + self.assertEqual(code, 0, out) + self.assertTrue(out.endswith('ALL PASS\n'), out) + self.assertIn('PASS vsock closed', out) + + def test_runs_again_and_changes_nothing(self): + self.assertEqual(self.install()[0], 0) + self.template() + before = self.tree() + self.clear_calls() + code, _, err = self.install() + self.assertEqual(code, 0, err) + self.assertEqual(self.tree(), before) + calls = self.calls() + self.assertFalse([c for c in calls if c.startswith(('useradd', 'ssh-keygen', 'systemctl stop', + 'systemctl restart'))], calls) + self.assertIn('systemctl start fork-fleet-desk-agent.service', calls) # a no-op for a running agent + self.assertNotIn('systemctl start --no-block t3code-fleet-refresh.service', calls) # the template is there + + def test_a_changed_setting_restarts_the_agent(self): + self.assertEqual(self.install()[0], 0) + self.machine({'version': 1, 'desk': {**MACHINE['desk'], 'reserveMb': 2048}}) + self.clear_calls() + self.assertEqual(self.install()[0], 0) + self.assertIn('systemctl restart fork-fleet-desk-agent.service', self.calls()) + + def test_a_change_written_by_a_killed_run_still_restarts_the_agent(self): + self.assertEqual(self.install()[0], 0) + self.template() + self.machine({'version': 1, 'desk': {**MACHINE['desk'], 'host': 'new-host.example'}}) + self.assertIn('stopped after', str(self.install(stop_after='settings')[0])) + code, out, _ = self.run_main('check') # desk.conf is new, the running agent still has the old one + self.assertEqual(code, 1) + self.assertIn('did not restart it', out) + self.clear_calls() + self.assertEqual(self.install()[0], 0) + self.assertIn('systemctl restart fork-fleet-desk-agent.service', self.calls()) + self.assertNotIn('restartPending', self.read('/etc/fork-fleet/desk-kit.json')) + self.clear_calls() + self.assertEqual(self.install()[0], 0) # and once restarted, not again + self.assertNotIn('systemctl restart fork-fleet-desk-agent.service', self.calls()) + + def test_a_refused_agent_check_keeps_the_restart_for_the_next_run(self): + self.assertEqual(self.install()[0], 0) + self.machine({'version': 1, 'desk': {**MACHINE['desk'], 'port': 2222}}) + self.write(os.path.join(self.fakes, 'agent-bad'), '') + self.assertEqual(self.install()[0], 1) + os.unlink(os.path.join(self.fakes, 'agent-bad')) + self.clear_calls() + self.assertEqual(self.install()[0], 0) + self.assertIn('systemctl restart fork-fleet-desk-agent.service', self.calls()) + + def test_stopped_at_any_step_the_next_run_ends_it(self): + self.assertEqual(self.install()[0], 0) + want = self.tree() + for step in ('check', 'users', 'stage', 'files', 'settings', 'units'): + with self.subTest(step=step): + shutil.rmtree(self.root) + with contextlib.suppress(FileNotFoundError): + os.unlink(os.path.join(self.fakes, 'users')) + self.write(self.path('/proc/sys/net/ipv4/ip_forward'), '1\n') + code, _, _ = self.install(stop_after=step) + self.assertIn('stopped after', str(code)) + if step in ('stage', 'files', 'settings'): + self.assertNotEqual(self.run_main('check')[0], 0) + code, _, err = self.install() + self.assertEqual(code, 0, err) + self.assertEqual(self.tree(), want) + + def test_upgrade_replaces_changed_files_and_removes_gone_ones(self): + self.assertEqual(self.install()[0], 0) + launcher = os.stat(self.path('/opt/fork-fleet/fork-fleet-launch.sh')).st_ino + new = self.release('release2', {'.github/scripts/fork-fleet-walls/wsl2': None, + '.github/scripts/fork-fleet-walls/wsl2.fixture.json': None, + '.github/scripts/fork-fleet-hook.sh': '#!/bin/bash\nexit 0 # new\n'}) + self.clear_calls() + code, _, err = self.install(sha=SHA2, source=new) + self.assertEqual(code, 0, err) + self.assertEqual(self.read('/opt/fork-fleet/SOURCE'), SHA2 + '\n') + self.assertFalse(os.path.exists(self.path('/opt/fork-fleet/fork-fleet-walls/wsl2'))) + self.assertIn('# new', self.read('/opt/fork-fleet/hook.sh')) + self.assertEqual(os.stat(self.path('/opt/fork-fleet/fork-fleet-launch.sh')).st_ino, launcher) # unchanged + calls = self.calls() + stop = 'systemctl stop t3code-fleet-refresh.timer t3code-fleet-refresh.service fork-fleet-desk-agent.service' + self.assertLess(calls.index(stop), calls.index('systemctl restart fork-fleet-desk-agent.service')) + self.assertEqual(json.loads(self.read('/etc/fork-fleet/desk-kit.json'))['sha'], SHA2) + + def test_project_fleet_config_is_used_and_kept(self): + project = os.path.join(self.tmp, 'project.json') + self.write(project, json.dumps({'pools': [{'name': 'desktop'}], 'project': True})) + code, _, err = self.install('--fleet-config', project) + self.assertEqual(code, 0, err) + self.assertIn('"project": true', self.read('/opt/fork-fleet/fork-fleet.json')) + code, _, err = self.install(sha=SHA2) # no --fleet-config: the project's stays + self.assertEqual(code, 0, err) + self.assertIn('"project": true', self.read('/opt/fork-fleet/fork-fleet.json')) + code, _, err = self.install('--release-fleet-config', sha=SHA2) # and back to the release's own + self.assertEqual(code, 0, err) + self.assertNotIn('"project": true', self.read('/opt/fork-fleet/fork-fleet.json')) + self.assertEqual(json.loads(self.read('/etc/fork-fleet/desk-kit.json'))['fleetConfig'], 'release') + code, _, err = self.install('--release-fleet-config', '--fleet-config', project) + self.assertEqual(code, 1) + self.assertIn('one or the other', err) + + def test_the_fleet_config_choice_outlives_a_run_killed_after_the_swap(self): + project = os.path.join(self.tmp, 'project.json') + self.write(project, json.dumps({'pools': [{'name': 'desktop'}], 'project': True})) + newer = self.release('release3', {'.github/fork-fleet.json': json.dumps({'pools': [{'name': 'desktop'}], + 'release3': True})}) + swap = self.m.swap + + def killed(*args): + swap(*args) + raise SystemExit('killed right after the swap') + for first, then, want, other in ((['--fleet-config', project], ['--release-fleet-config'], '"release3"', + '"project"'), + ([], ['--fleet-config', project], '"project"', '"release3"')): + with self.subTest(then[0]): + shutil.rmtree(self.root) + self.write(self.path('/proc/sys/net/ipv4/ip_forward'), '1\n') + self.assertEqual(self.install(*first)[0], 0) + self.m.swap = killed + try: + self.assertIn('killed', str(self.install(*then, sha=SHA2)[0])) + finally: + self.m.swap = swap + code, _, err = self.install(sha=SHA1, source=newer) # a plain later install keeps the last choice + self.assertEqual(code, 0, err) + self.assertIn(want, self.read('/opt/fork-fleet/fork-fleet.json')) + self.assertNotIn(other, self.read('/opt/fork-fleet/fork-fleet.json')) + + def test_a_run_killed_between_the_choice_and_the_swap_fails_check(self): + a, b = os.path.join(self.tmp, 'a.json'), os.path.join(self.tmp, 'b.json') + self.write(a, json.dumps({'pools': [{'name': 'desktop'}], 'which': 'a'})) + self.write(b, json.dumps({'pools': [{'name': 'desktop'}], 'which': 'b'})) + for first, then in ((['--fleet-config', a], ['--fleet-config', b]), + (['--fleet-config', a], ['--release-fleet-config']), + ([], ['--fleet-config', b])): + with self.subTest(then=then): + shutil.rmtree(self.root) + self.write(self.path('/proc/sys/net/ipv4/ip_forward'), '1\n') + self.assertEqual(self.install(*first)[0], 0) + self.template() + self.assertEqual(self.run_main('check')[0], 0) + swap = self.m.swap + + def killed(*args): + raise SystemExit('killed before the swap') + self.m.swap = killed + try: + self.assertIn('killed', str(self.install(*then)[0])) + finally: + self.m.swap = swap + code, out, _ = self.run_main('check') # the old config is still the installed one + self.assertEqual(code, 1, out) + self.assertEqual(self.install()[0], 0) + self.assertEqual(self.run_main('check')[0], 0) + + def test_a_fleet_config_link_into_an_open_folder_is_refused(self): + open_dir, safe = os.path.join(self.tmp, 'open'), os.path.join(self.tmp, 'safe') + os.makedirs(open_dir) + os.chmod(open_dir, 0o777) + self.write(os.path.join(open_dir, 'f.json'), json.dumps({'pools': [{'name': 'desktop'}]})) + os.makedirs(safe) + os.symlink(os.path.join(open_dir, 'f.json'), os.path.join(safe, 'link.json')) + code, _, err = self.install('--fleet-config', os.path.join(safe, 'link.json')) + self.assertEqual(code, 1) + self.assertIn(f'{open_dir} can be written by other users', err) + self.assertFalse(os.path.exists(self.path('/opt'))) + + def test_a_fleet_config_a_runner_user_owns_or_its_group_may_write_is_refused(self): + folder = os.path.join(self.tmp, 'cfg') + os.makedirs(folder) + config = os.path.join(folder, 'f.json') + self.write(config, json.dumps({'pools': [{'name': 'desktop'}]})) + me, group, other = os.geteuid(), os.getegid(), 2 ** 31 - 7 + + def runner(uid, gid): + def getpwnam(name): + if name != 'fork-runner-3': + raise KeyError(name) + return types.SimpleNamespace(pw_uid=uid, pw_gid=gid) + return types.SimpleNamespace(getpwnam=getpwnam) + real = self.m.pwd + try: + self.m.pwd = runner(me, other) # the file belongs to a runner user + self.assertEqual(self.m.job_writable(config), f'{config} belongs to a runner user') + self.m.pwd = runner(other, group) # its group is a runner user's, and the group may write it + os.chmod(config, 0o664) + self.assertEqual(self.m.job_writable(config), f'{config} can be written by other users') + os.chmod(config, 0o644) # the group may only read it: the file passes (a folder above may not) + self.assertNotIn(config + ' ', self.m.job_writable(config) or '') + finally: + self.m.pwd = real + + def test_install_sets_back_modes_check_names(self): + self.assertEqual(self.install()[0], 0) + self.template() + self.assertEqual(self.run_main('check')[0], 0) + os.chmod(self.path('/opt/fork-fleet/fork-fleet-launch.sh'), 0o777) + os.chmod(self.path('/opt/fork-fleet/fork-fleet-walls'), 0o777) + os.chmod(self.path('/etc/fork-fleet/desk-key'), 0o644) + os.chmod(self.path('/etc/fork-fleet'), 0o755) + code, out, _ = self.run_main('check') + self.assertEqual(code, 1) + for line in ('FAIL kit /opt/fork-fleet/fork-fleet-launch.sh: mode 777, not 755', + 'FAIL kit /opt/fork-fleet/fork-fleet-walls: mode 777, not 755', + 'FAIL kit /etc/fork-fleet/desk-key: mode 644, not 600', + 'FAIL kit /etc/fork-fleet: mode 755, not 700'): + self.assertIn(line, out) + self.assertEqual(self.install()[0], 0) + self.assertEqual(stat.S_IMODE(os.stat(self.path('/opt/fork-fleet/fork-fleet-launch.sh')).st_mode), 0o755) + code, out, _ = self.run_main('check') + self.assertEqual(code, 0, out) + + def test_a_second_install_or_uninstall_at_the_same_time_is_refused(self): + self.assertEqual(self.install()[0], 0) + fd = os.open(self.path('/etc/fork-fleet'), os.O_RDONLY | os.O_DIRECTORY) + try: + fcntl.flock(fd, fcntl.LOCK_EX) + stage = os.path.join(self.path('/opt/fork-fleet'), '.kit-stage-other') + os.makedirs(stage) # the other run's staging folder: not swept + for argv in (['install', '--source', self.source, '--sha', SHA2], ['uninstall']): + with self.subTest(argv[0]): + code, _, err = self.run_main(*argv) + self.assertEqual(code, 1) + self.assertIn('another desk-install install or uninstall is running', err) + self.assertTrue(os.path.isdir(stage)) + self.assertEqual(self.read('/opt/fork-fleet/SOURCE'), SHA1 + '\n') + finally: + os.close(fd) + self.assertEqual(self.install(sha=SHA2)[0], 0) + + def test_the_lock_is_in_the_root_only_folder(self): + """lens225r3 nit 4: a lock on the world-readable /opt/fork-fleet could be held by any local user; the lock is + on /etc/fork-fleet (0700), which only root can open.""" + self.assertEqual(self.install()[0], 0) + fd = os.open(self.path('/opt/fork-fleet'), os.O_RDONLY | os.O_DIRECTORY) + try: + fcntl.flock(fd, fcntl.LOCK_EX) # what any local user could do: install is not blocked by it + self.assertEqual(self.install(sha=SHA2)[0], 0) + self.assertEqual(self.run_main('uninstall')[0], 0) + finally: + os.close(fd) + + def test_a_linked_or_foreign_owned_folder_or_key_is_refused(self): + """lens225r3 nit 1 (probe): install neither followed nor fixed a linked or non-root /opt/fork-fleet, + /etc/fork-fleet or desk key, so `check` FAILed forever. Install now refuses before changing anything.""" + other = os.path.join(self.tmp, 'elsewhere') + os.makedirs(other) + os.makedirs(os.path.dirname(self.path('/opt/fork-fleet')), exist_ok=True) + os.symlink(other, self.path('/opt/fork-fleet')) + code, _, err = self.install() + self.assertEqual(code, 1) + self.assertIn('/opt/fork-fleet is a link', err) + self.assertEqual(os.listdir(other), []) + os.unlink(self.path('/opt/fork-fleet')) + self.assertEqual(self.install()[0], 0) + self.template() + self.assertEqual(self.run_main('check')[0], 0) + real_lstat, opt = os.lstat, self.path('/opt/fork-fleet') + + def lstat(path, *a, **kw): # /opt/fork-fleet owned by another user (a test cannot chown) + st = real_lstat(path, *a, **kw) + if os.fspath(path) != opt: + return st + return os.stat_result((st.st_mode, st.st_ino, st.st_dev, st.st_nlink, st.st_uid + 1, *st[5:])) + before = self.tree() + with unittest.mock.patch.object(self.m.os, 'lstat', lstat): + code, _, err = self.install(sha=SHA2) + self.assertIn('FAIL kit /opt/fork-fleet: a link or not owned by root', self.run_main('check')[1]) + self.assertEqual(code, 1) + self.assertIn('/opt/fork-fleet is not owned by root', err) + self.assertEqual(self.tree(), before) + + def test_a_unit_or_setting_with_another_owner_fails_check_and_install_puts_it_back(self): + """lens225r3 nit 5: units were checked for text and mode, not owner. A unit or setting that is a link or not + root's FAILs check and is written again (as root) by install.""" + self.assertEqual(self.install()[0], 0) + self.template() + unit = self.path('/etc/systemd/system/fork-fleet-desk-agent.service') + text = slurp(unit) + real = os.path.join(self.tmp, 'unit-elsewhere') + self.write(real, text) + os.unlink(unit) + os.symlink(real, unit) # same text through a link: not root's file + code, out, _ = self.run_main('check') + self.assertEqual(code, 1) + self.assertIn('fork-fleet-desk-agent.service: not the unit of release', out) + self.assertEqual(self.install()[0], 0) + self.assertFalse(os.path.islink(unit)) + self.assertEqual(self.run_main('check')[0], 0) + + def test_a_hand_made_user_in_other_groups_or_with_a_home_fails_check(self): + """lens225r3 nit 3: user_problem now matches the launcher's check_users (uid not 0, own group only) and wants + no home that is there.""" + self.assertEqual(self.install()[0], 0) + self.template() + os.makedirs(self.path('/home/fork-runner-1')) + for name, text, want in (('groups-fork-runner-1', '1001 27', 'fork-runner-1 is in other groups (1001 27)'), + ('uid-fork-runner-2', '0', 'fork-runner-2 has uid 0'), + ('home-fork-runner-1', '/home/fork-runner-1', 'fork-runner-1 has the home')): + with self.subTest(name): + self.write(os.path.join(self.fakes, name), text + '\n') + code, out, _ = self.run_main('check') + self.assertEqual(code, 1) + self.assertIn('FAIL runner-user ' + want, out) + os.unlink(os.path.join(self.fakes, name)) + self.assertEqual(self.run_main('check')[0], 0) + + def test_check_fails_on_a_runner_user_that_can_log_in(self): + self.assertEqual(self.install()[0], 0) + self.template() + self.assertEqual(self.run_main('check')[0], 0) + for name, text, want in (('shell-fork-runner-1', '/bin/bash', "fork-runner-1 has the login shell '/bin/bash'"), + ('pw-fork-runner-2', '', 'fork-runner-2 has a usable password or none')): + with self.subTest(name): + self.write(os.path.join(self.fakes, name), text + '\n') + code, out, _ = self.run_main('check') + self.assertEqual(code, 1) + self.assertIn('FAIL runner-user ' + want, out) + os.unlink(os.path.join(self.fakes, name)) + self.write(self.path('/etc/at.allow'), 'fork-runner-2\n') + self.assertIn('FAIL runner-user fork-runner-2 is in /etc/at.allow', self.run_main('check')[1]) + + def test_refusals_change_nothing(self): + cases = { + 'not a release sha': (lambda: None, ['--sha', 'abc'], '--sha must be'), + 'no machine file': (lambda: os.unlink(self.m.MACHINE), [], 'machine file'), + 'no desk section': (lambda: self.machine({'version': 1}), [], 'no "desk" section'), + 'bad host key': (lambda: self.machine({'version': 1, 'desk': {**MACHINE['desk'], + 'hostKey': 'ssh-rsa AAAA'}}), [], 'hostKey'), + 'too many slots': (lambda: self.machine({'version': 1, 'desk': {**MACHINE['desk'], 'slots': 9}}), [], + 'slots'), + 'bad net base': (lambda: self.machine({'version': 1, 'desk': {**MACHINE['desk'], 'netBase': '10.256'}}), + [], 'netBase'), + 'unknown key': (lambda: self.machine({'version': 1, 'desk': {**MACHINE['desk'], 'token': 'x'}}), [], + 'token'), + 'os not supported': (lambda: self.write(os.path.join(self.fakes, 'os-exit'), '3'), [], + 'not supported yet'), + 'an install the kit did not make': ( + lambda: self.write(self.path('/opt/fork-fleet/fork-fleet-launch.sh'), 'old'), [], + 'this kit did not make'), + 'a foreign unit': (lambda: self.write(self.path('/etc/systemd/system/fork-fleet-desk-agent.service'), 'x'), + [], 'this kit did not make'), + 'no such fleet config': (lambda: None, ['--fleet-config', '/nonexistent/x.json'], 'no such file'), + 'a drop-in for the agent': (lambda: self.write(self.path( + '/etc/systemd/system/fork-fleet-desk-agent.service.d/override.conf'), '[Service]\nExecStart=\n'), [], + 'override.conf changes fork-fleet-desk-agent.service'), + 'a dash-prefix drop-in': (lambda: self.write(self.path( + '/run/systemd/system/t3code-fleet-.timer.d/x.conf'), '[Timer]\n'), [], 'changes t3code-fleet'), + 'a record of another shape': (lambda: self.write(self.path('/etc/fork-fleet/desk-kit.json'), + '{"version": 1}'), [], 'not an install record'), + 'a hand-made runner user with a login shell': ( + lambda: (self.write(os.path.join(self.fakes, 'users'), 'fork-runner-1\n'), + self.write(os.path.join(self.fakes, 'shell-fork-runner-1'), '/bin/bash\n')), [], + "login shell '/bin/bash'"), + 'a hand-made runner user with a password': ( + lambda: (self.write(os.path.join(self.fakes, 'users'), 'fork-runner-2\n'), + self.write(os.path.join(self.fakes, 'pw-fork-runner-2'), '$6$salt$hash\n')), [], + 'usable password'), + 'a hand-made runner user with uid 0': ( + lambda: (self.write(os.path.join(self.fakes, 'users'), 'fork-runner-1\n'), + self.write(os.path.join(self.fakes, 'uid-fork-runner-1'), '0\n')), [], 'has uid 0'), + 'a hand-made runner user in other groups': ( + lambda: (self.write(os.path.join(self.fakes, 'users'), 'fork-runner-2\n'), + self.write(os.path.join(self.fakes, 'groups-fork-runner-2'), '1001 27\n')), [], + 'is in other groups (1001 27)'), + 'a hand-made runner user with a home': ( + lambda: (self.write(os.path.join(self.fakes, 'users'), 'fork-runner-1\n'), + os.makedirs(self.path('/home/fork-runner-1')), + self.write(os.path.join(self.fakes, 'home-fork-runner-1'), '/home/fork-runner-1\n')), [], + 'has the home /home/fork-runner-1'), + 'a linked /opt/fork-fleet': ( + lambda: (os.makedirs(os.path.join(self.tmp, 'elsewhere'), exist_ok=True), + os.makedirs(self.path('/opt')), + os.symlink(os.path.join(self.tmp, 'elsewhere'), self.path('/opt/fork-fleet'))), [], + '/opt/fork-fleet is a link'), + 'a linked /etc/fork-fleet': ( + lambda: (os.makedirs(os.path.join(self.tmp, 'elsewhere'), exist_ok=True), + os.makedirs(self.path('/etc')), + os.symlink(os.path.join(self.tmp, 'elsewhere'), self.path('/etc/fork-fleet'))), [], + '/etc/fork-fleet is a link'), + 'a linked desk key': ( + lambda: (self.write(self.path('/etc/fork-fleet/desk-kit.json'), json.dumps( + {'version': 1, 'createdUsers': [], 'fleetConfig': 'release', 'sha': SHA1}), 0o600), + self.write(os.path.join(self.tmp, 'elsewhere-key'), 'PRIVATE\n', 0o600), + os.symlink(os.path.join(self.tmp, 'elsewhere-key'), self.path('/etc/fork-fleet/desk-key'))), + [], '/etc/fork-fleet/desk-key is a link'), + 'a hand-made runner user in cron.allow': ( + lambda: (self.write(os.path.join(self.fakes, 'users'), 'fork-runner-1\n'), + self.write(self.path('/etc/cron.allow'), 'fork-runner-1\n')), [], 'is in /etc/cron.allow'), + 'a record that names a project config that is gone': ( + lambda: (self.write(self.path('/etc/fork-fleet/desk-kit.json'), json.dumps( + {'version': 1, 'createdUsers': [], 'fleetConfig': 'project', 'sha': SHA1}), 0o600)), [], + 'run again with --fleet-config FILE or --release-fleet-config'), + 'a fleet config others may write': (lambda: (os.makedirs(os.path.join(self.tmp, 'open'), exist_ok=True), + os.chmod(os.path.join(self.tmp, 'open'), 0o777), + self.write(os.path.join(self.tmp, 'open', 'f.json'), '{}')), + ['--fleet-config', os.path.join(self.tmp, 'open', 'f.json')], + 'can be written by other users'), + } + for name, (setup, args, msg) in cases.items(): + with self.subTest(name): + shutil.rmtree(self.root, ignore_errors=True) + os.makedirs(self.root) + for n in os.listdir(self.fakes): # the users of the case before + if n == 'users' or n.startswith(('shell-', 'pw-', 'uid-', 'groups-', 'home-')): + os.unlink(os.path.join(self.fakes, n)) + self.machine(MACHINE) + self.write(os.path.join(self.fakes, 'os-exit'), '0') + setup() + before = self.tree() + self.clear_calls() + argv = ['install', '--source', self.source, '--sha', SHA1, *args] + if '--sha' in args: + argv = ['install', '--source', self.source, *args] + code, _, err = self.run_main(*argv) + self.assertEqual(code, 1) + self.assertIn(msg, err) + self.assertEqual(self.tree(), before) + self.assertFalse([c for c in self.calls() if not c.startswith(('id ', 'getent ', 'systemctl show'))], + self.calls()) + + def test_bad_release_is_not_installed(self): + self.assertEqual(self.install()[0], 0) + before = {k: v for k, v in self.tree().items() if k != '/etc/fork-fleet/desk-kit.json'} + for name, extra, msg in ( + ('no pool', {'.github/fork-fleet.json': json.dumps({'pools': [{'name': 'vm'}]})}, "no pool 'desktop'"), + ('gate does not load', {'.github/scripts/fork-fleet.py': 'raise SystemExit("bad config")\n'}, + 'the gate does not load'), + ('launcher syntax', {'.github/scripts/fork-fleet-launch.sh': '#!/bin/bash\nif then\n'}, 'bash -n'), + ('pools not a list', {'.github/fork-fleet.json': json.dumps({'pools': {'desktop': {}}})}, + 'not a list of pools')): + with self.subTest(name): + bad = self.release('bad-' + name.replace(' ', '-'), extra) + code, _, err = self.install(sha=SHA2, source=bad) + self.assertEqual(code, 1) + self.assertIn(msg, err) + after = {k: v for k, v in self.tree().items() if k != '/etc/fork-fleet/desk-kit.json'} + self.assertEqual(after, before) + + def test_key_is_made_once_and_a_half_key_is_refused(self): + self.assertEqual(self.install()[0], 0) + os.unlink(self.path('/etc/fork-fleet/desk-key.pub')) + code, _, err = self.install() + self.assertEqual(code, 1) + self.assertIn('without', err) + os.unlink(self.path('/etc/fork-fleet/desk-key')) + self.write(self.path('/etc/fork-fleet/.desk-key.new'), 'torn') # a keygen that stopped + code, _, err = self.install() + self.assertEqual(code, 0, err) + self.assertEqual(self.read('/etc/fork-fleet/desk-key'), 'PRIVATE\n') + self.assertFalse(os.path.exists(self.path('/etc/fork-fleet/.desk-key.new'))) + + def test_reserve_and_port_reach_the_settings(self): + self.machine({'version': 1, 'desk': {**MACHINE['desk'], 'reserveMb': 4096, 'port': 2222, 'slots': 1, + 'netBase': '10.77'}}) + self.assertEqual(self.install()[0], 0) + self.assertIn('RUNNER_USERS=fork-runner-1\nRESERVE_MB=4096\nNET_BASE=10.77\n', + self.read('/etc/fork-fleet/launch.conf')) + self.assertTrue(self.read('/etc/fork-fleet/desk-known_hosts').endswith( + f'[fleet-host.example]:2222 {HOST_KEY}\n')) + + def test_existing_user_is_used_not_recorded(self): + self.write(os.path.join(self.fakes, 'users'), 'fork-runner-1\n') + self.assertEqual(self.install()[0], 0) + self.assertEqual(json.loads(self.read('/etc/fork-fleet/desk-kit.json'))['createdUsers'], ['fork-runner-2']) + self.assertEqual(self.run_main('uninstall')[0], 0) + self.assertEqual(slurp(os.path.join(self.fakes, 'users')).split(), ['fork-runner-1']) + + def test_a_user_made_by_a_run_that_died_is_removed_by_uninstall(self): + self.write(os.path.join(self.fakes, 'fail-after'), 'useradd\n') + code, _, _ = self.install() + self.assertEqual(code, 1) + self.assertEqual(slurp(os.path.join(self.fakes, 'users')).split(), ['fork-runner-1']) + self.assertEqual(self.run_main('uninstall')[0], 0) + self.assertEqual(slurp(os.path.join(self.fakes, 'users')).split(), []) + + def test_agent_refusing_its_settings_starts_nothing(self): + self.write(os.path.join(self.fakes, 'agent-bad'), '') + code, _, err = self.install() + self.assertEqual(code, 1) + self.assertIn('the desk agent refuses its settings', err) + self.assertFalse([c for c in self.calls() if c.startswith(('systemctl enable', 'systemctl restart'))]) + + # --------------------------------------------------------------------------------------------------- check + + def test_check_names_each_failure(self): + self.assertEqual(self.install()[0], 0) + self.write(os.path.join(self.fakes, 'wall-fails'), '') + self.write(os.path.join(self.fakes, 'agent-bad'), '') + self.write(os.path.join(self.fakes, 'down'), 'fork-fleet-desk-agent.service\n') + self.write(self.path('/opt/fork-fleet/hook.sh'), '#!/bin/bash\nevil\n', 0o755) + self.write(self.path('/proc/sys/net/ipv4/ip_forward'), '0\n') + os.unlink(os.path.join(self.fakes, 'users')) + code, out, _ = self.run_main('check') + self.assertEqual(code, 1) + for line in ('FAIL kit /opt/fork-fleet/hook.sh: differs', 'FAIL vsock closed: open', + 'FAIL agent-settings bad settings', 'FAIL unit fork-fleet-desk-agent.service', + 'PASS unit t3code-fleet-refresh.timer', 'FAIL template', 'FAIL runner-user fork-runner-1', + 'FAIL ip-forward'): + self.assertIn(line, out) + self.assertTrue(out.endswith('8 FAIL\n'), out) + + def test_check_fails_on_a_drop_in_and_on_settings_the_machine_file_no_longer_gives(self): + self.assertEqual(self.install()[0], 0) + self.template() + self.assertEqual(self.run_main('check')[0], 0) + self.machine({'version': 1, 'desk': {**MACHINE['desk'], 'slots': 1}}) + code, out, _ = self.run_main('check') + self.assertEqual(code, 1) + self.assertIn('FAIL settings /etc/fork-fleet/launch.conf differs from the machine file', out) + self.machine(MACHINE) + self.write(self.path('/etc/systemd/system/fork-fleet-desk-agent.service.d/override.conf'), + '[Service]\nExecStart=\nExecStart=/bin/sh -c evil\n') + code, out, _ = self.run_main('check') + self.assertEqual(code, 1) + self.assertIn('override.conf: a drop-in the kit did not write changes fork-fleet-desk-agent.service', out) + + def test_check_fails_when_a_unit_is_not_the_release_unit(self): + self.assertEqual(self.install()[0], 0) + self.template() + new = self.release('release2', {'.github/scripts/fork-fleet-desk-agent.service': '# v2\n[Service]\n'}) + self.assertIn('stopped after', str(self.install(sha=SHA2, source=new, stop_after='settings')[0])) + code, out, _ = self.run_main('check') + self.assertEqual(code, 1) + self.assertIn('fork-fleet-desk-agent.service: not the unit of release', out) + self.assertEqual(self.install(sha=SHA2, source=new)[0], 0) + self.assertEqual(self.run_main('check')[0], 0) + + def test_check_names_a_runner_user_left_by_fewer_slots(self): + self.assertEqual(self.install()[0], 0) + self.machine({'version': 1, 'desk': {**MACHINE['desk'], 'slots': 1}}) + self.assertEqual(self.install()[0], 0) + self.assertIn('PASS runner-user fork-runner-2 is no longer a slot', self.run_main('check')[1]) + + def test_check_before_install_fails_without_writing(self): + code, out, _ = self.run_main('check') + self.assertEqual(code, 1) + self.assertIn('FAIL kit', out) + self.assertFalse(os.path.exists(self.path('/opt'))) + + # ----------------------------------------------------------------------------------------------- uninstall + + def test_uninstall_leaves_nothing_of_its_own(self): + self.write(self.path('/etc/cron.deny'), 'someone\n') + self.write(self.path('/etc/systemd/system/other.service'), 'x') + self.write(os.path.join(self.fakes, 'users'), 'someone\n') + self.assertEqual(self.install()[0], 0) + self.write(os.path.join(self.fakes, 'active'), 'desktop-1\n') + self.write(self.path('/var/lib/fork-fleet/runners/x'), '') + self.write(os.path.join(self.fakes, 'netns'), 'fork-fleet-fork-runner-1\n') # the namespaces that are there + self.clear_calls() + code, out, err = self.run_main('uninstall') + self.assertEqual(code, 0, err) + calls = self.calls() + for c in ('systemctl disable --now fork-fleet-desk-agent.service', 'launch active', 'launch stop desktop-1', + 'launch prune', 'ip netns delete fork-fleet-fork-runner-1', 'nft delete table inet fork_fleet', + 'userdel fork-runner-1', 'userdel fork-runner-2'): + self.assertIn(c, calls) + self.assertLess(calls.index('launch stop desktop-1'), calls.index('userdel fork-runner-1')) + for path in ('/opt/fork-fleet', '/etc/fork-fleet', '/var/lib/fork-fleet', '/etc/sysctl.d/60-fork-fleet.conf', + '/etc/systemd/system/fork-fleet-desk-agent.service'): + self.assertFalse(os.path.lexists(self.path(path)), path) + self.assertEqual(self.read('/etc/cron.deny'), 'someone\n') + self.assertTrue(os.path.exists(self.path('/etc/systemd/system/other.service'))) + self.assertEqual(slurp(os.path.join(self.fakes, 'users')).split(), ['someone']) + code, out, _ = self.run_main('uninstall') # again: nothing to do + self.assertEqual(code, 0) + self.assertIn('nothing here was installed by this kit', out) + + def test_uninstall_keeps_what_the_kit_did_not_make(self): + self.assertEqual(self.install()[0], 0) + for path in ('/etc/fork-fleet/token', '/opt/fork-fleet/notes.txt', '/var/lib/fork-fleet/other/x'): + self.write(self.path(path), 'not the kit\'s\n') + self.write(self.path('/var/lib/fork-fleet/runners/desktop-1/x'), '') + code, out, err = self.run_main('uninstall') + self.assertEqual(code, 0, err) + left = {k for k in self.tree() if k.startswith(('/etc/fork-fleet', '/opt/fork-fleet', '/var/lib/fork-fleet'))} + self.assertEqual(left, {'/etc/fork-fleet', '/etc/fork-fleet/token', '/opt/fork-fleet', + '/opt/fork-fleet/notes.txt', '/var/lib/fork-fleet', '/var/lib/fork-fleet/other', + '/var/lib/fork-fleet/other/x'}) + self.assertIn('left /etc/fork-fleet/token: the kit did not make it', out) + + def test_uninstall_removes_the_deny_files_it_made(self): + self.write(self.path('/etc/cron.deny'), 'someone\n') # there before: kept, without the kit's lines + self.assertEqual(self.install()[0], 0) + self.assertEqual(self.read('/etc/at.deny'), 'fork-runner-1\nfork-runner-2\n') + self.assertEqual(self.run_main('uninstall')[0], 0) + self.assertFalse(os.path.lexists(self.path('/etc/at.deny'))) # with none, only root may use at again + self.assertEqual(self.read('/etc/cron.deny'), 'someone\n') + + def test_uninstall_removes_the_deny_lines_it_added_for_users_that_were_there(self): + self.write(os.path.join(self.fakes, 'users'), 'fork-runner-1\nfork-runner-2\n') # both made by hand + self.write(self.path('/etc/cron.deny'), 'someone\n') + self.assertEqual(self.install()[0], 0) + self.assertEqual(self.read('/etc/at.deny'), 'fork-runner-1\nfork-runner-2\n') + self.assertEqual(self.run_main('uninstall')[0], 0) + self.assertFalse(os.path.lexists(self.path('/etc/at.deny'))) # with none, only root may use at again + self.assertEqual(self.read('/etc/cron.deny'), 'someone\n') + + def test_uninstall_keeps_the_table_while_a_runner_still_runs(self): + self.assertEqual(self.install()[0], 0) + self.write(os.path.join(self.fakes, 'units'), 'fork-fleet-desktop-1.service\nfork-fleet-desktop-2.service\n') + self.write(os.path.join(self.fakes, 'stuck'), 'fork-fleet-desktop-2.service\n') + self.clear_calls() + code, _, err = self.run_main('uninstall') + self.assertEqual(code, 1) + self.assertIn('fork-fleet-desktop-2.service is still running', err) + calls = self.calls() + self.assertIn('systemctl stop fork-fleet-desktop-1.service', calls) # what the launcher did not stop + self.assertFalse([c for c in calls if c.startswith(('nft', 'userdel', 'ip netns delete'))], calls) + self.assertTrue(os.path.exists(self.path('/etc/fork-fleet/desk-kit.json'))) + os.unlink(os.path.join(self.fakes, 'stuck')) + self.assertEqual(self.run_main('uninstall')[0], 0) + self.assertIn('nft delete table inet fork_fleet', self.calls()) + + def test_uninstall_removes_every_runner_namespace(self): + self.write(os.path.join(self.fakes, 'users'), 'fork-runner-1\n') # made by hand, not by the kit + self.assertEqual(self.install()[0], 0) + self.write(os.path.join(self.fakes, 'netns'), 'fork-fleet-fork-runner-1\nfork-fleet-fork-runner-2\nother\n') + self.write(os.path.join(self.fakes, 'links'), 'ffh-1001\neth0\nffh-1009\n') + self.assertEqual(self.run_main('uninstall')[0], 0) + self.assertEqual(slurp(os.path.join(self.fakes, 'netns')).split(), ['other']) + self.assertEqual(slurp(os.path.join(self.fakes, 'links')).split(), ['eth0']) + + def test_uninstall_removes_its_record_last(self): + self.assertEqual(self.install()[0], 0) + gone, unlink, scandir = [], os.unlink, os.scandir + + def record(path, *a, **kw): + gone.append(os.path.basename(path)) + return unlink(path, *a, **kw) + + class Sorted: + """Folders listed by name, so the record (desk-kit.json sorts before desk.conf and launch.conf) comes + first: a whole-folder removal would take it before the rest, whatever order the file system gives.""" + def __init__(self, *a, **kw): + self.it = scandir(*a, **kw) + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.it.close() + + def __iter__(self): + return iter(sorted(self.it, key=lambda e: e.name)) + + def close(self): + self.it.close() + os.unlink, os.scandir = record, Sorted + try: + self.assertEqual(self.run_main('uninstall')[0], 0) + finally: + os.unlink, os.scandir = unlink, scandir + self.assertEqual(gone[-1], 'desk-kit.json', gone) + + def test_uninstall_refuses_a_record_of_another_shape(self): + self.assertEqual(self.install()[0], 0) + for bad in ('[]', '{"version": 1, "createdUsers": "fork-runner-1", "fleetConfig": "release"}', + '{"version": 1, "createdUsers": ["root;x"], "fleetConfig": "release"}'): + with self.subTest(bad): + self.write(self.path('/etc/fork-fleet/desk-kit.json'), bad, 0o600) + code, _, err = self.run_main('uninstall') + self.assertEqual(code, 1) + self.assertIn('not an install record', err) + + def test_uninstall_stopped_half_way_runs_again(self): + self.assertEqual(self.install()[0], 0) + self.write(os.path.join(self.fakes, 'fail'), 'userdel\n') + code, _, err = self.run_main('uninstall') + self.assertEqual(code, 1) + self.assertTrue(os.path.exists(self.path('/etc/fork-fleet/desk-kit.json'))) # the record stays + code, _, err = self.run_main('uninstall') + self.assertEqual(code, 0, err) + self.assertFalse(os.path.exists(self.path('/etc/fork-fleet'))) + self.assertEqual(slurp(os.path.join(self.fakes, 'users')).split(), []) + + def test_uninstall_of_a_foreign_install_changes_nothing(self): + self.write(self.path('/opt/fork-fleet/fork-fleet-launch.sh'), 'old') + before = self.tree() + code, out, _ = self.run_main('uninstall') + self.assertEqual(code, 0) + self.assertEqual(self.tree(), before) + self.assertEqual(self.calls(), []) + + +class Release(unittest.TestCase): + def test_every_file_the_kit_installs_is_in_this_repository(self): + m = load() + for src in [*m.FILES.values(), *m.UNITS.values(), m.WALLS]: + self.assertTrue(os.path.exists(os.path.join(REPO, src)), src) + self.assertTrue(os.listdir(os.path.join(REPO, m.WALLS))) + + def test_bootstrap_checks_its_arguments_before_anything(self): + boot = os.path.join(KIT, 'bootstrap.sh') + self.assertEqual(subprocess.run(['sh', '-n', boot]).returncode, 0) + env = {'PATH': '/usr/bin:/bin'} + for args, msg in ((['--repo', 'x'], '--repo must be'), (['--repo', 'a/b', '--sha', 'x'], '--sha must be'), + (['--nope'], 'unknown option'), + # flags after --fleet-config are checked too: a later --sha cannot replace a good one + (['--repo', 'a/b', '--sha', 'a' * 40, '--fleet-config', boot, '--sha', 'x'], '--sha must be'), + (['--repo', 'a/b', '--sha', 'a' * 40, '--fleet-config', boot, '--repo', 'x'], + '--repo must be'), + (['--repo', 'a/b', '--sha', 'a' * 40, '--fleet-config', '/nonexistent/f.json'], + 'no such file'), + (['--repo', 'a/b', '--sha', 'a' * 40, '--fleet-config'], 'needs a value')): + r = subprocess.run(['sh', boot, *args], capture_output=True, text=True, env=env) + self.assertEqual(r.returncode, 1) + self.assertIn(msg, r.stderr) + if os.geteuid() != 0: + r = subprocess.run(['sh', boot, '--repo', 'a/b', '--sha', 'a' * 40], capture_output=True, text=True, + env=env) + self.assertIn('run this as root', r.stderr) + + +if __name__ == '__main__': + unittest.main()