From 57e1cb53a060041f3535ab97a874fbe2f5a1c381 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:13:43 +0000 Subject: [PATCH 1/4] fork fleet: `seed` verb and `seeds` in the fleet config (D11) A desk pool may list warm caches ("seeds") in its fleet config entry. The always-on host's forced command answers a desktop's `seeds` (the pool's list) and `seed NAME` (that one ready-made tar, with a size and SHA-256 trailer) in an ssh session beside the agent's. Only names of the list, never a path; exit 3 while another stream for the pool runs, 4 when the file may not be sent. The desktop's fetcher follows in the next change. Co-Authored-By: Claude Opus 5.5 --- .github/fork-ci-tests.txt | 1 + .github/fork-fleet.md | 26 +++ .github/scripts/fork-fleet-desk-seed.py | 129 +++++++++++++ .github/scripts/fork-fleet-desk-seed.test.py | 186 +++++++++++++++++++ .github/scripts/fork-fleet-desk.py | 18 ++ .github/scripts/fork-fleet.py | 9 + 6 files changed, 369 insertions(+) create mode 100644 .github/scripts/fork-fleet-desk-seed.py create mode 100644 .github/scripts/fork-fleet-desk-seed.test.py diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index ccedb1f29c7de..e4cebd33243c1 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -9,6 +9,7 @@ .github/scripts/fork-contracts.test.py .github/scripts/fork-fleet-desk-agent.test.py .github/scripts/fork-fleet-desk-key.test.py +.github/scripts/fork-fleet-desk-seed.test.py .github/scripts/fork-fleet-desk.test.py .github/scripts/fork-fleet.test.py .github/scripts/fork-fleet-refresh.test.py diff --git a/.github/fork-fleet.md b/.github/fork-fleet.md index a42c9379baa9f..1a0b7c895f3a5 100644 --- a/.github/fork-fleet.md +++ b/.github/fork-fleet.md @@ -448,6 +448,7 @@ done files="fork-fleet.json:fork-fleet.json:644 scripts/fork-fleet.py:fork-fleet.py:644 scripts/fork-fleet-hook.sh:hook.sh:755 scripts/fork-fleet-launch.sh:fork-fleet-launch.sh:755 scripts/fork-fleet-desk.py:fork-fleet-desk.py:644 scripts/fork-fleet-desk-wire.py:fork-fleet-desk-wire.py:644 + scripts/fork-fleet-desk-seed.py:fork-fleet-desk-seed.py:644 scripts/fork-fleet-refresh.service:fork-fleet-refresh.service:644 scripts/fork-fleet-refresh.timer:fork-fleet-refresh.timer:644 scripts/fork-fleet-job-paths.txt:fork-fleet-job-paths.txt:644" @@ -683,6 +684,31 @@ Bun, Node and Rust are installed per job by the setup actions, into the job's own home, which goes with its copy. Leave a few tens of GB of disk for `/var/lib/fork-fleet/runners`. +#### Seeds (warm caches for a desk pool) + +A job's copy starts empty, so each job would download its tools and +dependencies again. A desk pool may list **seeds** in its fleet config entry, +ready-made caches the desktop keeps and its jobs start from: + +```json +"seeds": [{"name": "gradle", "every_hours": 6, "max_age_hours": 72, "max_mb": 4096}] +``` + +`every_hours` is how often a desktop takes a new one, `max_age_hours` (at +least `every_hours`) the age, from when it was made, at which it is dropped, +`max_mb` its largest size. Nothing in the fleet makes a seed: on the always-on +host the project's own timer or job writes a tar of a cache folder's contents +to `/var/lib/fork-fleet/seed-files/NAME.tar`, by a temp file and a rename (a +file changed while it is sent is refused). That folder and its files are +root's (or the project's), readable by the desk-link user and writable by no +one else; never put a secret in a seed: every job of the pool reads it. + +The desktop asks for them with the desk link's key, in an ssh session of its +own: the same forced command answers `seeds` with the pool's list and +`seed NAME` with that one file, only a name of that list, and a trailer with its +size and SHA-256 (`fork-fleet-desk-seed.py`, which step 3 installs; its header +has the exit codes). + ## Owner steps on the repository Run these from a machine where `gh` is logged in as the owner. None of them diff --git a/.github/scripts/fork-fleet-desk-seed.py b/.github/scripts/fork-fleet-desk-seed.py new file mode 100644 index 0000000000000..2a68f886fc7df --- /dev/null +++ b/.github/scripts/fork-fleet-desk-seed.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Fork only: warm caches ("seeds") for a desk pool's jobs (.github/fork-fleet.md, "Seeds"). + +A project lists the caches its desk pool's jobs may start from in the pool's `seeds` (fork-fleet.json): a name, how +often a desktop takes a new one (every_hours), the age at which it is dropped (max_age_hours, from when it was made) +and its largest size (max_mb). On the always-on host something of the project's own (a timer, a job) makes each one: +a tar of a folder's contents at FILES/NAME.tar, written to a temp file and renamed. Nothing here makes one. The +desktop pulls them with the desk link's key, in an ssh session of its own beside the agent's: + + ssh ... seeds the forced command (fork-fleet-desk.py serve POOL, the desk-link user) sees it in + SSH_ORIGINAL_COMMAND and prints the pool's list (one JSON object; `bytes` and `made` of each + seed the host offers now, `why` for one it does not) + ssh ... seed NAME streams FILES/NAME.tar: only a name of that list, never a path the desktop names. The last + stderr line is the trailer `seed NAME BYTES SHA256 MADE` (MADE: the file's mtime, epoch seconds) + +The host side exits 2 for any other command, 3 while another stream for the pool runs, 4 when the file may not be +sent (missing, a link, in a folder or a file others may write, above max_mb, older than max_age_hours, or changed +while it was sent: made in place, not by rename). Standard library only. +""" +import fcntl, hashlib, importlib.util, json, os, re, stat, sys, time + +sys.dont_write_bytecode = True +HERE = os.path.dirname(os.path.realpath(__file__)) + + +def module(name, file): + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, file)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +FILES = '/var/lib/fork-fleet/seed-files' # the always-on host's: NAME.tar, made by the project +CONFIG = os.path.join(HERE, 'fork-fleet.json') +PIECE = 1 << 20 +ITEM = {'everyHours': 'every_hours', 'maxAgeHours': 'max_age_hours', 'maxMb': 'max_mb'} + + +class Refused(Exception): + def __init__(self, msg, code): + super().__init__(msg) + self.code = code + + +def pool_seeds(pool, config): + try: + cfg = module('fork_fleet', 'fork-fleet.py').load(config) + except (OSError, ValueError) as e: + raise Refused(f'the fleet config: {e}', 2) from None + for p in cfg['pools']: + if p['name'] == pool: + return {s['name']: s for s in p.get('seeds', [])} + raise Refused(f'pool {pool} is not in {config}', 2) + + +def offered(seed, files, now): + """An open fd and the stat of FILES/NAME.tar; Refused (4) when it may not be sent.""" + path = os.path.join(files, seed['name'] + '.tar') + try: + st = os.lstat(files) + if not stat.S_ISDIR(st.st_mode) or st.st_mode & 0o022: + raise Refused(f'{files} must be a folder (not a link) that only its owner may write', 4) + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC) + except OSError as e: + raise Refused(f'{path}: {"not made yet" if isinstance(e, FileNotFoundError) else e.strerror}', 4) from None + st = os.fstat(fd) + hours = seed['max_age_hours'] + why = ('not a plain file' if not stat.S_ISREG(st.st_mode) else 'others may write it' if st.st_mode & 0o022 + else f'above max_mb ({seed["max_mb"]})' if st.st_size > seed['max_mb'] << 20 + else f'older than max_age_hours ({hours})' if now - st.st_mtime > hours * 3600 else None) + if why: + os.close(fd) + raise Refused(f'{path}: {why}', 4) + return fd, st + + +def offer(pool, command, lock_dir, out=1, err=sys.stderr, config=CONFIG, files=FILES, now=time.time): + try: + if os.geteuid() == 0: + raise Refused('serve never runs as root', 2) + word, _, name = command.partition(' ') + seeds = pool_seeds(pool, config) + if command == 'seeds': + got = [] + for s in seeds.values(): + item = {'name': s['name'], **{k: s[v] for k, v in ITEM.items()}} + try: + fd, st = offered(s, files, now()) + os.close(fd) + item.update(bytes=st.st_size, made=int(st.st_mtime)) + except Refused as e: + item['why'] = str(e) + got.append(item) + os.write(out, (json.dumps({'v': 1, 'seeds': got}) + '\n').encode()) + return 0 + if word != 'seed' or name not in seeds: + raise Refused(f'refused: {command[:60]!r} (only `seeds`, or `seed NAME` of pool {pool}\'s seeds)', 2) + lock = os.open(os.path.join(lock_dir, f'{pool}.seed.lock'), os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + try: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + raise Refused(f'another seed stream for pool {pool} runs', 3) from None + fd, st = offered(seeds[name], files, now()) + try: + sha, sent = send(fd, out) + after = os.fstat(fd) + finally: + os.close(fd) + finally: + os.close(lock) + if sent != st.st_size or (after.st_size, after.st_mtime_ns) != (st.st_size, st.st_mtime_ns): + raise Refused(f'{name}: it changed while it was sent (made in place, not by rename?)', 4) + print(f'seed {name} {sent} {sha} {int(st.st_mtime)}', file=err, flush=True) + return 0 + except Refused as e: + print(f'fork-fleet-desk seed: {e}', file=err, flush=True) + return e.code + + +def send(fd, out): + sha, sent = hashlib.sha256(), 0 + while data := os.read(fd, PIECE): + sha.update(data) + sent += len(data) + view = memoryview(data) + while view: + view = view[os.write(out, view):] + return sha.hexdigest(), sent diff --git a/.github/scripts/fork-fleet-desk-seed.test.py b/.github/scripts/fork-fleet-desk-seed.test.py new file mode 100644 index 0000000000000..6fb4307c5a897 --- /dev/null +++ b/.github/scripts/fork-fleet-desk-seed.test.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Tests of fork-fleet-desk-seed.py (a desk pool's warm caches) and the `seeds` check of fork-fleet.json: the host's +offer in-process and the forced command's divert in fork-fleet-desk.py. Temp folders only; no root, no ssh, no +network. Run: python3 -I -B . Linux only.""" +import fcntl, hashlib, importlib.util, io, json, os, shutil, subprocess, sys, tarfile, tempfile, time, unittest +from unittest import mock + +sys.dont_write_bytecode = True +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def load(name, file): + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, file)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +seed, ff = load('fork_fleet_desk_seed', 'fork-fleet-desk-seed.py'), load('fork_fleet', 'fork-fleet.py') +with open(os.path.join(HERE, '..', 'fork-fleet.json'), encoding='utf-8') as _f: + CONFIG = json.load(_f) +POOL = CONFIG['pools'][0]['name'] +SEEDS = [{'name': 'm2', 'every_hours': 6, 'max_age_hours': 1000, 'max_mb': 2}, + {'name': 'npm', 'every_hours': 1, 'max_age_hours': 48, 'max_mb': 1}] + +class Base(unittest.TestCase): + def setUp(self): + self.t = os.path.realpath(tempfile.mkdtemp(prefix='seed-test-')) + self.addCleanup(shutil.rmtree, self.t, True) + for d in ('files', 'run', 'src'): + os.mkdir(os.path.join(self.t, d), 0o755) + self.config(SEEDS) + + def config(self, seeds): + cfg = json.loads(json.dumps(CONFIG)) + cfg['pools'][0]['seeds'] = seeds + with open(os.path.join(self.t, 'fork-fleet.json'), 'w', encoding='utf-8') as f: + json.dump(cfg, f) + + def make(self, name, members, mtime=None): + """FILES/NAME.tar of `members` ({path: bytes}), made as a project would: a temp file, then a rename.""" + tmp = os.path.join(self.t, 'files', f'.{name}.tmp') + with tarfile.open(tmp, 'w', format=tarfile.PAX_FORMAT) as tar: + for path, data in members.items(): + info = tarfile.TarInfo(path) + info.size, info.mode = len(data), 0o644 + tar.addfile(info, io.BytesIO(data)) + if mtime is not None: + os.utime(tmp, (mtime, mtime)) + os.chmod(tmp, 0o644) + os.replace(tmp, os.path.join(self.t, 'files', f'{name}.tar')) + with open(os.path.join(self.t, 'files', f'{name}.tar'), 'rb') as f: + return f.read() + + def offer(self, command, **kw): + r, w = os.pipe() + err = io.StringIO() + with os.fdopen(r, 'rb') as rf: + try: + code = seed.offer(POOL, command, os.path.join(self.t, 'run'), out=w, err=err, + config=os.path.join(self.t, 'fork-fleet.json'), files=os.path.join(self.t, 'files'), + **kw) + finally: + os.close(w) + return code, rf.read(), err.getvalue() + + +class Config(unittest.TestCase): + def test_seeds_are_checked(self): + def problems(seeds): + cfg = json.loads(json.dumps(CONFIG)) + cfg['pools'][0]['seeds'] = seeds + return [p for p in ff.check_config(cfg) if 'seed' in p] + self.assertEqual(problems(SEEDS), []) + self.assertEqual(problems([]), []) + good = SEEDS[0] + for bad in ({}, [{**good, 'name': 'M2'}], [{**good, 'name': '../x'}], [{**good, 'every_hours': 0}], + [{**good, 'max_mb': True}], [{**good, 'max_age_hours': 5}], [{**good, 'max_mb': '2'}], + [{k: v for k, v in good.items() if k != 'max_mb'}], ['m2'], [good, good]): + with self.subTest(bad): + self.assertEqual(len(problems(bad)), 1, bad) + + +class Host(Base): + def test_the_list_says_what_is_offered_and_why_not(self): + self.make('m2', {'a': b'x' * 100}) + code, out, err = self.offer('seeds') + self.assertEqual(code, 0, err) + got = json.loads(out) + self.assertEqual(got['v'], 1) + m2, npm = got['seeds'] + st = os.stat(os.path.join(self.t, 'files', 'm2.tar')) + self.assertEqual(m2, {'name': 'm2', 'everyHours': 6, 'maxAgeHours': 1000, 'maxMb': 2, 'bytes': st.st_size, + 'made': int(st.st_mtime)}) + self.assertNotIn('made', npm) + self.assertIn('not made yet', npm['why']) + + def test_a_seed_streams_whole_with_its_trailer(self): + data = self.make('m2', {'a/b': b'hello', 'c': b'x' * 5000}) + code, out, err = self.offer('seed m2') + self.assertEqual(code, 0, err) + self.assertEqual(out, data) + st = os.stat(os.path.join(self.t, 'files', 'm2.tar')) + self.assertEqual(err, f'seed m2 {len(data)} {hashlib.sha256(data).hexdigest()} {int(st.st_mtime)}\n') + + def test_other_commands_and_names_are_refused_and_send_nothing(self): + self.make('m2', {'a': b'x'}) + for command in ('seed', 'seed ../m2', 'seed /etc/passwd', 'seed m2 x', 'seed other', 'seeds x', 'serve x', + 'rm -rf /', 'seed M2'): + with self.subTest(command): + code, out, err = self.offer(command) + self.assertEqual((code, out), (2, b''), err) + + def test_a_file_that_may_not_be_sent_is_exit_4(self): + files = os.path.join(self.t, 'files') + now = time.time() + cases = {'missing': lambda: None, + 'a link': lambda: os.symlink(os.path.join(self.t, 'src', 'x.tar'), f'{files}/m2.tar'), + 'group-writable': lambda: (self.make('m2', {'a': b'x'}), os.chmod(f'{files}/m2.tar', 0o664)), + 'too big': lambda: self.make('m2', {'a': os.urandom(3 << 20)}), + 'too old': lambda: self.make('m2', {'a': b'x'}, mtime=now - 1001 * 3600), + 'a folder': lambda: os.mkdir(f'{files}/m2.tar'), + 'folder others write': lambda: (self.make('m2', {'a': b'x'}), os.chmod(files, 0o777))} + with open(os.path.join(self.t, 'src', 'x.tar'), 'wb') as f: + f.write(b'x') + os.chmod(os.path.join(self.t, 'src', 'x.tar'), 0o644) # only the link itself may be the reason + for name, make in cases.items(): + with self.subTest(name): + make() + code, out, err = self.offer('seed m2') + self.assertEqual((code, out), (4, b''), err) + os.chmod(files, 0o755) + p = f'{files}/m2.tar' + if os.path.isdir(p) and not os.path.islink(p): + os.rmdir(p) + elif os.path.lexists(p): + os.unlink(p) + + def test_a_second_stream_for_the_pool_is_exit_3(self): + self.make('m2', {'a': b'x'}) + with open(os.path.join(self.t, 'run', f'{POOL}.seed.lock'), 'a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + self.assertEqual(self.offer('seed m2')[:2], (3, b'')) + self.assertEqual(self.offer('seed m2')[0], 0) # the lock is gone with its holder + + def test_a_file_changed_while_sent_is_exit_4(self): + self.make('m2', {'a': b'x' * 3000}) + path, send = os.path.join(self.t, 'files', 'm2.tar'), seed.send + + def changing(fd, out): + with open(path, 'ab') as f: # made in place, not by rename + f.write(b'\0' * 512) + return send(fd, out) + with mock.patch.object(seed, 'send', changing): + code, _, err = self.offer('seed m2') + self.assertEqual(code, 4) + self.assertIn('changed while it was sent', err) + + def test_never_as_root_and_a_bad_config_is_exit_2(self): + with mock.patch.object(seed.os, 'geteuid', return_value=0): + self.assertEqual(self.offer('seeds')[:2], (2, b'')) + with open(os.path.join(self.t, 'fork-fleet.json'), 'w') as f: + f.write('{') + self.assertEqual(self.offer('seeds')[:2], (2, b'')) + + def test_the_forced_command_diverts_seed_requests(self): + opt = os.path.join(self.t, 'opt') # the host's install: the scripts and fork-fleet.json in one folder + os.mkdir(opt) + for f in ('fork-fleet-desk.py', 'fork-fleet-desk-wire.py', 'fork-fleet-desk-seed.py', 'fork-fleet.py'): + shutil.copy(os.path.join(HERE, f), opt) + shutil.copy(os.path.join(self.t, 'fork-fleet.json'), opt) + desk = os.path.join(opt, 'fork-fleet-desk.py') + base = {'PATH': os.environ['PATH'], 'FORK_FLEET_DESK_DIR': os.path.join(self.t, 'run2')} + got = subprocess.run([sys.executable, '-I', '-B', desk, 'serve', POOL], capture_output=True, text=True, + env={**base, 'SSH_ORIGINAL_COMMAND': 'seeds'}, timeout=30, stdin=subprocess.DEVNULL) + self.assertEqual(got.returncode, 0, got.stderr) + self.assertEqual([s['name'] for s in json.loads(got.stdout)['seeds']], ['m2', 'npm']) + got = subprocess.run([sys.executable, '-I', '-B', desk, 'serve', POOL], capture_output=True, text=True, + env={**base, 'SSH_ORIGINAL_COMMAND': 'seed nothing-like-this'}, timeout=30, + stdin=subprocess.DEVNULL) + self.assertEqual((got.returncode, got.stdout), (2, '')) + self.assertFalse(os.path.exists(os.path.join(self.t, 'run2', f'{POOL}.sock'))) # no session was served + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/scripts/fork-fleet-desk.py b/.github/scripts/fork-fleet-desk.py index 254aec7510e7f..eb08f20058a57 100755 --- a/.github/scripts/fork-fleet-desk.py +++ b/.github/scripts/fork-fleet-desk.py @@ -17,6 +17,8 @@ POOL]. Sends one launcher verb (`start` with its stdin) through the live session and prints the desktop's stdout and stderr and exits with its exit code. No live session: exit 3 at once, nothing sent. No answer in the verb's time (fork-fleet-desk-wire.py, "Time"): exit 75, the verb may or may not have run. + The same forced command asked `seeds` or `seed NAME` (SSH_ORIGINAL_COMMAND): a desktop's warm caches, not a + session (fork-fleet-desk-seed.py; its lock DIR/POOL.seed.lock). DIR is /run/fork-fleet-desk, made at boot by systemd-tmpfiles (runbook): owned by the desk-link user, its group the socket group (who may connect: the dispatcher's user, or root), mode 0710 or 0750. The socket is 0660 in that group, and serve lets in only root, its own user and members of that group (SO_PEERCRED). Not as root (tests), @@ -391,9 +393,25 @@ def launch(pool, verb, args, stdin, path, out=sys.stdout, err=sys.stderr, wait=N return got['exit'] +def seed_side(pool, asked, env): + """A desktop's seed session (fork-fleet-desk-seed.py): the same key and forced command, another command.""" + pool_ok(pool) + if os.geteuid() == 0: + raise Refused('serve never runs as root', 2) + path = sock_dir(env) + safe_dir(path, (os.geteuid(),), make=True) + spec = importlib.util.spec_from_file_location('fork_fleet_desk_seed', os.path.join(HERE, 'fork-fleet-desk-seed.py')) + seed = importlib.util.module_from_spec(spec) + spec.loader.exec_module(seed) + return seed.offer(pool, asked, path) + + def main(argv=None, env=os.environ): argv = sys.argv[1:] if argv is None else argv try: + if len(argv) == 2 and argv[0] == 'serve' and (env.get('SSH_ORIGINAL_COMMAND') == 'seeds' or + env.get('SSH_ORIGINAL_COMMAND', '').startswith('seed ')): + return seed_side(argv[1], env['SSH_ORIGINAL_COMMAND'], env) if len(argv) == 2 and argv[0] == 'serve': for sig in (signal.SIGTERM, signal.SIGHUP): signal.signal(sig, lambda *_: sys.exit(1)) # through serve's finally: the socket goes, clients learn diff --git a/.github/scripts/fork-fleet.py b/.github/scripts/fork-fleet.py index 95a1be08f69b6..bec1f34d7718d 100644 --- a/.github/scripts/fork-fleet.py +++ b/.github/scripts/fork-fleet.py @@ -139,6 +139,15 @@ def check_config(cfg): who = f'pool {p.get("name")!r}' if not (isinstance(p.get('name'), str) and POOL.fullmatch(p['name'])): out.append(f'{who}: name must be a-z, 0-9, _ and - (it names the dispatcher\'s lock file)') + seeds = p.get('seeds', []) # a desk pool's warm caches: fork-fleet-desk-seed.py + if not (isinstance(seeds, list) and all( + isinstance(x, dict) and isinstance(x.get('name'), str) and POOL.fullmatch(x['name']) + and all(positive(x.get(k)) for k in ('every_hours', 'max_age_hours', 'max_mb')) + and x['max_age_hours'] >= x['every_hours'] for x in seeds)): + out.append(f'{who}: seeds must list {{"name" (a-z, 0-9, _, -), "every_hours", "max_age_hours" (at least ' + 'every_hours), "max_mb"}, each a positive whole number') + elif len({x['name'] for x in seeds}) != len(seeds): + out.append(f'{who}: two seeds with one name') if not positive(p.get('slots')): out.append(f'{who}: slots must be a positive whole number') if not isinstance(p.get('waits'), bool): From 7026afd9ace39bf0d86a2b1e6f2b41fe1ee418d7 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:46:19 +0000 Subject: [PATCH 2/4] fork fleet: seed fix round 1 (D11) - seeds live in FILES/POOL/NAME.tar: pools never share a file - a file dated more than five minutes in the future is refused (exit 4) - the stream stops at the size that was checked; ctime and inode join the changed-file check - the forced command sends every command that starts with `seed` to the seed handler, so `seeds x` and `seed` exit 2 and never reach a session - a desktop that takes nothing for STALL seconds is cut off (exit 1) and the pool's lock let go - the lock file must be this user's; a clear refusal otherwise Co-Authored-By: Claude Opus 5.5 --- .github/fork-fleet.md | 19 +++-- .github/scripts/fork-fleet-desk-seed.py | 89 +++++++++++++------- .github/scripts/fork-fleet-desk-seed.test.py | 88 ++++++++++++++++--- .github/scripts/fork-fleet-desk.py | 9 +- 4 files changed, 153 insertions(+), 52 deletions(-) diff --git a/.github/fork-fleet.md b/.github/fork-fleet.md index 1a0b7c895f3a5..ca2414b079264 100644 --- a/.github/fork-fleet.md +++ b/.github/fork-fleet.md @@ -698,16 +698,21 @@ ready-made caches the desktop keeps and its jobs start from: least `every_hours`) the age, from when it was made, at which it is dropped, `max_mb` its largest size. Nothing in the fleet makes a seed: on the always-on host the project's own timer or job writes a tar of a cache folder's contents -to `/var/lib/fork-fleet/seed-files/NAME.tar`, by a temp file and a rename (a -file changed while it is sent is refused). That folder and its files are -root's (or the project's), readable by the desk-link user and writable by no -one else; never put a secret in a seed: every job of the pool reads it. +to `/var/lib/fork-fleet/seed-files/POOL/NAME.tar` (a folder per pool, so two +pools never share a file), by a temp file and a rename. A file changed while it +is sent (size, mtime or ctime), dated more than five minutes in the future, older +than `max_age_hours` or above `max_mb` is refused. Both folders and the files are +root's (or the project's), readable by the desk-link user and writable by no one +else (the owner itself is not checked, only that no one else may write); never +put a secret in a seed: every job of the pool reads it. The desktop asks for them with the desk link's key, in an ssh session of its -own: the same forced command answers `seeds` with the pool's list and +own: the same forced command sends every command that starts with `seed` to +`fork-fleet-desk-seed.py` (which step 3 installs; its header has the exit +codes), never to a session. It answers `seeds` with the pool's list and `seed NAME` with that one file, only a name of that list, and a trailer with its -size and SHA-256 (`fork-fleet-desk-seed.py`, which step 3 installs; its header -has the exit codes). +size and SHA-256; anything else is refused. One stream per pool at a time; a +desktop that takes nothing for two minutes is cut off and the next one may go. ## Owner steps on the repository diff --git a/.github/scripts/fork-fleet-desk-seed.py b/.github/scripts/fork-fleet-desk-seed.py index 2a68f886fc7df..8aebe9fbd05f3 100644 --- a/.github/scripts/fork-fleet-desk-seed.py +++ b/.github/scripts/fork-fleet-desk-seed.py @@ -4,20 +4,23 @@ A project lists the caches its desk pool's jobs may start from in the pool's `seeds` (fork-fleet.json): a name, how often a desktop takes a new one (every_hours), the age at which it is dropped (max_age_hours, from when it was made) and its largest size (max_mb). On the always-on host something of the project's own (a timer, a job) makes each one: -a tar of a folder's contents at FILES/NAME.tar, written to a temp file and renamed. Nothing here makes one. The -desktop pulls them with the desk link's key, in an ssh session of its own beside the agent's: +a tar of a folder's contents at FILES/POOL/NAME.tar (a folder per pool: pools never share a file), written to a temp +file and renamed. Nothing here makes one. The desktop pulls them with the desk link's key, in an ssh session of its +own beside the agent's: ssh ... seeds the forced command (fork-fleet-desk.py serve POOL, the desk-link user) sees it in SSH_ORIGINAL_COMMAND and prints the pool's list (one JSON object; `bytes` and `made` of each seed the host offers now, `why` for one it does not) - ssh ... seed NAME streams FILES/NAME.tar: only a name of that list, never a path the desktop names. The last + ssh ... seed NAME streams FILES/POOL/NAME.tar: only a name of that list, never a path the desktop names. The last stderr line is the trailer `seed NAME BYTES SHA256 MADE` (MADE: the file's mtime, epoch seconds) -The host side exits 2 for any other command, 3 while another stream for the pool runs, 4 when the file may not be -sent (missing, a link, in a folder or a file others may write, above max_mb, older than max_age_hours, or changed -while it was sent: made in place, not by rename). Standard library only. +The forced command sends every command that starts with `seed` here. The host side exits 2 for any other command, +3 while another stream for the pool runs, 4 when the file may not be sent (missing, a link, in a folder or a file +others may write, above max_mb, older than max_age_hours or dated in the future, or changed while it was sent: made +in place, not by rename), 1 when the desktop stops reading for STALL seconds (the pool's lock is then let go). +Standard library only. """ -import fcntl, hashlib, importlib.util, json, os, re, stat, sys, time +import fcntl, hashlib, importlib.util, json, os, signal, stat, sys, time sys.dont_write_bytecode = True HERE = os.path.dirname(os.path.realpath(__file__)) @@ -30,9 +33,9 @@ def module(name, file): return mod -FILES = '/var/lib/fork-fleet/seed-files' # the always-on host's: NAME.tar, made by the project +FILES = '/var/lib/fork-fleet/seed-files' # the always-on host's: POOL/NAME.tar, made by the project CONFIG = os.path.join(HERE, 'fork-fleet.json') -PIECE = 1 << 20 +PIECE, STALL, AHEAD = 1 << 20, 120, 300 # AHEAD: seconds a file's date may be ahead of now (a clock set back) ITEM = {'everyHours': 'every_hours', 'maxAgeHours': 'max_age_hours', 'maxMb': 'max_mb'} @@ -53,13 +56,14 @@ def pool_seeds(pool, config): raise Refused(f'pool {pool} is not in {config}', 2) -def offered(seed, files, now): - """An open fd and the stat of FILES/NAME.tar; Refused (4) when it may not be sent.""" - path = os.path.join(files, seed['name'] + '.tar') +def offered(seed, folder, now): + """An open fd and the stat of FOLDER/NAME.tar (FOLDER: FILES/POOL); Refused (4) when it may not be sent.""" + path = os.path.join(folder, seed['name'] + '.tar') try: - st = os.lstat(files) - if not stat.S_ISDIR(st.st_mode) or st.st_mode & 0o022: - raise Refused(f'{files} must be a folder (not a link) that only its owner may write', 4) + for d in (os.path.dirname(folder), folder): + st = os.lstat(d) + if not stat.S_ISDIR(st.st_mode) or st.st_mode & 0o022: + raise Refused(f'{d} must be a folder (not a link) that only its owner may write', 4) fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC) except OSError as e: raise Refused(f'{path}: {"not made yet" if isinstance(e, FileNotFoundError) else e.strerror}', 4) from None @@ -67,7 +71,8 @@ def offered(seed, files, now): hours = seed['max_age_hours'] why = ('not a plain file' if not stat.S_ISREG(st.st_mode) else 'others may write it' if st.st_mode & 0o022 else f'above max_mb ({seed["max_mb"]})' if st.st_size > seed['max_mb'] << 20 - else f'older than max_age_hours ({hours})' if now - st.st_mtime > hours * 3600 else None) + else f'older than max_age_hours ({hours})' if now - st.st_mtime > hours * 3600 + else 'dated in the future' if st.st_mtime > now + AHEAD else None) if why: os.close(fd) raise Refused(f'{path}: {why}', 4) @@ -79,13 +84,13 @@ def offer(pool, command, lock_dir, out=1, err=sys.stderr, config=CONFIG, files=F if os.geteuid() == 0: raise Refused('serve never runs as root', 2) word, _, name = command.partition(' ') - seeds = pool_seeds(pool, config) + seeds, folder = pool_seeds(pool, config), os.path.join(files, pool) if command == 'seeds': got = [] for s in seeds.values(): item = {'name': s['name'], **{k: s[v] for k, v in ITEM.items()}} try: - fd, st = offered(s, files, now()) + fd, st = offered(s, folder, now()) os.close(fd) item.update(bytes=st.st_size, made=int(st.st_mtime)) except Refused as e: @@ -95,21 +100,28 @@ def offer(pool, command, lock_dir, out=1, err=sys.stderr, config=CONFIG, files=F return 0 if word != 'seed' or name not in seeds: raise Refused(f'refused: {command[:60]!r} (only `seeds`, or `seed NAME` of pool {pool}\'s seeds)', 2) - lock = os.open(os.path.join(lock_dir, f'{pool}.seed.lock'), os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + lock_path = os.path.join(lock_dir, f'{pool}.seed.lock') try: + lock = os.open(lock_path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + except OSError as e: + raise Refused(f'{lock_path}: {e.strerror} (a leftover of another user? remove it)', 1) from None + try: + if os.fstat(lock).st_uid != os.geteuid(): + raise Refused(f'{lock_path} belongs to uid {os.fstat(lock).st_uid}, not this user: remove it', 1) try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError: raise Refused(f'another seed stream for pool {pool} runs', 3) from None - fd, st = offered(seeds[name], files, now()) + fd, st = offered(seeds[name], folder, now()) try: - sha, sent = send(fd, out) + sha, sent = send(fd, out, st.st_size) after = os.fstat(fd) finally: os.close(fd) finally: os.close(lock) - if sent != st.st_size or (after.st_size, after.st_mtime_ns) != (st.st_size, st.st_mtime_ns): + same = ('st_size', 'st_mtime_ns', 'st_ctime_ns', 'st_ino') + if sent != st.st_size or any(getattr(after, k) != getattr(st, k) for k in same): raise Refused(f'{name}: it changed while it was sent (made in place, not by rename?)', 4) print(f'seed {name} {sent} {sha} {int(st.st_mtime)}', file=err, flush=True) return 0 @@ -118,12 +130,31 @@ def offer(pool, command, lock_dir, out=1, err=sys.stderr, config=CONFIG, files=F return e.code -def send(fd, out): +class Stalled(Exception): + pass + + +def stalled(*_): + raise Stalled + + +def send(fd, out, size, stall=None): + """Sends at most SIZE bytes of FD (the size that was checked: a file that grows is cut there, and then fails the + changed-file check) to OUT; Refused (1) when OUT takes nothing for STALL seconds.""" + stall = STALL if stall is None else stall sha, sent = hashlib.sha256(), 0 - while data := os.read(fd, PIECE): - sha.update(data) - sent += len(data) - view = memoryview(data) - while view: - view = view[os.write(out, view):] + old = signal.signal(signal.SIGALRM, stalled) + try: + while sent < size and (data := os.read(fd, min(PIECE, size - sent))): + sha.update(data) + sent += len(data) + view = memoryview(data) + while view: + signal.alarm(stall) # each write that moves anything starts the wait again + view = view[os.write(out, view):] + except Stalled: + raise Refused(f'the desktop took nothing for {stall} seconds: stopped', 1) from None + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, old) return sha.hexdigest(), sent diff --git a/.github/scripts/fork-fleet-desk-seed.test.py b/.github/scripts/fork-fleet-desk-seed.test.py index 6fb4307c5a897..d6fcba4cd80e8 100644 --- a/.github/scripts/fork-fleet-desk-seed.test.py +++ b/.github/scripts/fork-fleet-desk-seed.test.py @@ -27,7 +27,7 @@ class Base(unittest.TestCase): def setUp(self): self.t = os.path.realpath(tempfile.mkdtemp(prefix='seed-test-')) self.addCleanup(shutil.rmtree, self.t, True) - for d in ('files', 'run', 'src'): + for d in ('files', f'files/{POOL}', 'run', 'src'): os.mkdir(os.path.join(self.t, d), 0o755) self.config(SEEDS) @@ -38,8 +38,8 @@ def config(self, seeds): json.dump(cfg, f) def make(self, name, members, mtime=None): - """FILES/NAME.tar of `members` ({path: bytes}), made as a project would: a temp file, then a rename.""" - tmp = os.path.join(self.t, 'files', f'.{name}.tmp') + """FILES/POOL/NAME.tar of `members` ({path: bytes}), made as a project would: a temp file, then a rename.""" + tmp = os.path.join(self.t, 'files', POOL, f'.{name}.tmp') with tarfile.open(tmp, 'w', format=tarfile.PAX_FORMAT) as tar: for path, data in members.items(): info = tarfile.TarInfo(path) @@ -48,10 +48,13 @@ def make(self, name, members, mtime=None): if mtime is not None: os.utime(tmp, (mtime, mtime)) os.chmod(tmp, 0o644) - os.replace(tmp, os.path.join(self.t, 'files', f'{name}.tar')) - with open(os.path.join(self.t, 'files', f'{name}.tar'), 'rb') as f: + os.replace(tmp, self.tar(name)) + with open(self.tar(name), 'rb') as f: return f.read() + def tar(self, name): + return os.path.join(self.t, 'files', POOL, f'{name}.tar') + def offer(self, command, **kw): r, w = os.pipe() err = io.StringIO() @@ -89,7 +92,7 @@ def test_the_list_says_what_is_offered_and_why_not(self): got = json.loads(out) self.assertEqual(got['v'], 1) m2, npm = got['seeds'] - st = os.stat(os.path.join(self.t, 'files', 'm2.tar')) + st = os.stat(self.tar('m2')) self.assertEqual(m2, {'name': 'm2', 'everyHours': 6, 'maxAgeHours': 1000, 'maxMb': 2, 'bytes': st.st_size, 'made': int(st.st_mtime)}) self.assertNotIn('made', npm) @@ -100,7 +103,7 @@ def test_a_seed_streams_whole_with_its_trailer(self): code, out, err = self.offer('seed m2') self.assertEqual(code, 0, err) self.assertEqual(out, data) - st = os.stat(os.path.join(self.t, 'files', 'm2.tar')) + st = os.stat(self.tar('m2')) self.assertEqual(err, f'seed m2 {len(data)} {hashlib.sha256(data).hexdigest()} {int(st.st_mtime)}\n') def test_other_commands_and_names_are_refused_and_send_nothing(self): @@ -112,13 +115,15 @@ def test_other_commands_and_names_are_refused_and_send_nothing(self): self.assertEqual((code, out), (2, b''), err) def test_a_file_that_may_not_be_sent_is_exit_4(self): - files = os.path.join(self.t, 'files') + top, files = os.path.join(self.t, 'files'), os.path.join(self.t, 'files', POOL) now = time.time() cases = {'missing': lambda: None, 'a link': lambda: os.symlink(os.path.join(self.t, 'src', 'x.tar'), f'{files}/m2.tar'), 'group-writable': lambda: (self.make('m2', {'a': b'x'}), os.chmod(f'{files}/m2.tar', 0o664)), 'too big': lambda: self.make('m2', {'a': os.urandom(3 << 20)}), 'too old': lambda: self.make('m2', {'a': b'x'}, mtime=now - 1001 * 3600), + 'dated in the future': lambda: self.make('m2', {'a': b'x'}, mtime=now + 10 * 365 * 86400), + 'the files folder others write': lambda: (self.make('m2', {'a': b'x'}), os.chmod(top, 0o777)), 'a folder': lambda: os.mkdir(f'{files}/m2.tar'), 'folder others write': lambda: (self.make('m2', {'a': b'x'}), os.chmod(files, 0o777))} with open(os.path.join(self.t, 'src', 'x.tar'), 'wb') as f: @@ -130,6 +135,7 @@ def test_a_file_that_may_not_be_sent_is_exit_4(self): code, out, err = self.offer('seed m2') self.assertEqual((code, out), (4, b''), err) os.chmod(files, 0o755) + os.chmod(top, 0o755) p = f'{files}/m2.tar' if os.path.isdir(p) and not os.path.islink(p): os.rmdir(p) @@ -145,17 +151,69 @@ def test_a_second_stream_for_the_pool_is_exit_3(self): def test_a_file_changed_while_sent_is_exit_4(self): self.make('m2', {'a': b'x' * 3000}) - path, send = os.path.join(self.t, 'files', 'm2.tar'), seed.send + path, send = self.tar('m2'), seed.send - def changing(fd, out): + def changing(fd, out, size): with open(path, 'ab') as f: # made in place, not by rename f.write(b'\0' * 512) - return send(fd, out) + return send(fd, out, size) with mock.patch.object(seed, 'send', changing): + code, out, err = self.offer('seed m2') + self.assertEqual(code, 4) + self.assertIn('changed while it was sent', err) + self.assertEqual(len(out), os.stat(path).st_size - 512) # cut at the size that was checked + + def test_a_same_size_rewrite_with_its_old_date_put_back_is_exit_4(self): + self.make('m2', {'a': b'x' * 3000}) + path, send = self.tar('m2'), seed.send + + def rewriting(fd, out, size): + st = os.stat(path) + with open(path, 'r+b') as f: # in place, same size, the old mtime put back: only ctime tells + f.write(b'y' * 512) + os.utime(path, ns=(st.st_atime_ns, st.st_mtime_ns)) + return send(fd, out, size) + with mock.patch.object(seed, 'send', rewriting): code, _, err = self.offer('seed m2') self.assertEqual(code, 4) self.assertIn('changed while it was sent', err) + def test_a_desktop_that_stops_reading_lets_the_lock_go(self): + self.make('m2', {'a': os.urandom(1 << 20)}) + r, w = os.pipe() # nobody reads r: the pipe fills and the write waits + err = io.StringIO() + try: + with mock.patch.object(seed, 'STALL', 1): + t = time.monotonic() + code = seed.offer(POOL, 'seed m2', os.path.join(self.t, 'run'), out=w, err=err, + config=os.path.join(self.t, 'fork-fleet.json'), files=os.path.join(self.t, 'files')) + self.assertLess(time.monotonic() - t, 30) + finally: + os.close(r) + os.close(w) + self.assertEqual(code, 1) + self.assertIn('took nothing for 1 seconds', err.getvalue()) + self.assertEqual(self.offer('seed m2')[0], 0) # the pool's lock was let go + + def test_pools_never_share_a_seed_file(self): + cfg = json.loads(json.dumps(CONFIG)) + cfg['pools'][0]['seeds'] = SEEDS + cfg['pools'].append({**cfg['pools'][0], 'name': 'other-pool'}) + with open(os.path.join(self.t, 'fork-fleet.json'), 'w', encoding='utf-8') as f: + json.dump(cfg, f) + self.make('m2', {'a': b'x'}) + os.mkdir(os.path.join(self.t, 'files', 'other-pool'), 0o755) + r, w = os.pipe() + err = io.StringIO() + try: + code = seed.offer('other-pool', 'seed m2', os.path.join(self.t, 'run'), out=w, err=err, + config=os.path.join(self.t, 'fork-fleet.json'), files=os.path.join(self.t, 'files')) + finally: + os.close(w) + os.close(r) + self.assertEqual(code, 4) + self.assertIn('not made yet', err.getvalue()) + def test_never_as_root_and_a_bad_config_is_exit_2(self): with mock.patch.object(seed.os, 'geteuid', return_value=0): self.assertEqual(self.offer('seeds')[:2], (2, b'')) @@ -180,6 +238,14 @@ def test_the_forced_command_diverts_seed_requests(self): stdin=subprocess.DEVNULL) self.assertEqual((got.returncode, got.stdout), (2, '')) self.assertFalse(os.path.exists(os.path.join(self.t, 'run2', f'{POOL}.sock'))) # no session was served + for command in ('seeds x', 'seed', 'seedsx', 'seeds '): + with self.subTest(command): + got = subprocess.run([sys.executable, '-I', '-B', desk, 'serve', POOL], capture_output=True, + text=True, env={**base, 'SSH_ORIGINAL_COMMAND': command}, timeout=30, + stdin=subprocess.DEVNULL) + self.assertEqual((got.returncode, got.stdout), (2, ''), got.stderr) + self.assertIn('fork-fleet-desk seed: refused', got.stderr) + self.assertFalse(os.path.exists(os.path.join(self.t, 'run2', f'{POOL}.sock'))) if __name__ == '__main__': diff --git a/.github/scripts/fork-fleet-desk.py b/.github/scripts/fork-fleet-desk.py index eb08f20058a57..fd8ff12cb13d9 100755 --- a/.github/scripts/fork-fleet-desk.py +++ b/.github/scripts/fork-fleet-desk.py @@ -17,8 +17,8 @@ POOL]. Sends one launcher verb (`start` with its stdin) through the live session and prints the desktop's stdout and stderr and exits with its exit code. No live session: exit 3 at once, nothing sent. No answer in the verb's time (fork-fleet-desk-wire.py, "Time"): exit 75, the verb may or may not have run. - The same forced command asked `seeds` or `seed NAME` (SSH_ORIGINAL_COMMAND): a desktop's warm caches, not a - session (fork-fleet-desk-seed.py; its lock DIR/POOL.seed.lock). + The same forced command asked anything that starts with `seed` (SSH_ORIGINAL_COMMAND; `seeds`, `seed NAME`): a + desktop's warm caches, never a session (fork-fleet-desk-seed.py; its lock DIR/POOL.seed.lock). DIR is /run/fork-fleet-desk, made at boot by systemd-tmpfiles (runbook): owned by the desk-link user, its group the socket group (who may connect: the dispatcher's user, or root), mode 0710 or 0750. The socket is 0660 in that group, and serve lets in only root, its own user and members of that group (SO_PEERCRED). Not as root (tests), @@ -409,9 +409,8 @@ def seed_side(pool, asked, env): def main(argv=None, env=os.environ): argv = sys.argv[1:] if argv is None else argv try: - if len(argv) == 2 and argv[0] == 'serve' and (env.get('SSH_ORIGINAL_COMMAND') == 'seeds' or - env.get('SSH_ORIGINAL_COMMAND', '').startswith('seed ')): - return seed_side(argv[1], env['SSH_ORIGINAL_COMMAND'], env) + if len(argv) == 2 and argv[0] == 'serve' and env.get('SSH_ORIGINAL_COMMAND', '').startswith('seed'): + return seed_side(argv[1], env['SSH_ORIGINAL_COMMAND'], env) # every seed* command; it refuses the rest if len(argv) == 2 and argv[0] == 'serve': for sig in (signal.SIGTERM, signal.SIGHUP): signal.signal(sig, lambda *_: sys.exit(1)) # through serve's finally: the socket goes, clients learn From 099f4387de19348368a29818358a025ebeb07cf8 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sun, 15 Nov 2026 21:31:58 +0000 Subject: [PATCH 3/4] fork fleet: seed tests read the small stream and use a valid second pool (D11) The lock check after a stall sent 1 MB into a pipe nobody read until the call returned, and the second pool reused the first one's label and beacon, which the config check refuses. The pool test also puts a tar at the old shared path, so the old code fails it. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-fleet-desk-seed.test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/scripts/fork-fleet-desk-seed.test.py b/.github/scripts/fork-fleet-desk-seed.test.py index d6fcba4cd80e8..c237f4173a99e 100644 --- a/.github/scripts/fork-fleet-desk-seed.test.py +++ b/.github/scripts/fork-fleet-desk-seed.test.py @@ -193,15 +193,19 @@ def test_a_desktop_that_stops_reading_lets_the_lock_go(self): os.close(w) self.assertEqual(code, 1) self.assertIn('took nothing for 1 seconds', err.getvalue()) + self.make('m2', {'a': b'x'}) # small enough for the pipe: offer() below has no reader until it returns self.assertEqual(self.offer('seed m2')[0], 0) # the pool's lock was let go def test_pools_never_share_a_seed_file(self): cfg = json.loads(json.dumps(CONFIG)) cfg['pools'][0]['seeds'] = SEEDS - cfg['pools'].append({**cfg['pools'][0], 'name': 'other-pool'}) + cfg['pools'].append({**cfg['pools'][0], 'name': 'other-pool', 'label': 'fleet-other-t3code', + 'beacon': 'FLEET_BEACON_OTHER'}) + self.assertEqual(ff.check_config(cfg), []) with open(os.path.join(self.t, 'fork-fleet.json'), 'w', encoding='utf-8') as f: json.dump(cfg, f) self.make('m2', {'a': b'x'}) + shutil.copy(self.tar('m2'), os.path.join(self.t, 'files', 'm2.tar')) # one file for all pools, as before os.mkdir(os.path.join(self.t, 'files', 'other-pool'), 0o755) r, w = os.pipe() err = io.StringIO() From 3fb86174250b9b0502709133371b60a09ab04b63 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:53:05 +0000 Subject: [PATCH 4/4] fork fleet: seed fix round 2 (rename while sent, deadline, max_mb cap) (D11) - A seed replaced by rename while it streams is sent whole: when the open file has no links left (st_nlink 0) it was replaced, not rewritten, so its ctime is not compared; size and mtime still are. The st_ino compare (fstat of the same fd, it never differed) is gone. - The stall wait restarts per 64 KiB piece written, not per 1 MiB, and a stream has an overall deadline: 600 s plus one second per 256 KiB. - max_mb is at most 16384 (fork-fleet.py SEED_MAX_MB). - Tests: the lens rename test, the lock-owner refusal, every refused file must say its own reason (the future-dated case now fails on round-0's path), the deadline, and a slow but steady reader that is kept. Co-Authored-By: Claude Opus 5.5 --- .github/fork-fleet.md | 10 ++- .github/scripts/fork-fleet-desk-seed.py | 29 +++++-- .github/scripts/fork-fleet-desk-seed.test.py | 82 +++++++++++++++++--- .github/scripts/fork-fleet.py | 5 +- 4 files changed, 102 insertions(+), 24 deletions(-) diff --git a/.github/fork-fleet.md b/.github/fork-fleet.md index ca2414b079264..bc28a77eaa928 100644 --- a/.github/fork-fleet.md +++ b/.github/fork-fleet.md @@ -696,11 +696,12 @@ ready-made caches the desktop keeps and its jobs start from: `every_hours` is how often a desktop takes a new one, `max_age_hours` (at least `every_hours`) the age, from when it was made, at which it is dropped, -`max_mb` its largest size. Nothing in the fleet makes a seed: on the always-on +`max_mb` its largest size (at most 16384). Nothing in the fleet makes a seed: on the always-on host the project's own timer or job writes a tar of a cache folder's contents to `/var/lib/fork-fleet/seed-files/POOL/NAME.tar` (a folder per pool, so two -pools never share a file), by a temp file and a rename. A file changed while it -is sent (size, mtime or ctime), dated more than five minutes in the future, older +pools never share a file), by a temp file and a rename. A file changed in place +while it is sent (size, mtime or ctime) is refused; one replaced by a rename while +it is sent is fine (the old one is sent whole). A file dated more than five minutes in the future, older than `max_age_hours` or above `max_mb` is refused. Both folders and the files are root's (or the project's), readable by the desk-link user and writable by no one else (the owner itself is not checked, only that no one else may write); never @@ -712,7 +713,8 @@ own: the same forced command sends every command that starts with `seed` to codes), never to a session. It answers `seeds` with the pool's list and `seed NAME` with that one file, only a name of that list, and a trailer with its size and SHA-256; anything else is refused. One stream per pool at a time; a -desktop that takes nothing for two minutes is cut off and the next one may go. +desktop that takes less than 64 KiB in two minutes, or not the whole file within +ten minutes plus one second per 256 KiB, is cut off and the next one may go. ## Owner steps on the repository diff --git a/.github/scripts/fork-fleet-desk-seed.py b/.github/scripts/fork-fleet-desk-seed.py index 8aebe9fbd05f3..b24b98ea9fed3 100644 --- a/.github/scripts/fork-fleet-desk-seed.py +++ b/.github/scripts/fork-fleet-desk-seed.py @@ -17,10 +17,12 @@ The forced command sends every command that starts with `seed` here. The host side exits 2 for any other command, 3 while another stream for the pool runs, 4 when the file may not be sent (missing, a link, in a folder or a file others may write, above max_mb, older than max_age_hours or dated in the future, or changed while it was sent: made -in place, not by rename), 1 when the desktop stops reading for STALL seconds (the pool's lock is then let go). +in place, not by rename; a file replaced by rename while it is sent is fine, the old one is sent whole), 1 when the +desktop takes less than WRITE bytes in STALL seconds or the whole file not by its deadline (GRACE seconds plus +MIN_RATE bytes a second): the pool's lock is then let go. Standard library only. """ -import fcntl, hashlib, importlib.util, json, os, signal, stat, sys, time +import fcntl, hashlib, importlib.util, json, math, os, signal, stat, sys, time sys.dont_write_bytecode = True HERE = os.path.dirname(os.path.realpath(__file__)) @@ -35,7 +37,8 @@ def module(name, file): FILES = '/var/lib/fork-fleet/seed-files' # the always-on host's: POOL/NAME.tar, made by the project CONFIG = os.path.join(HERE, 'fork-fleet.json') -PIECE, STALL, AHEAD = 1 << 20, 120, 300 # AHEAD: seconds a file's date may be ahead of now (a clock set back) +PIECE, WRITE, STALL, AHEAD = 1 << 20, 64 << 10, 120, 300 # AHEAD: seconds a file's date may be ahead of now +GRACE, MIN_RATE = 600, 256 << 10 # a stream's deadline: GRACE seconds plus one second per MIN_RATE bytes ITEM = {'everyHours': 'every_hours', 'maxAgeHours': 'max_age_hours', 'maxMb': 'max_mb'} @@ -120,7 +123,9 @@ def offer(pool, command, lock_dir, out=1, err=sys.stderr, config=CONFIG, files=F os.close(fd) finally: os.close(lock) - same = ('st_size', 'st_mtime_ns', 'st_ctime_ns', 'st_ino') + # A rename over NAME.tar unlinks the open file (st_nlink 0), which changes its ctime but not its bytes: + # only a file still in place must keep its ctime (a rewrite with the old mtime put back). + same = ('st_size', 'st_mtime_ns') + (('st_ctime_ns',) if after.st_nlink else ()) if sent != st.st_size or any(getattr(after, k) != getattr(st, k) for k in same): raise Refused(f'{name}: it changed while it was sent (made in place, not by rename?)', 4) print(f'seed {name} {sent} {sha} {int(st.st_mtime)}', file=err, flush=True) @@ -140,9 +145,11 @@ def stalled(*_): def send(fd, out, size, stall=None): """Sends at most SIZE bytes of FD (the size that was checked: a file that grows is cut there, and then fails the - changed-file check) to OUT; Refused (1) when OUT takes nothing for STALL seconds.""" + changed-file check) to OUT; Refused (1) when OUT takes less than WRITE bytes in STALL seconds, or not all of it + within GRACE seconds plus SIZE / MIN_RATE.""" stall = STALL if stall is None else stall sha, sent = hashlib.sha256(), 0 + deadline = time.monotonic() + GRACE + size / MIN_RATE old = signal.signal(signal.SIGALRM, stalled) try: while sent < size and (data := os.read(fd, min(PIECE, size - sent))): @@ -150,10 +157,16 @@ def send(fd, out, size, stall=None): sent += len(data) view = memoryview(data) while view: - signal.alarm(stall) # each write that moves anything starts the wait again - view = view[os.write(out, view):] + left = deadline - time.monotonic() + if left <= 0: + raise Stalled + signal.alarm(max(1, min(stall, math.ceil(left)))) # each piece that moves starts the wait again + view = view[os.write(out, view[:WRITE]):] except Stalled: - raise Refused(f'the desktop took nothing for {stall} seconds: stopped', 1) from None + if time.monotonic() >= deadline: + raise Refused(f'the desktop did not take {size} bytes within {GRACE + size / MIN_RATE:.0f} seconds ' + f'(at least {MIN_RATE >> 10} KiB a second): stopped', 1) from None + raise Refused(f'the desktop took less than {WRITE >> 10} KiB in {stall} seconds: stopped', 1) from None finally: signal.alarm(0) signal.signal(signal.SIGALRM, old) diff --git a/.github/scripts/fork-fleet-desk-seed.test.py b/.github/scripts/fork-fleet-desk-seed.test.py index c237f4173a99e..8171f0d8ca14e 100644 --- a/.github/scripts/fork-fleet-desk-seed.test.py +++ b/.github/scripts/fork-fleet-desk-seed.test.py @@ -2,7 +2,8 @@ """Tests of fork-fleet-desk-seed.py (a desk pool's warm caches) and the `seeds` check of fork-fleet.json: the host's offer in-process and the forced command's divert in fork-fleet-desk.py. Temp folders only; no root, no ssh, no network. Run: python3 -I -B . Linux only.""" -import fcntl, hashlib, importlib.util, io, json, os, shutil, subprocess, sys, tarfile, tempfile, time, unittest +import fcntl, hashlib, importlib.util, io, json, os, shutil, subprocess, sys, tarfile, tempfile, threading, time +import unittest from unittest import mock sys.dont_write_bytecode = True @@ -76,9 +77,11 @@ def problems(seeds): return [p for p in ff.check_config(cfg) if 'seed' in p] self.assertEqual(problems(SEEDS), []) self.assertEqual(problems([]), []) + self.assertEqual(problems([{**SEEDS[0], 'max_mb': ff.SEED_MAX_MB}]), []) good = SEEDS[0] for bad in ({}, [{**good, 'name': 'M2'}], [{**good, 'name': '../x'}], [{**good, 'every_hours': 0}], [{**good, 'max_mb': True}], [{**good, 'max_age_hours': 5}], [{**good, 'max_mb': '2'}], + [{**good, 'max_mb': ff.SEED_MAX_MB + 1}], [{k: v for k, v in good.items() if k != 'max_mb'}], ['m2'], [good, good]): with self.subTest(bad): self.assertEqual(len(problems(bad)), 1, bad) @@ -117,15 +120,16 @@ def test_other_commands_and_names_are_refused_and_send_nothing(self): def test_a_file_that_may_not_be_sent_is_exit_4(self): top, files = os.path.join(self.t, 'files'), os.path.join(self.t, 'files', POOL) now = time.time() - cases = {'missing': lambda: None, - 'a link': lambda: os.symlink(os.path.join(self.t, 'src', 'x.tar'), f'{files}/m2.tar'), - 'group-writable': lambda: (self.make('m2', {'a': b'x'}), os.chmod(f'{files}/m2.tar', 0o664)), - 'too big': lambda: self.make('m2', {'a': os.urandom(3 << 20)}), - 'too old': lambda: self.make('m2', {'a': b'x'}, mtime=now - 1001 * 3600), + cases = {'not made yet': lambda: None, + 'Too many levels of symbolic links': lambda: os.symlink(os.path.join(self.t, 'src', 'x.tar'), + f'{files}/m2.tar'), + 'others may write it': lambda: (self.make('m2', {'a': b'x'}), os.chmod(f'{files}/m2.tar', 0o664)), + 'above max_mb': lambda: self.make('m2', {'a': os.urandom(3 << 20)}), + 'older than max_age_hours': lambda: self.make('m2', {'a': b'x'}, mtime=now - 1001 * 3600), 'dated in the future': lambda: self.make('m2', {'a': b'x'}, mtime=now + 10 * 365 * 86400), - 'the files folder others write': lambda: (self.make('m2', {'a': b'x'}), os.chmod(top, 0o777)), - 'a folder': lambda: os.mkdir(f'{files}/m2.tar'), - 'folder others write': lambda: (self.make('m2', {'a': b'x'}), os.chmod(files, 0o777))} + f'{top} must be a folder': lambda: (self.make('m2', {'a': b'x'}), os.chmod(top, 0o777)), + 'not a plain file': lambda: os.mkdir(f'{files}/m2.tar'), + f'{files} must be a folder': lambda: (self.make('m2', {'a': b'x'}), os.chmod(files, 0o777))} with open(os.path.join(self.t, 'src', 'x.tar'), 'wb') as f: f.write(b'x') os.chmod(os.path.join(self.t, 'src', 'x.tar'), 0o644) # only the link itself may be the reason @@ -134,6 +138,7 @@ def test_a_file_that_may_not_be_sent_is_exit_4(self): make() code, out, err = self.offer('seed m2') self.assertEqual((code, out), (4, b''), err) + self.assertIn(name, err) # refused for this reason, read where the pool's files are os.chmod(files, 0o755) os.chmod(top, 0o755) p = f'{files}/m2.tar' @@ -163,6 +168,25 @@ def changing(fd, out, size): self.assertIn('changed while it was sent', err) self.assertEqual(len(out), os.stat(path).st_size - 512) # cut at the size that was checked + def test_a_rename_while_sent_is_fine(self): + old, send = self.make('m2', {'a': b'x' * 3000}), seed.send + + def renaming(fd, out, size): + self.make('m2', {'a': b'y' * 3000}) # the documented refresh: a temp file, then a rename + return send(fd, out, size) + with mock.patch.object(seed, 'send', renaming): + code, out, err = self.offer('seed m2') + self.assertEqual((code, out), (0, old), err) + self.assertTrue(err.startswith(f'seed m2 {len(old)} {hashlib.sha256(old).hexdigest()} '), err) + + def test_a_lock_of_another_user_is_refused(self): + self.make('m2', {'a': b'x'}) + open(os.path.join(self.t, 'run', f'{POOL}.seed.lock'), 'a').close() + with mock.patch.object(seed.os, 'geteuid', return_value=os.geteuid() + 4242): + code, out, err = self.offer('seed m2') + self.assertEqual((code, out), (1, b''), err) + self.assertIn(f'belongs to uid {os.geteuid()}', err) + def test_a_same_size_rewrite_with_its_old_date_put_back_is_exit_4(self): self.make('m2', {'a': b'x' * 3000}) path, send = self.tar('m2'), seed.send @@ -192,10 +216,48 @@ def test_a_desktop_that_stops_reading_lets_the_lock_go(self): os.close(r) os.close(w) self.assertEqual(code, 1) - self.assertIn('took nothing for 1 seconds', err.getvalue()) + self.assertIn('took less than 64 KiB in 1 seconds', err.getvalue()) self.make('m2', {'a': b'x'}) # small enough for the pipe: offer() below has no reader until it returns self.assertEqual(self.offer('seed m2')[0], 0) # the pool's lock was let go + def test_a_slow_desktop_meets_the_deadline(self): + self.make('m2', {'a': os.urandom(1 << 20)}) + r, w = os.pipe() # nobody reads r; the stall wait alone (100 s) would hold the lock much longer + err = io.StringIO() + try: + with mock.patch.multiple(seed, STALL=100, GRACE=1, MIN_RATE=1 << 40): + t = time.monotonic() + code = seed.offer(POOL, 'seed m2', os.path.join(self.t, 'run'), out=w, err=err, + config=os.path.join(self.t, 'fork-fleet.json'), files=os.path.join(self.t, 'files')) + self.assertLess(time.monotonic() - t, 30) + finally: + os.close(r) + os.close(w) + self.assertEqual(code, 1, err.getvalue()) + self.assertIn('within 1 seconds', err.getvalue()) + + def test_a_desktop_reading_slowly_but_steadily_is_not_dropped(self): + data = self.make('m2', {'a': os.urandom(300 << 10)}) + r, w = os.pipe() + got = bytearray() + + def reader(): + with os.fdopen(r, 'rb', buffering=0) as f: + while chunk := f.read(16 << 10): # 16 KiB, then a pause: under a stall wait per 64 KiB piece + got.extend(chunk) + time.sleep(0.3) + th = threading.Thread(target=reader) + th.start() + err = io.StringIO() + try: + with mock.patch.object(seed, 'STALL', 2): + code = seed.offer(POOL, 'seed m2', os.path.join(self.t, 'run'), out=w, err=err, + config=os.path.join(self.t, 'fork-fleet.json'), files=os.path.join(self.t, 'files')) + finally: + os.close(w) + th.join(60) + self.assertEqual((code, bytes(got)), (0, data), err.getvalue()) + def test_pools_never_share_a_seed_file(self): cfg = json.loads(json.dumps(CONFIG)) cfg['pools'][0]['seeds'] = SEEDS diff --git a/.github/scripts/fork-fleet.py b/.github/scripts/fork-fleet.py index bec1f34d7718d..2581a950e3cfb 100644 --- a/.github/scripts/fork-fleet.py +++ b/.github/scripts/fork-fleet.py @@ -55,6 +55,7 @@ VARIABLE = re.compile(r'(?!GITHUB_)[A-Z][A-Z0-9_]{0,99}') JOB_ID = re.compile(r'[a-z0-9_]{1,40}') POOL = re.compile(r'[a-z0-9][a-z0-9_-]{0,39}') +SEED_MAX_MB = 16384 # a seed's largest max_mb: 16 GiB KINDS = ('push', 'draft', 'ready') # a pool's `takes`: pushes, draft pull requests, pull requests ready for review # Events whose code and workflow are this repository's own (a pull request event: only when head and base are too). OWN_EVENTS = ('push', 'workflow_dispatch', 'schedule') @@ -143,9 +144,9 @@ def check_config(cfg): if not (isinstance(seeds, list) and all( isinstance(x, dict) and isinstance(x.get('name'), str) and POOL.fullmatch(x['name']) and all(positive(x.get(k)) for k in ('every_hours', 'max_age_hours', 'max_mb')) - and x['max_age_hours'] >= x['every_hours'] for x in seeds)): + and x['max_age_hours'] >= x['every_hours'] and x['max_mb'] <= SEED_MAX_MB for x in seeds)): out.append(f'{who}: seeds must list {{"name" (a-z, 0-9, _, -), "every_hours", "max_age_hours" (at least ' - 'every_hours), "max_mb"}, each a positive whole number') + f'every_hours), "max_mb" (at most {SEED_MAX_MB})}}, each a positive whole number') elif len({x['name'] for x in seeds}) != len(seeds): out.append(f'{who}: two seeds with one name') if not positive(p.get('slots')):