diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index ccedb1f29c7de..e4cebd33243c1 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -9,6 +9,7 @@ .github/scripts/fork-contracts.test.py .github/scripts/fork-fleet-desk-agent.test.py .github/scripts/fork-fleet-desk-key.test.py +.github/scripts/fork-fleet-desk-seed.test.py .github/scripts/fork-fleet-desk.test.py .github/scripts/fork-fleet.test.py .github/scripts/fork-fleet-refresh.test.py diff --git a/.github/fork-fleet.md b/.github/fork-fleet.md index a42c9379baa9f..bc28a77eaa928 100644 --- a/.github/fork-fleet.md +++ b/.github/fork-fleet.md @@ -448,6 +448,7 @@ done files="fork-fleet.json:fork-fleet.json:644 scripts/fork-fleet.py:fork-fleet.py:644 scripts/fork-fleet-hook.sh:hook.sh:755 scripts/fork-fleet-launch.sh:fork-fleet-launch.sh:755 scripts/fork-fleet-desk.py:fork-fleet-desk.py:644 scripts/fork-fleet-desk-wire.py:fork-fleet-desk-wire.py:644 + scripts/fork-fleet-desk-seed.py:fork-fleet-desk-seed.py:644 scripts/fork-fleet-refresh.service:fork-fleet-refresh.service:644 scripts/fork-fleet-refresh.timer:fork-fleet-refresh.timer:644 scripts/fork-fleet-job-paths.txt:fork-fleet-job-paths.txt:644" @@ -683,6 +684,38 @@ Bun, Node and Rust are installed per job by the setup actions, into the job's own home, which goes with its copy. Leave a few tens of GB of disk for `/var/lib/fork-fleet/runners`. +#### Seeds (warm caches for a desk pool) + +A job's copy starts empty, so each job would download its tools and +dependencies again. A desk pool may list **seeds** in its fleet config entry, +ready-made caches the desktop keeps and its jobs start from: + +```json +"seeds": [{"name": "gradle", "every_hours": 6, "max_age_hours": 72, "max_mb": 4096}] +``` + +`every_hours` is how often a desktop takes a new one, `max_age_hours` (at +least `every_hours`) the age, from when it was made, at which it is dropped, +`max_mb` its largest size (at most 16384). Nothing in the fleet makes a seed: on the always-on +host the project's own timer or job writes a tar of a cache folder's contents +to `/var/lib/fork-fleet/seed-files/POOL/NAME.tar` (a folder per pool, so two +pools never share a file), by a temp file and a rename. A file changed in place +while it is sent (size, mtime or ctime) is refused; one replaced by a rename while +it is sent is fine (the old one is sent whole). A file dated more than five minutes in the future, older +than `max_age_hours` or above `max_mb` is refused. Both folders and the files are +root's (or the project's), readable by the desk-link user and writable by no one +else (the owner itself is not checked, only that no one else may write); never +put a secret in a seed: every job of the pool reads it. + +The desktop asks for them with the desk link's key, in an ssh session of its +own: the same forced command sends every command that starts with `seed` to +`fork-fleet-desk-seed.py` (which step 3 installs; its header has the exit +codes), never to a session. It answers `seeds` with the pool's list and +`seed NAME` with that one file, only a name of that list, and a trailer with its +size and SHA-256; anything else is refused. One stream per pool at a time; a +desktop that takes less than 64 KiB in two minutes, or not the whole file within +ten minutes plus one second per 256 KiB, is cut off and the next one may go. + ## Owner steps on the repository Run these from a machine where `gh` is logged in as the owner. None of them diff --git a/.github/scripts/fork-fleet-desk-seed.py b/.github/scripts/fork-fleet-desk-seed.py new file mode 100644 index 0000000000000..b24b98ea9fed3 --- /dev/null +++ b/.github/scripts/fork-fleet-desk-seed.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Fork only: warm caches ("seeds") for a desk pool's jobs (.github/fork-fleet.md, "Seeds"). + +A project lists the caches its desk pool's jobs may start from in the pool's `seeds` (fork-fleet.json): a name, how +often a desktop takes a new one (every_hours), the age at which it is dropped (max_age_hours, from when it was made) +and its largest size (max_mb). On the always-on host something of the project's own (a timer, a job) makes each one: +a tar of a folder's contents at FILES/POOL/NAME.tar (a folder per pool: pools never share a file), written to a temp +file and renamed. Nothing here makes one. The desktop pulls them with the desk link's key, in an ssh session of its +own beside the agent's: + + ssh ... seeds the forced command (fork-fleet-desk.py serve POOL, the desk-link user) sees it in + SSH_ORIGINAL_COMMAND and prints the pool's list (one JSON object; `bytes` and `made` of each + seed the host offers now, `why` for one it does not) + ssh ... seed NAME streams FILES/POOL/NAME.tar: only a name of that list, never a path the desktop names. The last + stderr line is the trailer `seed NAME BYTES SHA256 MADE` (MADE: the file's mtime, epoch seconds) + +The forced command sends every command that starts with `seed` here. The host side exits 2 for any other command, +3 while another stream for the pool runs, 4 when the file may not be sent (missing, a link, in a folder or a file +others may write, above max_mb, older than max_age_hours or dated in the future, or changed while it was sent: made +in place, not by rename; a file replaced by rename while it is sent is fine, the old one is sent whole), 1 when the +desktop takes less than WRITE bytes in STALL seconds or the whole file not by its deadline (GRACE seconds plus +MIN_RATE bytes a second): the pool's lock is then let go. +Standard library only. +""" +import fcntl, hashlib, importlib.util, json, math, os, signal, stat, sys, time + +sys.dont_write_bytecode = True +HERE = os.path.dirname(os.path.realpath(__file__)) + + +def module(name, file): + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, file)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +FILES = '/var/lib/fork-fleet/seed-files' # the always-on host's: POOL/NAME.tar, made by the project +CONFIG = os.path.join(HERE, 'fork-fleet.json') +PIECE, WRITE, STALL, AHEAD = 1 << 20, 64 << 10, 120, 300 # AHEAD: seconds a file's date may be ahead of now +GRACE, MIN_RATE = 600, 256 << 10 # a stream's deadline: GRACE seconds plus one second per MIN_RATE bytes +ITEM = {'everyHours': 'every_hours', 'maxAgeHours': 'max_age_hours', 'maxMb': 'max_mb'} + + +class Refused(Exception): + def __init__(self, msg, code): + super().__init__(msg) + self.code = code + + +def pool_seeds(pool, config): + try: + cfg = module('fork_fleet', 'fork-fleet.py').load(config) + except (OSError, ValueError) as e: + raise Refused(f'the fleet config: {e}', 2) from None + for p in cfg['pools']: + if p['name'] == pool: + return {s['name']: s for s in p.get('seeds', [])} + raise Refused(f'pool {pool} is not in {config}', 2) + + +def offered(seed, folder, now): + """An open fd and the stat of FOLDER/NAME.tar (FOLDER: FILES/POOL); Refused (4) when it may not be sent.""" + path = os.path.join(folder, seed['name'] + '.tar') + try: + for d in (os.path.dirname(folder), folder): + st = os.lstat(d) + if not stat.S_ISDIR(st.st_mode) or st.st_mode & 0o022: + raise Refused(f'{d} must be a folder (not a link) that only its owner may write', 4) + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC) + except OSError as e: + raise Refused(f'{path}: {"not made yet" if isinstance(e, FileNotFoundError) else e.strerror}', 4) from None + st = os.fstat(fd) + hours = seed['max_age_hours'] + why = ('not a plain file' if not stat.S_ISREG(st.st_mode) else 'others may write it' if st.st_mode & 0o022 + else f'above max_mb ({seed["max_mb"]})' if st.st_size > seed['max_mb'] << 20 + else f'older than max_age_hours ({hours})' if now - st.st_mtime > hours * 3600 + else 'dated in the future' if st.st_mtime > now + AHEAD else None) + if why: + os.close(fd) + raise Refused(f'{path}: {why}', 4) + return fd, st + + +def offer(pool, command, lock_dir, out=1, err=sys.stderr, config=CONFIG, files=FILES, now=time.time): + try: + if os.geteuid() == 0: + raise Refused('serve never runs as root', 2) + word, _, name = command.partition(' ') + seeds, folder = pool_seeds(pool, config), os.path.join(files, pool) + if command == 'seeds': + got = [] + for s in seeds.values(): + item = {'name': s['name'], **{k: s[v] for k, v in ITEM.items()}} + try: + fd, st = offered(s, folder, now()) + os.close(fd) + item.update(bytes=st.st_size, made=int(st.st_mtime)) + except Refused as e: + item['why'] = str(e) + got.append(item) + os.write(out, (json.dumps({'v': 1, 'seeds': got}) + '\n').encode()) + return 0 + if word != 'seed' or name not in seeds: + raise Refused(f'refused: {command[:60]!r} (only `seeds`, or `seed NAME` of pool {pool}\'s seeds)', 2) + lock_path = os.path.join(lock_dir, f'{pool}.seed.lock') + try: + lock = os.open(lock_path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + except OSError as e: + raise Refused(f'{lock_path}: {e.strerror} (a leftover of another user? remove it)', 1) from None + try: + if os.fstat(lock).st_uid != os.geteuid(): + raise Refused(f'{lock_path} belongs to uid {os.fstat(lock).st_uid}, not this user: remove it', 1) + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + raise Refused(f'another seed stream for pool {pool} runs', 3) from None + fd, st = offered(seeds[name], folder, now()) + try: + sha, sent = send(fd, out, st.st_size) + after = os.fstat(fd) + finally: + os.close(fd) + finally: + os.close(lock) + # A rename over NAME.tar unlinks the open file (st_nlink 0), which changes its ctime but not its bytes: + # only a file still in place must keep its ctime (a rewrite with the old mtime put back). + same = ('st_size', 'st_mtime_ns') + (('st_ctime_ns',) if after.st_nlink else ()) + if sent != st.st_size or any(getattr(after, k) != getattr(st, k) for k in same): + raise Refused(f'{name}: it changed while it was sent (made in place, not by rename?)', 4) + print(f'seed {name} {sent} {sha} {int(st.st_mtime)}', file=err, flush=True) + return 0 + except Refused as e: + print(f'fork-fleet-desk seed: {e}', file=err, flush=True) + return e.code + + +class Stalled(Exception): + pass + + +def stalled(*_): + raise Stalled + + +def send(fd, out, size, stall=None): + """Sends at most SIZE bytes of FD (the size that was checked: a file that grows is cut there, and then fails the + changed-file check) to OUT; Refused (1) when OUT takes less than WRITE bytes in STALL seconds, or not all of it + within GRACE seconds plus SIZE / MIN_RATE.""" + stall = STALL if stall is None else stall + sha, sent = hashlib.sha256(), 0 + deadline = time.monotonic() + GRACE + size / MIN_RATE + old = signal.signal(signal.SIGALRM, stalled) + try: + while sent < size and (data := os.read(fd, min(PIECE, size - sent))): + sha.update(data) + sent += len(data) + view = memoryview(data) + while view: + left = deadline - time.monotonic() + if left <= 0: + raise Stalled + signal.alarm(max(1, min(stall, math.ceil(left)))) # each piece that moves starts the wait again + view = view[os.write(out, view[:WRITE]):] + except Stalled: + if time.monotonic() >= deadline: + raise Refused(f'the desktop did not take {size} bytes within {GRACE + size / MIN_RATE:.0f} seconds ' + f'(at least {MIN_RATE >> 10} KiB a second): stopped', 1) from None + raise Refused(f'the desktop took less than {WRITE >> 10} KiB in {stall} seconds: stopped', 1) from None + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, old) + return sha.hexdigest(), sent diff --git a/.github/scripts/fork-fleet-desk-seed.test.py b/.github/scripts/fork-fleet-desk-seed.test.py new file mode 100644 index 0000000000000..8171f0d8ca14e --- /dev/null +++ b/.github/scripts/fork-fleet-desk-seed.test.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Tests of fork-fleet-desk-seed.py (a desk pool's warm caches) and the `seeds` check of fork-fleet.json: the host's +offer in-process and the forced command's divert in fork-fleet-desk.py. Temp folders only; no root, no ssh, no +network. Run: python3 -I -B . Linux only.""" +import fcntl, hashlib, importlib.util, io, json, os, shutil, subprocess, sys, tarfile, tempfile, threading, time +import unittest +from unittest import mock + +sys.dont_write_bytecode = True +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def load(name, file): + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, file)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +seed, ff = load('fork_fleet_desk_seed', 'fork-fleet-desk-seed.py'), load('fork_fleet', 'fork-fleet.py') +with open(os.path.join(HERE, '..', 'fork-fleet.json'), encoding='utf-8') as _f: + CONFIG = json.load(_f) +POOL = CONFIG['pools'][0]['name'] +SEEDS = [{'name': 'm2', 'every_hours': 6, 'max_age_hours': 1000, 'max_mb': 2}, + {'name': 'npm', 'every_hours': 1, 'max_age_hours': 48, 'max_mb': 1}] + +class Base(unittest.TestCase): + def setUp(self): + self.t = os.path.realpath(tempfile.mkdtemp(prefix='seed-test-')) + self.addCleanup(shutil.rmtree, self.t, True) + for d in ('files', f'files/{POOL}', 'run', 'src'): + os.mkdir(os.path.join(self.t, d), 0o755) + self.config(SEEDS) + + def config(self, seeds): + cfg = json.loads(json.dumps(CONFIG)) + cfg['pools'][0]['seeds'] = seeds + with open(os.path.join(self.t, 'fork-fleet.json'), 'w', encoding='utf-8') as f: + json.dump(cfg, f) + + def make(self, name, members, mtime=None): + """FILES/POOL/NAME.tar of `members` ({path: bytes}), made as a project would: a temp file, then a rename.""" + tmp = os.path.join(self.t, 'files', POOL, f'.{name}.tmp') + with tarfile.open(tmp, 'w', format=tarfile.PAX_FORMAT) as tar: + for path, data in members.items(): + info = tarfile.TarInfo(path) + info.size, info.mode = len(data), 0o644 + tar.addfile(info, io.BytesIO(data)) + if mtime is not None: + os.utime(tmp, (mtime, mtime)) + os.chmod(tmp, 0o644) + os.replace(tmp, self.tar(name)) + with open(self.tar(name), 'rb') as f: + return f.read() + + def tar(self, name): + return os.path.join(self.t, 'files', POOL, f'{name}.tar') + + def offer(self, command, **kw): + r, w = os.pipe() + err = io.StringIO() + with os.fdopen(r, 'rb') as rf: + try: + code = seed.offer(POOL, command, os.path.join(self.t, 'run'), out=w, err=err, + config=os.path.join(self.t, 'fork-fleet.json'), files=os.path.join(self.t, 'files'), + **kw) + finally: + os.close(w) + return code, rf.read(), err.getvalue() + + +class Config(unittest.TestCase): + def test_seeds_are_checked(self): + def problems(seeds): + cfg = json.loads(json.dumps(CONFIG)) + cfg['pools'][0]['seeds'] = seeds + return [p for p in ff.check_config(cfg) if 'seed' in p] + self.assertEqual(problems(SEEDS), []) + self.assertEqual(problems([]), []) + self.assertEqual(problems([{**SEEDS[0], 'max_mb': ff.SEED_MAX_MB}]), []) + good = SEEDS[0] + for bad in ({}, [{**good, 'name': 'M2'}], [{**good, 'name': '../x'}], [{**good, 'every_hours': 0}], + [{**good, 'max_mb': True}], [{**good, 'max_age_hours': 5}], [{**good, 'max_mb': '2'}], + [{**good, 'max_mb': ff.SEED_MAX_MB + 1}], + [{k: v for k, v in good.items() if k != 'max_mb'}], ['m2'], [good, good]): + with self.subTest(bad): + self.assertEqual(len(problems(bad)), 1, bad) + + +class Host(Base): + def test_the_list_says_what_is_offered_and_why_not(self): + self.make('m2', {'a': b'x' * 100}) + code, out, err = self.offer('seeds') + self.assertEqual(code, 0, err) + got = json.loads(out) + self.assertEqual(got['v'], 1) + m2, npm = got['seeds'] + st = os.stat(self.tar('m2')) + self.assertEqual(m2, {'name': 'm2', 'everyHours': 6, 'maxAgeHours': 1000, 'maxMb': 2, 'bytes': st.st_size, + 'made': int(st.st_mtime)}) + self.assertNotIn('made', npm) + self.assertIn('not made yet', npm['why']) + + def test_a_seed_streams_whole_with_its_trailer(self): + data = self.make('m2', {'a/b': b'hello', 'c': b'x' * 5000}) + code, out, err = self.offer('seed m2') + self.assertEqual(code, 0, err) + self.assertEqual(out, data) + st = os.stat(self.tar('m2')) + self.assertEqual(err, f'seed m2 {len(data)} {hashlib.sha256(data).hexdigest()} {int(st.st_mtime)}\n') + + def test_other_commands_and_names_are_refused_and_send_nothing(self): + self.make('m2', {'a': b'x'}) + for command in ('seed', 'seed ../m2', 'seed /etc/passwd', 'seed m2 x', 'seed other', 'seeds x', 'serve x', + 'rm -rf /', 'seed M2'): + with self.subTest(command): + code, out, err = self.offer(command) + self.assertEqual((code, out), (2, b''), err) + + def test_a_file_that_may_not_be_sent_is_exit_4(self): + top, files = os.path.join(self.t, 'files'), os.path.join(self.t, 'files', POOL) + now = time.time() + cases = {'not made yet': lambda: None, + 'Too many levels of symbolic links': lambda: os.symlink(os.path.join(self.t, 'src', 'x.tar'), + f'{files}/m2.tar'), + 'others may write it': lambda: (self.make('m2', {'a': b'x'}), os.chmod(f'{files}/m2.tar', 0o664)), + 'above max_mb': lambda: self.make('m2', {'a': os.urandom(3 << 20)}), + 'older than max_age_hours': lambda: self.make('m2', {'a': b'x'}, mtime=now - 1001 * 3600), + 'dated in the future': lambda: self.make('m2', {'a': b'x'}, mtime=now + 10 * 365 * 86400), + f'{top} must be a folder': lambda: (self.make('m2', {'a': b'x'}), os.chmod(top, 0o777)), + 'not a plain file': lambda: os.mkdir(f'{files}/m2.tar'), + f'{files} must be a folder': lambda: (self.make('m2', {'a': b'x'}), os.chmod(files, 0o777))} + with open(os.path.join(self.t, 'src', 'x.tar'), 'wb') as f: + f.write(b'x') + os.chmod(os.path.join(self.t, 'src', 'x.tar'), 0o644) # only the link itself may be the reason + for name, make in cases.items(): + with self.subTest(name): + make() + code, out, err = self.offer('seed m2') + self.assertEqual((code, out), (4, b''), err) + self.assertIn(name, err) # refused for this reason, read where the pool's files are + os.chmod(files, 0o755) + os.chmod(top, 0o755) + p = f'{files}/m2.tar' + if os.path.isdir(p) and not os.path.islink(p): + os.rmdir(p) + elif os.path.lexists(p): + os.unlink(p) + + def test_a_second_stream_for_the_pool_is_exit_3(self): + self.make('m2', {'a': b'x'}) + with open(os.path.join(self.t, 'run', f'{POOL}.seed.lock'), 'a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + self.assertEqual(self.offer('seed m2')[:2], (3, b'')) + self.assertEqual(self.offer('seed m2')[0], 0) # the lock is gone with its holder + + def test_a_file_changed_while_sent_is_exit_4(self): + self.make('m2', {'a': b'x' * 3000}) + path, send = self.tar('m2'), seed.send + + def changing(fd, out, size): + with open(path, 'ab') as f: # made in place, not by rename + f.write(b'\0' * 512) + return send(fd, out, size) + with mock.patch.object(seed, 'send', changing): + code, out, err = self.offer('seed m2') + self.assertEqual(code, 4) + self.assertIn('changed while it was sent', err) + self.assertEqual(len(out), os.stat(path).st_size - 512) # cut at the size that was checked + + def test_a_rename_while_sent_is_fine(self): + old, send = self.make('m2', {'a': b'x' * 3000}), seed.send + + def renaming(fd, out, size): + self.make('m2', {'a': b'y' * 3000}) # the documented refresh: a temp file, then a rename + return send(fd, out, size) + with mock.patch.object(seed, 'send', renaming): + code, out, err = self.offer('seed m2') + self.assertEqual((code, out), (0, old), err) + self.assertTrue(err.startswith(f'seed m2 {len(old)} {hashlib.sha256(old).hexdigest()} '), err) + + def test_a_lock_of_another_user_is_refused(self): + self.make('m2', {'a': b'x'}) + open(os.path.join(self.t, 'run', f'{POOL}.seed.lock'), 'a').close() + with mock.patch.object(seed.os, 'geteuid', return_value=os.geteuid() + 4242): + code, out, err = self.offer('seed m2') + self.assertEqual((code, out), (1, b''), err) + self.assertIn(f'belongs to uid {os.geteuid()}', err) + + def test_a_same_size_rewrite_with_its_old_date_put_back_is_exit_4(self): + self.make('m2', {'a': b'x' * 3000}) + path, send = self.tar('m2'), seed.send + + def rewriting(fd, out, size): + st = os.stat(path) + with open(path, 'r+b') as f: # in place, same size, the old mtime put back: only ctime tells + f.write(b'y' * 512) + os.utime(path, ns=(st.st_atime_ns, st.st_mtime_ns)) + return send(fd, out, size) + with mock.patch.object(seed, 'send', rewriting): + code, _, err = self.offer('seed m2') + self.assertEqual(code, 4) + self.assertIn('changed while it was sent', err) + + def test_a_desktop_that_stops_reading_lets_the_lock_go(self): + self.make('m2', {'a': os.urandom(1 << 20)}) + r, w = os.pipe() # nobody reads r: the pipe fills and the write waits + err = io.StringIO() + try: + with mock.patch.object(seed, 'STALL', 1): + t = time.monotonic() + code = seed.offer(POOL, 'seed m2', os.path.join(self.t, 'run'), out=w, err=err, + config=os.path.join(self.t, 'fork-fleet.json'), files=os.path.join(self.t, 'files')) + self.assertLess(time.monotonic() - t, 30) + finally: + os.close(r) + os.close(w) + self.assertEqual(code, 1) + self.assertIn('took less than 64 KiB in 1 seconds', err.getvalue()) + self.make('m2', {'a': b'x'}) # small enough for the pipe: offer() below has no reader until it returns + self.assertEqual(self.offer('seed m2')[0], 0) # the pool's lock was let go + + def test_a_slow_desktop_meets_the_deadline(self): + self.make('m2', {'a': os.urandom(1 << 20)}) + r, w = os.pipe() # nobody reads r; the stall wait alone (100 s) would hold the lock much longer + err = io.StringIO() + try: + with mock.patch.multiple(seed, STALL=100, GRACE=1, MIN_RATE=1 << 40): + t = time.monotonic() + code = seed.offer(POOL, 'seed m2', os.path.join(self.t, 'run'), out=w, err=err, + config=os.path.join(self.t, 'fork-fleet.json'), files=os.path.join(self.t, 'files')) + self.assertLess(time.monotonic() - t, 30) + finally: + os.close(r) + os.close(w) + self.assertEqual(code, 1, err.getvalue()) + self.assertIn('within 1 seconds', err.getvalue()) + + def test_a_desktop_reading_slowly_but_steadily_is_not_dropped(self): + data = self.make('m2', {'a': os.urandom(300 << 10)}) + r, w = os.pipe() + got = bytearray() + + def reader(): + with os.fdopen(r, 'rb', buffering=0) as f: + while chunk := f.read(16 << 10): # 16 KiB, then a pause: under a stall wait per 64 KiB piece + got.extend(chunk) + time.sleep(0.3) + th = threading.Thread(target=reader) + th.start() + err = io.StringIO() + try: + with mock.patch.object(seed, 'STALL', 2): + code = seed.offer(POOL, 'seed m2', os.path.join(self.t, 'run'), out=w, err=err, + config=os.path.join(self.t, 'fork-fleet.json'), files=os.path.join(self.t, 'files')) + finally: + os.close(w) + th.join(60) + self.assertEqual((code, bytes(got)), (0, data), err.getvalue()) + + def test_pools_never_share_a_seed_file(self): + cfg = json.loads(json.dumps(CONFIG)) + cfg['pools'][0]['seeds'] = SEEDS + cfg['pools'].append({**cfg['pools'][0], 'name': 'other-pool', 'label': 'fleet-other-t3code', + 'beacon': 'FLEET_BEACON_OTHER'}) + self.assertEqual(ff.check_config(cfg), []) + with open(os.path.join(self.t, 'fork-fleet.json'), 'w', encoding='utf-8') as f: + json.dump(cfg, f) + self.make('m2', {'a': b'x'}) + shutil.copy(self.tar('m2'), os.path.join(self.t, 'files', 'm2.tar')) # one file for all pools, as before + os.mkdir(os.path.join(self.t, 'files', 'other-pool'), 0o755) + r, w = os.pipe() + err = io.StringIO() + try: + code = seed.offer('other-pool', 'seed m2', os.path.join(self.t, 'run'), out=w, err=err, + config=os.path.join(self.t, 'fork-fleet.json'), files=os.path.join(self.t, 'files')) + finally: + os.close(w) + os.close(r) + self.assertEqual(code, 4) + self.assertIn('not made yet', err.getvalue()) + + def test_never_as_root_and_a_bad_config_is_exit_2(self): + with mock.patch.object(seed.os, 'geteuid', return_value=0): + self.assertEqual(self.offer('seeds')[:2], (2, b'')) + with open(os.path.join(self.t, 'fork-fleet.json'), 'w') as f: + f.write('{') + self.assertEqual(self.offer('seeds')[:2], (2, b'')) + + def test_the_forced_command_diverts_seed_requests(self): + opt = os.path.join(self.t, 'opt') # the host's install: the scripts and fork-fleet.json in one folder + os.mkdir(opt) + for f in ('fork-fleet-desk.py', 'fork-fleet-desk-wire.py', 'fork-fleet-desk-seed.py', 'fork-fleet.py'): + shutil.copy(os.path.join(HERE, f), opt) + shutil.copy(os.path.join(self.t, 'fork-fleet.json'), opt) + desk = os.path.join(opt, 'fork-fleet-desk.py') + base = {'PATH': os.environ['PATH'], 'FORK_FLEET_DESK_DIR': os.path.join(self.t, 'run2')} + got = subprocess.run([sys.executable, '-I', '-B', desk, 'serve', POOL], capture_output=True, text=True, + env={**base, 'SSH_ORIGINAL_COMMAND': 'seeds'}, timeout=30, stdin=subprocess.DEVNULL) + self.assertEqual(got.returncode, 0, got.stderr) + self.assertEqual([s['name'] for s in json.loads(got.stdout)['seeds']], ['m2', 'npm']) + got = subprocess.run([sys.executable, '-I', '-B', desk, 'serve', POOL], capture_output=True, text=True, + env={**base, 'SSH_ORIGINAL_COMMAND': 'seed nothing-like-this'}, timeout=30, + stdin=subprocess.DEVNULL) + self.assertEqual((got.returncode, got.stdout), (2, '')) + self.assertFalse(os.path.exists(os.path.join(self.t, 'run2', f'{POOL}.sock'))) # no session was served + for command in ('seeds x', 'seed', 'seedsx', 'seeds '): + with self.subTest(command): + got = subprocess.run([sys.executable, '-I', '-B', desk, 'serve', POOL], capture_output=True, + text=True, env={**base, 'SSH_ORIGINAL_COMMAND': command}, timeout=30, + stdin=subprocess.DEVNULL) + self.assertEqual((got.returncode, got.stdout), (2, ''), got.stderr) + self.assertIn('fork-fleet-desk seed: refused', got.stderr) + self.assertFalse(os.path.exists(os.path.join(self.t, 'run2', f'{POOL}.sock'))) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/scripts/fork-fleet-desk.py b/.github/scripts/fork-fleet-desk.py index 254aec7510e7f..fd8ff12cb13d9 100755 --- a/.github/scripts/fork-fleet-desk.py +++ b/.github/scripts/fork-fleet-desk.py @@ -17,6 +17,8 @@ POOL]. Sends one launcher verb (`start` with its stdin) through the live session and prints the desktop's stdout and stderr and exits with its exit code. No live session: exit 3 at once, nothing sent. No answer in the verb's time (fork-fleet-desk-wire.py, "Time"): exit 75, the verb may or may not have run. + The same forced command asked anything that starts with `seed` (SSH_ORIGINAL_COMMAND; `seeds`, `seed NAME`): a + desktop's warm caches, never a session (fork-fleet-desk-seed.py; its lock DIR/POOL.seed.lock). DIR is /run/fork-fleet-desk, made at boot by systemd-tmpfiles (runbook): owned by the desk-link user, its group the socket group (who may connect: the dispatcher's user, or root), mode 0710 or 0750. The socket is 0660 in that group, and serve lets in only root, its own user and members of that group (SO_PEERCRED). Not as root (tests), @@ -391,9 +393,24 @@ def launch(pool, verb, args, stdin, path, out=sys.stdout, err=sys.stderr, wait=N return got['exit'] +def seed_side(pool, asked, env): + """A desktop's seed session (fork-fleet-desk-seed.py): the same key and forced command, another command.""" + pool_ok(pool) + if os.geteuid() == 0: + raise Refused('serve never runs as root', 2) + path = sock_dir(env) + safe_dir(path, (os.geteuid(),), make=True) + spec = importlib.util.spec_from_file_location('fork_fleet_desk_seed', os.path.join(HERE, 'fork-fleet-desk-seed.py')) + seed = importlib.util.module_from_spec(spec) + spec.loader.exec_module(seed) + return seed.offer(pool, asked, path) + + def main(argv=None, env=os.environ): argv = sys.argv[1:] if argv is None else argv try: + if len(argv) == 2 and argv[0] == 'serve' and env.get('SSH_ORIGINAL_COMMAND', '').startswith('seed'): + return seed_side(argv[1], env['SSH_ORIGINAL_COMMAND'], env) # every seed* command; it refuses the rest if len(argv) == 2 and argv[0] == 'serve': for sig in (signal.SIGTERM, signal.SIGHUP): signal.signal(sig, lambda *_: sys.exit(1)) # through serve's finally: the socket goes, clients learn diff --git a/.github/scripts/fork-fleet.py b/.github/scripts/fork-fleet.py index 95a1be08f69b6..2581a950e3cfb 100644 --- a/.github/scripts/fork-fleet.py +++ b/.github/scripts/fork-fleet.py @@ -55,6 +55,7 @@ VARIABLE = re.compile(r'(?!GITHUB_)[A-Z][A-Z0-9_]{0,99}') JOB_ID = re.compile(r'[a-z0-9_]{1,40}') POOL = re.compile(r'[a-z0-9][a-z0-9_-]{0,39}') +SEED_MAX_MB = 16384 # a seed's largest max_mb: 16 GiB KINDS = ('push', 'draft', 'ready') # a pool's `takes`: pushes, draft pull requests, pull requests ready for review # Events whose code and workflow are this repository's own (a pull request event: only when head and base are too). OWN_EVENTS = ('push', 'workflow_dispatch', 'schedule') @@ -139,6 +140,15 @@ def check_config(cfg): who = f'pool {p.get("name")!r}' if not (isinstance(p.get('name'), str) and POOL.fullmatch(p['name'])): out.append(f'{who}: name must be a-z, 0-9, _ and - (it names the dispatcher\'s lock file)') + seeds = p.get('seeds', []) # a desk pool's warm caches: fork-fleet-desk-seed.py + if not (isinstance(seeds, list) and all( + isinstance(x, dict) and isinstance(x.get('name'), str) and POOL.fullmatch(x['name']) + and all(positive(x.get(k)) for k in ('every_hours', 'max_age_hours', 'max_mb')) + and x['max_age_hours'] >= x['every_hours'] and x['max_mb'] <= SEED_MAX_MB for x in seeds)): + out.append(f'{who}: seeds must list {{"name" (a-z, 0-9, _, -), "every_hours", "max_age_hours" (at least ' + f'every_hours), "max_mb" (at most {SEED_MAX_MB})}}, each a positive whole number') + elif len({x['name'] for x in seeds}) != len(seeds): + out.append(f'{who}: two seeds with one name') if not positive(p.get('slots')): out.append(f'{who}: slots must be a positive whole number') if not isinstance(p.get('waits'), bool):