From ec41bd876207d62718749348df5273eae1148f42 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:14:00 +0000 Subject: [PATCH 1/6] fork fleet: a desktop fetches its desk pool's seeds (D11b) fork-fleet-desk-seed.py run, hourly from t3code-fleet-seed.timer (root, sandboxed), asks the always-on host for the pool's seed list over the desk link's key, fetches a seed only when it changed and every_hours passed, unpacks it with tarfile's data filter, checks the trailer's size and SHA-256 and swaps it in by rename. Jobs see /var/lib/fork-fleet/seeds read-only (ALLOW_PATHS). Torn downloads keep the old seed; old, future-dated or unlisted seeds are dropped; replaced versions stay a day for running jobs. The desk kit installs the script and the timer and removes the seeds on uninstall. Co-Authored-By: Claude Opus 5.5 --- .github/fork-fleet.md | 8 + .github/scripts/fork-fleet-desk-seed.py | 257 ++++++++++++++++++- .github/scripts/fork-fleet-desk-seed.test.py | 236 ++++++++++++++++- .github/scripts/fork-fleet-seed.service | 44 ++++ .github/scripts/fork-fleet-seed.timer | 13 + fork-kit/desk/README.md | 7 +- fork-kit/desk/sbin/desk-install | 19 +- fork-kit/desk/tests/install.test.py | 19 +- 8 files changed, 585 insertions(+), 18 deletions(-) create mode 100644 .github/scripts/fork-fleet-seed.service create mode 100644 .github/scripts/fork-fleet-seed.timer diff --git a/.github/fork-fleet.md b/.github/fork-fleet.md index 1a0b7c895f3a..ae52011a38f6 100644 --- a/.github/fork-fleet.md +++ b/.github/fork-fleet.md @@ -709,6 +709,14 @@ own: the same forced command answers `seeds` with the pool's list and size and SHA-256 (`fork-fleet-desk-seed.py`, which step 3 installs; its header has the exit codes). +The desktop's side is the same script's `run`, hourly from +`t3code-fleet-seed.timer` (the desk kit installs both). Each new seed is +unpacked with Python's tar data filter, checked against the trailer, and +swapped in by a rename; jobs see `/var/lib/fork-fleet/seeds/NAME` read-only +(the launcher's `ALLOW_PATHS`), copy what they need from it when it is there, +and run cold when it is not. The state table (a torn download, the host down, +clocks that disagree, a run that stopped half-way) is in the script's header. + ## Owner steps on the repository Run these from a machine where `gh` is logged in as the owner. None of them diff --git a/.github/scripts/fork-fleet-desk-seed.py b/.github/scripts/fork-fleet-desk-seed.py index 2a68f886fc7d..40a862221417 100644 --- a/.github/scripts/fork-fleet-desk-seed.py +++ b/.github/scripts/fork-fleet-desk-seed.py @@ -12,12 +12,33 @@ seed the host offers now, `why` for one it does not) ssh ... seed NAME streams FILES/NAME.tar: only a name of that list, never a path the desktop names. The last stderr line is the trailer `seed NAME BYTES SHA256 MADE` (MADE: the file's mtime, epoch seconds) + python3 -I fork-fleet-desk-seed.py [--config FILE] run + the desktop, as root, hourly (t3code-fleet-seed.timer), with the agent's settings (FILE, default + its CONF). Each new seed is unpacked with tarfile's data filter into a folder of its own, checked + against the trailer, flushed, and swapped in by rename: SEEDS/NAME is a link to + .v/NAME.STAMP/data, root's (folders 0755, files no one else may write), unpacked in .new + (one folder: a rename never crosses a mount of the unit's sandbox). Jobs see SEEDS read-only + (the launcher's ALLOW_PATHS): a job uses /var/lib/fork-fleet/seeds/NAME when it is there and runs + cold when it is not. +Each run (one at a time: a lock) meets this state: + the same seed on the host (bytes and MADE) nothing is fetched + a new one under every_hours after the last nothing yet + a torn copy (the stream cut or past FETCH_WAIT, a trailer that does not match, more than max_mb, a tar member + that would leave its folder or is a device) the copy is removed, the old seed stays, the next hour tries again + no seed (never fetched, the host down) jobs run cold + older than max_age_hours, made over AHEAD in the future (the clocks disagree), or no longer in the pool's list + the link is removed: jobs run cold + a label that is missing or unreadable the link is removed and the seed fetched again in the same run + a run that stopped half-way SEEDS/.new is emptied by the next run; a link is the old one or the new + a replaced version kept KEEP_OLD after it was replaced, for a job still reading it (a job + runs at most the launcher's MAX_SECONDS, 1 h by default), then removed The host side exits 2 for any other command, 3 while another stream for the pool runs, 4 when the file may not be sent (missing, a link, in a folder or a file others may write, above max_mb, older than max_age_hours, or changed while it was sent: made in place, not by rename). Standard library only. """ -import fcntl, hashlib, importlib.util, json, os, re, stat, sys, time +import argparse, fcntl, hashlib, importlib.util, json, os, re, shutil, stat, subprocess, sys, tarfile, tempfile +import threading, time sys.dont_write_bytecode = True HERE = os.path.dirname(os.path.realpath(__file__)) @@ -31,7 +52,12 @@ def module(name, file): FILES = '/var/lib/fork-fleet/seed-files' # the always-on host's: NAME.tar, made by the project +SEEDS = '/var/lib/fork-fleet/seeds' # the desktop's: what jobs see CONFIG = os.path.join(HERE, 'fork-fleet.json') +NAME = re.compile(r'[a-z0-9][a-z0-9_-]{0,39}') +LINK = re.compile(r'\.v/((?P[a-z0-9][a-z0-9_-]{0,39})\.[0-9]{1,20})/data') +LIST_WAIT, FETCH_WAIT = 30, 30 * 60 # seconds for the list, and for one seed's whole stream +KEEP_OLD, AHEAD = 24 * 3600, 3600 PIECE = 1 << 20 ITEM = {'everyHours': 'every_hours', 'maxAgeHours': 'max_age_hours', 'maxMb': 'max_mb'} @@ -42,6 +68,8 @@ def __init__(self, msg, code): self.code = code +# ---------------------------------------------------------------- the always-on host (serve's forced command) + def pool_seeds(pool, config): try: cfg = module('fork_fleet', 'fork-fleet.py').load(config) @@ -127,3 +155,230 @@ def send(fd, out): while view: view = view[os.write(out, view):] return sha.hexdigest(), sent + + +# ---------------------------------------------------------------- the desktop + +class Torn(Exception): + pass + + +class Summed: + """The ssh stream as tarfile reads it: counted, hashed, cut past `most` bytes.""" + + def __init__(self, raw, most): + self.raw, self.most, self.size, self.sha = raw, most, 0, hashlib.sha256() + + def read(self, n=PIECE): + data = self.raw.read(n if n is not None and n >= 0 else PIECE) + self.size += len(data) + if self.size > self.most: + raise Torn(f'more than {self.most >> 20} MB (max_mb)') + self.sha.update(data) + return data + + +def ssh(agent, conf, *words): + return agent.ssh_argv(conf)[:-2] + list(words) # the agent's ssh, another command than `serve POOL` + + +def listing(agent, conf): + """{name: item} of the host's list; Torn when it cannot be had or is not well formed.""" + with tempfile.TemporaryFile() as err: + try: + r = subprocess.run(ssh(agent, conf, 'seeds'), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=err, + env=agent.ENV, timeout=LIST_WAIT, start_new_session=True) + except subprocess.TimeoutExpired: + raise Torn(f'no list in {LIST_WAIT} s') from None + except OSError as e: + raise Torn(f'no ssh: {e}') from None + if r.returncode: + raise Torn(f'no list (exit {r.returncode}): {tail(err)}') + try: + got = json.loads(r.stdout) + assert isinstance(got, dict) and got.get('v') == 1 and isinstance(got.get('seeds'), list) + out = {} + for s in got['seeds']: + assert isinstance(s, dict) and isinstance(s.get('name'), str) and NAME.fullmatch(s['name']) + assert all(whole(s.get(k)) and s[k] > 0 for k in ITEM) and ('made' not in s or whole(s.get('bytes')) + and whole(s['made'])) + out[s['name']] = s + return out + except (ValueError, AssertionError): + raise Torn('the host\'s list is not well formed') from None + + +def whole(x): + return isinstance(x, int) and not isinstance(x, bool) and x >= 0 + + +def tail(f): + f.seek(0) + return ' '.join(f.read()[-2000:].decode('utf-8', 'replace').split())[-300:] or 'nothing on stderr' + + +def fetch(agent, conf, item, root, now): + """The seed `item` into a new version, swapped in; its label. Torn (or OSError, TarError) leaves the old one.""" + name, stamp = item['name'], time.time_ns() + new = os.path.join(root + '/.new', f'{name}.{stamp}') + os.mkdir(new, 0o755) + os.mkdir(os.path.join(new, 'data'), 0o755) + with tempfile.TemporaryFile() as err: + p = subprocess.Popen(ssh(agent, conf, 'seed', name), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=err, env=agent.ENV, start_new_session=True) + timer = threading.Timer(FETCH_WAIT, p.kill) + timer.start() + try: + got = Summed(p.stdout, item['maxMb'] << 20) + with tarfile.open(fileobj=got, mode='r|') as t: + t.extractall(os.path.join(new, 'data'), filter='data') + while got.read(PIECE): # the tar's end blocks and padding count in the trailer's size and sha256 + pass + code = p.wait(timeout=30) + finally: + timer.cancel() + p.kill() + p.wait() + p.stdout.close() + err.seek(0) + last = (err.read()[-4096:].decode('utf-8', 'replace').strip().splitlines() or [''])[-1].split() + if code or last[:4] != ['seed', name, str(got.size), got.sha.hexdigest()] or len(last) != 5 or \ + not last[4].isdigit(): + raise Torn(f'exit {code}, {got.size} bytes, the trailer does not match: {tail(err)}') + label = {'v': 1, 'name': name, 'bytes': got.size, 'sha256': got.sha.hexdigest(), 'made': int(last[4]), + 'fetched': int(now()), 'maxAgeHours': item['maxAgeHours']} + with open(os.path.join(new, 'label.json'), 'x', encoding='utf-8') as f: + json.dump(label, f) + os.sync() # the new version is on disk before any link names it + version = os.path.join(root, '.v', os.path.basename(new)) + os.rename(new, version) + link = os.path.join(root, f'.{name}.link') + remove(link) + os.symlink(f'.v/{name}.{stamp}/data', link) + old = current(root, name)[0] + if old: + os.utime(old, (now(), now())) # replaced now: kept KEEP_OLD for a job still reading it + os.replace(link, os.path.join(root, name)) + return label + + +def current(root, name): + """(version folder or None, label or None) of the link SEEDS/NAME.""" + try: + m = LINK.fullmatch(os.readlink(os.path.join(root, name))) + except OSError: + return None, None + if not m or m['name'] != name: + return None, None + version = os.path.join(root, '.v', m[1]) + try: + with open(os.path.join(version, 'label.json'), encoding='utf-8') as f: + label = json.load(f) + ok = isinstance(label, dict) and label.get('name') == name and all( + whole(label.get(k)) for k in ('bytes', 'made', 'fetched', 'maxAgeHours')) + except (OSError, ValueError): + ok = False + return version, label if ok else None + + +def remove(path): + try: + st = os.lstat(path) + except FileNotFoundError: + return + if stat.S_ISDIR(st.st_mode): + shutil.rmtree(path) + else: + os.unlink(path) + + +def drop(root, name, version, now): + if version and os.path.isdir(version): + os.utime(version, (now(), now())) # first: a run that stops here leaves it linked, never pruned early + remove(os.path.join(root, name)) + + +def run(agent, conf, root=SEEDS, now=time.time, say=print): + if not hasattr(tarfile, 'data_filter'): + raise agent.Bad('this Python has no tarfile data filter (3.12, or 3.8.17 and later): no seeds') + os.umask(0o022) + for d in (root, root + '/.new', root + '/.v'): # root: made by the unit (StateDirectory) or here + os.makedirs(d, 0o755, exist_ok=True) + with open(os.path.join(root + '/.new', '.lock'), 'a') as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + say('another seed run is running') + return 1 + for e in os.listdir(root + '/.new'): # a run that stopped half-way + if e != '.lock': + remove(os.path.join(root + '/.new', e)) + try: + items = listing(agent, conf) + except Torn as e: + items = None + say(f'seeds: {e}: each seed stays until its age drops it') + for name in sorted(n for n in os.listdir(root) if NAME.fullmatch(n)): + version, label = current(root, name) + if label is None: + why = 'its label is missing or cannot be read (fetched again now)' + elif items is not None and name not in items: + why = 'no longer one of the pool\'s seeds' + else: + age = now() - label['made'] + hours = items[name]['maxAgeHours'] if items else label['maxAgeHours'] + why = (f'older than {hours} h (max_age_hours)' if age > hours * 3600 + else 'made in the future (the clocks disagree)' if age < -AHEAD else None) + if why: + drop(root, name, version, now) + say(f'seed {name}: dropped, {why}; jobs run cold until a new one comes') + for name, item in sorted((items or {}).items()): + label = current(root, name)[1] + if 'made' not in item: + say(f'seed {name}: the host offers none ({item.get("why", "no reason")}): ' + f'{"the one here stays" if label else "jobs run cold"}') + continue + if label and (label['bytes'], label['made']) == (item['bytes'], item['made']): + continue + if label and 0 <= now() - label['fetched'] < item['everyHours'] * 3600: + continue + if not -AHEAD <= now() - item['made'] <= item['maxAgeHours'] * 3600: + say(f'seed {name}: the host\'s is too old or made in the future (the clocks disagree): not fetched') + continue + try: + got = fetch(agent, conf, item, root, now) + say(f'seed {name}: {got["bytes"] >> 20} MB, sha256 checked, swapped in') + except (Torn, OSError, tarfile.TarError, subprocess.SubprocessError) as e: + for e2 in os.listdir(root + '/.new'): + if e2 != '.lock': + remove(os.path.join(root + '/.new', e2)) + say(f'seed {name}: keeps {"the old one" if label else "none"}: {e.__class__.__name__}: ' + f'{str(e)[:300]}; tried again next hour') + prune(root, now) + return 0 + + +def prune(root, now): + """Versions no link names, KEEP_OLD after they were replaced or dropped.""" + linked = {current(root, n)[0] for n in os.listdir(root) if NAME.fullmatch(n)} + for e in os.listdir(root + '/.v'): + path = os.path.join(root, '.v', e) + if path not in linked and now() - os.lstat(path).st_mtime > KEEP_OLD: + remove(path) + + +def main(argv=None): + a = argparse.ArgumentParser(description='a desk pool\'s warm caches, the desktop\'s side (as root)') + agent = module('fork_fleet_desk_agent', 'fork-fleet-desk-agent.py') + a.add_argument('--config', default=agent.CONF) + a.add_argument('verb', choices=('run',)) + a = a.parse_args(argv) + try: + return run(agent, agent.load(a.config)) + except agent.Bad as e: + print(f'fork-fleet-desk-seed: {e}', file=sys.stderr) + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/scripts/fork-fleet-desk-seed.test.py b/.github/scripts/fork-fleet-desk-seed.test.py index 6fb4307c5a89..9a13b16bfe10 100644 --- a/.github/scripts/fork-fleet-desk-seed.test.py +++ b/.github/scripts/fork-fleet-desk-seed.test.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """Tests of fork-fleet-desk-seed.py (a desk pool's warm caches) and the `seeds` check of fork-fleet.json: the host's -offer in-process and the forced command's divert in fork-fleet-desk.py. Temp folders only; no root, no ssh, no -network. Run: python3 -I -B . Linux only.""" -import fcntl, hashlib, importlib.util, io, json, os, shutil, subprocess, sys, tarfile, tempfile, time, unittest +offer in-process, the desktop's run end to end through a fake ssh that runs the host's side, and the forced command's +divert in fork-fleet-desk.py. Temp folders only; no root, no ssh, no network. Run: python3 -I -B . Linux only.""" +import fcntl, hashlib, importlib.util, io, json, os, shutil, stat, subprocess, sys, tarfile, tempfile, time, unittest from unittest import mock sys.dont_write_bytecode = True @@ -17,12 +17,49 @@ def load(name, file): seed, ff = load('fork_fleet_desk_seed', 'fork-fleet-desk-seed.py'), load('fork_fleet', 'fork-fleet.py') +agent = load('fork_fleet_desk_agent', 'fork-fleet-desk-agent.py') with open(os.path.join(HERE, '..', 'fork-fleet.json'), encoding='utf-8') as _f: CONFIG = json.load(_f) POOL = CONFIG['pools'][0]['name'] SEEDS = [{'name': 'm2', 'every_hours': 6, 'max_age_hours': 1000, 'max_mb': 2}, {'name': 'npm', 'every_hours': 1, 'max_age_hours': 48, 'max_mb': 1}] +FAKE_SSH = '''#!{python} -I +import io, os, sys +sys.dont_write_bytecode = True +import importlib.util +spec = importlib.util.spec_from_file_location('s', {module!r}) +seed = importlib.util.module_from_spec(spec) +spec.loader.exec_module(seed) +args = sys.argv[1:] +words = args[args.index('--') + 2:] +with open({t!r} + '/calls', 'a') as f: + f.write(' '.join(words) + '\\n') +try: + with open({t!r} + '/mode') as f: + mode = f.read().strip() +except FileNotFoundError: + mode = '' +if mode == 'down': + print('ssh: connect to host h port 22: No route to host', file=sys.stderr) + sys.exit(255) +kw = dict(config={t!r} + '/fork-fleet.json', files={t!r} + '/files') +if words[0] == 'seed' and mode in ('cut', 'trailer'): + with open({t!r} + '/sent', 'w+b') as out: + err = io.StringIO() + code = seed.offer({pool!r}, ' '.join(words), {t!r} + '/run', out=out.fileno(), err=err, **kw) + out.seek(0) + data = out.read() + if mode == 'cut': + os.write(1, data[:len(data) // 2]) + sys.exit(255) + os.write(1, data) + print(err.getvalue().rsplit(' ', 2)[0] + ' ' + '0' * 64 + ' 1', file=sys.stderr) # a wrong sha256 + sys.exit(code) +sys.exit(seed.offer({pool!r}, ' '.join(words), {t!r} + '/run', **kw)) +''' + + class Base(unittest.TestCase): def setUp(self): self.t = os.path.realpath(tempfile.mkdtemp(prefix='seed-test-')) @@ -182,5 +219,198 @@ def test_the_forced_command_diverts_seed_requests(self): self.assertFalse(os.path.exists(os.path.join(self.t, 'run2', f'{POOL}.sock'))) # no session was served +class Desktop(Base): + def setUp(self): + super().setUp() + self.root = os.path.join(self.t, 'seeds') + ssh = os.path.join(self.t, 'ssh') + with open(ssh, 'w') as f: + f.write(FAKE_SSH.format(python=sys.executable, module=seed.__file__, t=self.t, pool=POOL)) + os.chmod(ssh, 0o755) + self.conf = {'SSH': ssh, 'KEY': '/nonexistent/key', 'KNOWN_HOSTS': '/nonexistent/known', 'PORT': '22', + 'USER': 'fleet-desk', 'HOST': 'fleet-host.example', 'POOL': POOL} + self.said = [] + sync = mock.patch.object(seed.os, 'sync') + sync.start() + self.addCleanup(sync.stop) + + def run_seed(self, now=None): + self.said = [] + return seed.run(agent, self.conf, self.root, now=now or time.time, say=self.said.append) + + def mode(self, text): + with open(os.path.join(self.t, 'mode'), 'w') as f: + f.write(text) + + def fetches(self): + try: + with open(os.path.join(self.t, 'calls')) as f: + return [line for line in f.read().splitlines() if line.startswith('seed ')] + except FileNotFoundError: + return [] + + def files(self, name): + top = os.path.join(self.root, name) + if not os.path.exists(top): + return None + out = {} + for d, _, names in os.walk(top): + for n in names: + with open(os.path.join(d, n), 'rb') as f: + out[os.path.relpath(os.path.join(d, n), top)] = f.read() + return out + + def test_fetched_checked_and_swapped_in_then_not_again(self): + self.make('m2', {'repo/a.jar': b'jar', 'repo/b/c.pom': b'pom'}) + self.assertEqual(self.run_seed(), 0) + self.assertEqual(self.files('m2'), {'repo/a.jar': b'jar', 'repo/b/c.pom': b'pom'}) + self.assertTrue(os.readlink(os.path.join(self.root, 'm2')).startswith('.v/m2.')) + self.assertIsNone(self.files('npm')) + self.assertTrue(any('m2:' in s and 'swapped in' in s for s in self.said), self.said) + self.assertTrue(any('npm' in s and 'jobs run cold' in s for s in self.said), self.said) + for d, dirs, names in os.walk(self.root): + for n in dirs + names: + st = os.lstat(os.path.join(d, n)) + if not stat.S_ISLNK(st.st_mode): # a link's own mode is always 0777 and means nothing + self.assertFalse(st.st_mode & 0o022, os.path.join(d, n)) + label = json.load(open(os.path.join(seed.current(self.root, 'm2')[0], 'label.json'))) + self.assertEqual(label['bytes'], os.path.getsize(os.path.join(self.t, 'files', 'm2.tar'))) + self.assertEqual(self.run_seed(), 0) # the same seed on the host: nothing fetched + self.assertEqual(self.fetches(), ['seed m2']) + self.assertEqual(os.listdir(os.path.join(self.root, '.new')), ['.lock']) + + def test_a_new_seed_waits_for_every_hours_and_the_old_stays_for_running_jobs(self): + self.make('m2', {'a': b'one'}) + self.run_seed() + old = seed.current(self.root, 'm2')[0] + self.make('m2', {'a': b'two'}, mtime=time.time() + 5) + self.run_seed() + self.assertEqual(self.files('m2'), {'a': b'one'}) # under every_hours since the last fetch + later = time.time() + seed.KEEP_OLD + 3600 # a swap long after the old one came: kept from the swap on + self.run_seed(now=lambda: later) + self.assertEqual(self.files('m2'), {'a': b'two'}) + self.assertTrue(os.path.isdir(old)) # a job still reading it keeps it + self.run_seed(now=lambda: later + seed.KEEP_OLD + 60) + self.assertFalse(os.path.exists(old)) + self.assertEqual(self.files('m2'), {'a': b'two'}) + self.assertEqual(os.listdir(os.path.join(self.root, '.v')), [os.path.basename(seed.current(self.root, + 'm2')[0])]) + + def torn(self, setup, want_in_say): + self.make('m2', {'a': b'one'}) + self.run_seed() + setup() + self.run_seed(now=lambda: time.time() + 7 * 3600) + self.assertEqual(self.files('m2'), {'a': b'one'}) + self.assertEqual(os.listdir(os.path.join(self.root, '.new')), ['.lock']) + self.assertEqual(len(os.listdir(os.path.join(self.root, '.v'))), 1) + self.assertTrue(any('keeps the old one' in s and any(w in s for w in want_in_say.split('|')) + for s in self.said), self.said) + + def test_a_cut_stream_leaves_the_old_seed(self): + self.torn(lambda: (self.make('m2', {'a': os.urandom(200000)}, mtime=time.time() + 5), self.mode('cut')), + 'Torn|ReadError') # cut in the tar or after it + + def test_a_trailer_that_does_not_match_leaves_the_old_seed(self): + self.torn(lambda: (self.make('m2', {'a': b'two'}, mtime=time.time() + 5), self.mode('trailer')), + 'trailer does not match') + + def test_more_than_max_mb_leaves_the_old_seed(self): + def bigger(): + self.config([{**SEEDS[0], 'max_mb': 3}, SEEDS[1]]) # the host offers it; the list the desktop sees + self.make('m2', {'a': os.urandom(5 << 19)}, mtime=time.time() + 5) + with mock.patch.object(seed, 'listing', lambda a, c, real=seed.listing: { + k: {**v, 'maxMb': 2} for k, v in real(a, c).items()}): + self.run_seed(now=lambda: time.time() + 7 * 3600) + self.make('m2', {'a': b'one'}) + self.run_seed() + bigger() + self.assertEqual(self.files('m2'), {'a': b'one'}) + self.assertTrue(any('max_mb' in s for s in self.said), self.said) + + def test_a_tar_that_would_leave_its_folder_writes_nothing_outside(self): + bad = {'a ../ path': ('../../escaped', tarfile.REGTYPE, ''), 'a device': ('escaped', tarfile.CHRTYPE, ''), + 'an absolute link': ('escaped', tarfile.SYMTYPE, '/'), + 'a link up': ('escaped', tarfile.SYMTYPE, '../../..'), + 'a hard link out': ('escaped', tarfile.LNKTYPE, '/etc/hostname')} + for why, (name, kind, target) in bad.items(): + def evil(): + tmp = os.path.join(self.t, 'files', '.m2.tmp') + with tarfile.open(tmp, 'w') as tar: # one bad member each: the filter stops at the first + ok = tarfile.TarInfo('ok') + ok.size = 1 + tar.addfile(ok, io.BytesIO(b'x')) + info = tarfile.TarInfo(name) + info.type, info.linkname = kind, target + tar.addfile(info) + os.utime(tmp, (time.time() + 5,) * 2) + os.replace(tmp, os.path.join(self.t, 'files', 'm2.tar')) + with self.subTest(why): + shutil.rmtree(self.root, True) + self.torn(evil, 'Error') + for d, dirs, names in os.walk(self.t): + self.assertNotIn('escaped', dirs + names, d) + + def test_dropped_when_no_longer_listed_or_too_old_or_from_the_future(self): + self.make('m2', {'a': b'one'}) + self.run_seed() + self.run_seed(now=lambda: time.time() - 2 * 3600) # the desktop's clock is behind: made in the future + self.assertIsNone(self.files('m2')) + self.assertTrue(any('clocks disagree' in s for s in self.said), self.said) + self.run_seed() + self.assertIsNotNone(self.files('m2')) + old = seed.current(self.root, 'm2')[0] + self.run_seed(now=lambda: time.time() + 1001 * 3600) # over max_age_hours, and the host's is too + self.assertIsNone(self.files('m2')) + self.assertTrue(os.path.isdir(old)) # dropped, but a job still reading it keeps it + self.assertTrue(any('older than 1000 h' in s for s in self.said), self.said) + self.run_seed() + self.config([SEEDS[1]]) + self.run_seed() + self.assertIsNone(self.files('m2')) + self.assertTrue(any('no longer one of the pool' in s for s in self.said), self.said) + + def test_host_down_keeps_each_seed_until_its_own_age_drops_it(self): + self.make('m2', {'a': b'one'}) + self.run_seed() + self.mode('down') + self.assertEqual(self.run_seed(now=lambda: time.time() + 999 * 3600), 0) + self.assertEqual(self.files('m2'), {'a': b'one'}) + self.assertTrue(any('no list' in s for s in self.said), self.said) + self.run_seed(now=lambda: time.time() + 1001 * 3600) + self.assertIsNone(self.files('m2')) + + def test_a_run_that_stopped_half_way_and_bad_links_or_labels(self): + self.make('m2', {'a': b'one'}) + self.run_seed() + os.makedirs(os.path.join(self.root, '.new', 'm2.123', 'data')) # a torn unpack + os.symlink('.v/m2.999/data', os.path.join(self.root, '.m2.link')) # a link made, not yet swapped in + os.symlink('/etc', os.path.join(self.root, 'npm')) # not a link this program makes + os.unlink(os.path.join(seed.current(self.root, 'm2')[0], 'label.json')) + self.run_seed() + self.assertEqual(os.listdir(os.path.join(self.root, '.new')), ['.lock']) + self.assertFalse(os.path.lexists(os.path.join(self.root, 'npm'))) + self.assertTrue(os.path.isdir('/etc')) + self.assertEqual(self.files('m2'), {'a': b'one'}) # fetched again in the same run + self.assertEqual(len(self.fetches()), 2) + self.assertTrue(any('label is missing' in s for s in self.said), self.said) + + def test_one_run_at_a_time(self): + os.makedirs(os.path.join(self.root, '.new')) + with open(os.path.join(self.root, '.new', '.lock'), 'a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + self.assertEqual(self.run_seed(), 1) + self.assertEqual(self.fetches(), []) + + def test_bad_settings_are_exit_2(self): + bad = os.path.join(self.t, 'desk.conf') + with open(bad, 'w') as f: + f.write('POOL=x\n') + os.chmod(bad, 0o666) + with mock.patch('sys.stderr', io.StringIO()) as err: + self.assertEqual(seed.main(['--config', bad, 'run']), 2) + self.assertIn('fork-fleet-desk-seed:', err.getvalue()) + + if __name__ == '__main__': unittest.main() diff --git a/.github/scripts/fork-fleet-seed.service b/.github/scripts/fork-fleet-seed.service new file mode 100644 index 000000000000..901eee69a986 --- /dev/null +++ b/.github/scripts/fork-fleet-seed.service @@ -0,0 +1,44 @@ +# Fork only: a desk pool's warm caches, the desktop's side (fork-fleet-desk-seed.py run; .github/fork-fleet.md, +# "Seeds"). The desk kit installs it as /etc/systemd/system/t3code-fleet-seed.service with its timer; not as +# fork-fleet-*, the runner units' prefix. Safe to start by hand: a second run while one runs exits at once (a lock). +# +# Root, so the seeds are root's and no job can change one; sandboxed: it may write only the seeds' folder +# (StateDirectory makes it, 0755; new copies are unpacked inside it, so a rename never crosses a mount), reads the +# desk link's settings and key in /etc/fork-fleet, reaches the network (ssh to the always-on host only, with the +# agent's pinned host key) and has a memory, CPU and task limit. +[Unit] +Description=t3code fork fleet: warm caches for a desk pool's jobs +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/bin/python3 -I /opt/fork-fleet/fork-fleet-desk-seed.py run +TimeoutStartSec=3h +Nice=10 +IOSchedulingClass=idle +UMask=0022 +StateDirectory=fork-fleet/seeds +StateDirectoryMode=0755 +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +PrivateDevices=yes +NoNewPrivileges=yes +RestrictSUIDSGID=yes +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +RestrictNamespaces=yes +CapabilityBoundingSet=CAP_DAC_OVERRIDE +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes +LockPersonality=yes +RestrictRealtime=yes +SystemCallArchitectures=native +MemoryMax=512M +MemorySwapMax=0 +CPUQuota=100% +TasksMax=32 diff --git a/.github/scripts/fork-fleet-seed.timer b/.github/scripts/fork-fleet-seed.timer new file mode 100644 index 000000000000..8cb4bfb1424f --- /dev/null +++ b/.github/scripts/fork-fleet-seed.timer @@ -0,0 +1,13 @@ +# Fork only: runs t3code-fleet-seed.service every hour, at a random minute in its first 10, and after a wake or a +# boot when a run was missed (a desktop that slept). A run with nothing new asks the always-on host for one short +# list and fetches nothing; each seed's every_hours (fork-fleet.json) paces the downloads. +[Unit] +Description=t3code fork fleet: hourly warm caches for a desk pool + +[Timer] +OnCalendar=hourly +RandomizedDelaySec=10min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/fork-kit/desk/README.md b/fork-kit/desk/README.md index 01e97eb99b5f..57f2268d951e 100644 --- a/fork-kit/desk/README.md +++ b/fork-kit/desk/README.md @@ -48,6 +48,9 @@ the refresh timer; `check` says FAIL for `template` until it is there. `check` only reads: the installed files against their manifest, every wall of this OS (`fork-fleet-walls.py describe`), the agent's settings, the units, the template, the runner users and IPv4 forwarding. +The seed timer (`t3code-fleet-seed.timer`) fetches the pool's warm caches, if its fleet config lists any, into +`/var/lib/fork-fleet/seeds`, which jobs see read-only ([fork-fleet.md](../../.github/fork-fleet.md), "Seeds"). + ## Windows (WSL2) [windows/start-desk.ps1](windows/start-desk.ps1) does the Windows side and then this kit inside one WSL2 distro. @@ -80,8 +83,8 @@ stay (`wsl --unregister DISTRO` removes the distro). before anything changes. Remove the older (by hand) install first, the runbook's steps in reverse. - **Empty:** `check` before an install says FAIL and writes nothing; `uninstall` with no record does nothing. -Uninstall stops every runner, removes the namespaces and the `fork_fleet` nft table, the kit's units, files and -settings, and only the users it made itself. +Uninstall stops every runner, removes the namespaces and the `fork_fleet` nft table, the kit's units, files, +settings and seeds, and only the users it made itself. ## Why one flat folder diff --git a/fork-kit/desk/sbin/desk-install b/fork-kit/desk/sbin/desk-install index 59e560a91f11..ba3dede670ad 100755 --- a/fork-kit/desk/sbin/desk-install +++ b/fork-kit/desk/sbin/desk-install @@ -8,8 +8,8 @@ install reads the `desk` section of the machine file, makes the runner users, stages the fleet files of one release (launcher, walls, desk agent, gate hook and the fleet config), checks the staged copy, stops the agent, moves each file into /opt/fork-fleet, writes MANIFEST.sha256 and then SOURCE (last), writes the settings in /etc/fork-fleet and -the units, makes the desk key once, and starts the agent and the refresh timer. Safe to run again and to stop at -any step: a stopped run leaves SOURCE naming the old release (check says what differs) and the next run ends it. +the units, makes the desk key once, and starts the agent and the refresh and seed timers. Safe to run again and to +stop at any step: a stopped run leaves SOURCE naming the old release (check says what differs) and the next run ends it. """ import argparse, contextlib, hashlib, importlib.util, json, os, re, shutil, subprocess, sys, tempfile @@ -27,6 +27,7 @@ FILES = {'fork-fleet-launch.sh': '.github/scripts/fork-fleet-launch.sh', 'fork-fleet-walls.py': '.github/scripts/fork-fleet-walls.py', 'fork-fleet-desk-agent.py': '.github/scripts/fork-fleet-desk-agent.py', 'fork-fleet-desk-wire.py': '.github/scripts/fork-fleet-desk-wire.py', + 'fork-fleet-desk-seed.py': '.github/scripts/fork-fleet-desk-seed.py', 'hook.sh': '.github/scripts/fork-fleet-hook.sh', 'fork-fleet.py': '.github/scripts/fork-fleet.py', 'fork-fleet.json': '.github/fork-fleet.json', 'desk-install': 'fork-kit/desk/sbin/desk-install', 'machine_file.py': 'fork-kit/common/machine_file.py', 'README.md': 'fork-kit/desk/README.md', @@ -34,8 +35,12 @@ FILES = {'fork-fleet-launch.sh': '.github/scripts/fork-fleet-launch.sh', WALLS = '.github/scripts/fork-fleet-walls' # copied whole: every OS's walls file and fixture UNITS = {'fork-fleet-desk-agent.service': '.github/scripts/fork-fleet-desk-agent.service', 't3code-fleet-refresh.service': '.github/scripts/fork-fleet-refresh.service', - 't3code-fleet-refresh.timer': '.github/scripts/fork-fleet-refresh.timer'} + 't3code-fleet-refresh.timer': '.github/scripts/fork-fleet-refresh.timer', + 't3code-fleet-seed.service': '.github/scripts/fork-fleet-seed.service', + 't3code-fleet-seed.timer': '.github/scripts/fork-fleet-seed.timer'} AGENT, TIMER, REFRESH = 'fork-fleet-desk-agent.service', 't3code-fleet-refresh.timer', 't3code-fleet-refresh.service' +SEED_TIMER, SEED = 't3code-fleet-seed.timer', 't3code-fleet-seed.service' +SEEDS = WORK + '/seeds' # the warm caches (fork-fleet-desk-seed.py); jobs see them read-only (ALLOW_PATHS) FOREIGN = (OPT + '/fork-fleet-launch.sh', ETC + '/launch.conf', ETC + '/desk.conf', KEY, *(f'{UNIT_DIR}/{u}' for u in UNITS)) CMD = {k: [k] for k in ('useradd', 'userdel', 'loginctl', 'systemctl', 'sysctl', 'ssh-keygen', 'ip', 'nft', 'bash', @@ -252,7 +257,7 @@ def make_users(users, state): def settings(m, users): known = (m['host'] if m['port'] == 22 else f'[{m["host"]}]:{m["port"]}') + ' ' + m['hostKey'] - files = {ETC + '/launch.conf': (f'RUNNER_USERS={",".join(users)}\n' + files = {ETC + '/launch.conf': (f'RUNNER_USERS={",".join(users)}\nALLOW_PATHS={SEEDS}\n' + (f'RESERVE_MB={m["reserveMb"]}\n' if m['reserveMb'] is not None else '') + (f'NET_BASE={m["netBase"]}\n' if m['netBase'] else '') + (f'FREE_COMMAND={m["freeCommand"]}\n' if m['freeCommand'] else ''), 0o600), @@ -334,7 +339,7 @@ def install(a, stop_after=None): if r.returncode: raise Refused(f'the desk agent refuses its settings (not started): {r.stderr.strip()[:300]}') run('systemctl', 'daemon-reload') - run('systemctl', 'enable', '--now', TIMER) + run('systemctl', 'enable', '--now', TIMER, SEED_TIMER) if not os.path.exists(p(OPT + '/template/runner-template/.fork-fleet-version')): run(p(OPT + '/fork-fleet-launch.sh'), 'prune', check=False) # makes the launch lock refresh opens run('systemctl', 'start', '--no-block', REFRESH) # the first runner template, in the background @@ -369,7 +374,7 @@ def check(): row(False, 'walls', f'exit {walls.returncode}: {walls.stderr.strip()[:200]}') r = py(p(OPT + '/fork-fleet-desk-agent.py'), 'check') row(r.returncode == 0, 'agent-settings', (r.stdout or r.stderr).strip()[:200]) - for unit in (AGENT, TIMER): + for unit in (AGENT, TIMER, SEED_TIMER): on = run('systemctl', 'is-active', '--quiet', unit, check=False).returncode == 0 row(on and run('systemctl', 'is-enabled', '--quiet', unit, check=False).returncode == 0, 'unit', f'{unit} enabled and active') @@ -429,7 +434,7 @@ def uninstall(): if state is None: say(f'no {STATE}: nothing here was installed by this kit (nothing changed)') return 0 - for unit in (AGENT, TIMER, REFRESH): + for unit in (AGENT, TIMER, REFRESH, SEED_TIMER, SEED): run('systemctl', 'disable', '--now', unit, check=False) launcher = p(OPT + '/fork-fleet-launch.sh') if os.path.exists(launcher): diff --git a/fork-kit/desk/tests/install.test.py b/fork-kit/desk/tests/install.test.py index 2de38ec65819..614834371c4c 100644 --- a/fork-kit/desk/tests/install.test.py +++ b/fork-kit/desk/tests/install.test.py @@ -64,6 +64,7 @@ def put(n, items): ' print("bad settings", file=sys.stderr); sys.exit(2)\n' 'print("settings ok")\n', '.github/scripts/fork-fleet-desk-wire.py': '# wire\n', + '.github/scripts/fork-fleet-desk-seed.py': '# seed\n', '.github/scripts/fork-fleet-hook.sh': '#!/bin/bash\nexit 0\n', '.github/scripts/fork-fleet-job-paths.txt': '/usr\n', '.github/fork-fleet.json': json.dumps({'pools': [{'name': 'desktop'}, {'name': 'vm'}]}), @@ -74,6 +75,8 @@ def put(n, items): '.github/scripts/fork-fleet-desk-agent.service': '# agent unit\n[Service]\n', '.github/scripts/fork-fleet-refresh.service': '# refresh unit\n[Service]\n', '.github/scripts/fork-fleet-refresh.timer': '# refresh timer\n[Timer]\n', + '.github/scripts/fork-fleet-seed.service': '# seed unit\n[Service]\n', + '.github/scripts/fork-fleet-seed.timer': '# seed timer\n[Timer]\n', } @@ -194,7 +197,8 @@ def test_fresh_install_files_settings_users_key_units(self): self.assertFalse([n for n in os.listdir(self.path('/opt/fork-fleet')) if n.startswith('.kit-stage')]) launch = self.read('/etc/fork-fleet/launch.conf') self.assertTrue(launch.startswith(self.m.HEADER)) - self.assertIn('RUNNER_USERS=fork-runner-1,fork-runner-2\n', launch) + self.assertIn('RUNNER_USERS=fork-runner-1,fork-runner-2\nALLOW_PATHS=/var/lib/fork-fleet/seeds\n', launch) + self.assertTrue(os.path.isfile(self.path('/opt/fork-fleet/fork-fleet-desk-seed.py'))) self.assertNotIn('RESERVE_MB', launch) self.assertIn('POOL=desktop\nHOST=fleet-host.example\nPORT=22\nUSER=fleet-desk\n', self.read('/etc/fork-fleet/desk.conf')) @@ -215,7 +219,8 @@ def test_fresh_install_files_settings_users_key_units(self): ['fork-runner-1', 'fork-runner-2']) order = [c for c in calls if c.startswith(('systemctl', 'launch'))] self.assertEqual(order, ['systemctl stop fork-fleet-desk-agent.service', 'systemctl daemon-reload', - 'systemctl enable --now t3code-fleet-refresh.timer', 'launch prune', + 'systemctl enable --now t3code-fleet-refresh.timer t3code-fleet-seed.timer', + 'launch prune', 'systemctl start --no-block t3code-fleet-refresh.service', 'systemctl enable fork-fleet-desk-agent.service', 'systemctl restart fork-fleet-desk-agent.service']) @@ -355,7 +360,8 @@ def test_reserve_and_port_reach_the_settings(self): self.machine({'version': 1, 'desk': {**MACHINE['desk'], 'reserveMb': 4096, 'port': 2222, 'slots': 1, 'netBase': '10.77'}}) self.assertEqual(self.install()[0], 0) - self.assertIn('RUNNER_USERS=fork-runner-1\nRESERVE_MB=4096\nNET_BASE=10.77\n', + self.assertIn('RUNNER_USERS=fork-runner-1\nALLOW_PATHS=/var/lib/fork-fleet/seeds\nRESERVE_MB=4096\n' + 'NET_BASE=10.77\n', self.read('/etc/fork-fleet/launch.conf')) self.assertTrue(self.read('/etc/fork-fleet/desk-known_hosts').endswith( f'[fleet-host.example]:2222 {HOST_KEY}\n')) @@ -415,7 +421,8 @@ def test_check_names_each_failure(self): self.assertEqual(code, 1) for line in ('FAIL kit /opt/fork-fleet/hook.sh: differs', 'FAIL vsock closed: open', 'FAIL agent-settings bad settings', 'FAIL unit fork-fleet-desk-agent.service', - 'PASS unit t3code-fleet-refresh.timer', 'FAIL template', 'FAIL runner-user fork-runner-1', + 'PASS unit t3code-fleet-refresh.timer', 'PASS unit t3code-fleet-seed.timer', + 'FAIL template', 'FAIL runner-user fork-runner-1', 'FAIL ip-forward'): self.assertIn(line, out) self.assertTrue(out.endswith('8 FAIL\n'), out) @@ -435,11 +442,13 @@ def test_uninstall_leaves_nothing_of_its_own(self): self.assertEqual(self.install()[0], 0) self.write(os.path.join(self.fakes, 'active'), 'desktop-1\n') self.write(self.path('/var/lib/fork-fleet/runners/x'), '') + self.write(self.path('/var/lib/fork-fleet/seeds/.v/m2.1/data/x'), '') self.clear_calls() code, out, err = self.run_main('uninstall') self.assertEqual(code, 0, err) calls = self.calls() - for c in ('systemctl disable --now fork-fleet-desk-agent.service', 'launch active', 'launch stop desktop-1', + for c in ('systemctl disable --now fork-fleet-desk-agent.service', + 'systemctl disable --now t3code-fleet-seed.timer', 'launch active', 'launch stop desktop-1', 'launch prune', 'ip netns delete fork-fleet-fork-runner-1', 'nft delete table inet fork_fleet', 'userdel fork-runner-1', 'userdel fork-runner-2'): self.assertIn(c, calls) From bfe2987dd50d1f627194b08275021e71a51e548e Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:50:22 +0000 Subject: [PATCH 2/6] fork fleet: desktop seed fetch fix round 1 (D11b) - the desktop takes only the names its own installed fleet config lists for its pool, and every_hours, max_age_hours and max_mb from it (max_mb: the smaller of its own and the host's); the host can no longer raise them - ssh's stderr keeps only its last 4 KB; the host's list is read up to 1 MB - after unpacking, a walk refuses a link that leads out of the seed and any entry that is not a folder, file or link, and makes everything readable by jobs (0600 cache files) - an odd trailer (a MADE int() cannot read) is a torn copy, never a crash - members' sizes and count limited (sparse members, many tiny ones) - a leftover .NAME.link is removed at the start of each run - seeds come from the host's per-pool folder (D11); SKEW names the desktop's clock tolerance - the unit's sandbox lines are marked safety-critical Co-Authored-By: Claude Opus 5.5 --- .github/fork-fleet.md | 19 +- .github/scripts/fork-fleet-desk-seed.py | 184 ++++++++++++++----- .github/scripts/fork-fleet-desk-seed.test.py | 112 ++++++++++- .github/scripts/fork-fleet-seed.service | 2 + 4 files changed, 255 insertions(+), 62 deletions(-) diff --git a/.github/fork-fleet.md b/.github/fork-fleet.md index 68e82be6663d..77e778db8d2b 100644 --- a/.github/fork-fleet.md +++ b/.github/fork-fleet.md @@ -715,11 +715,20 @@ size and SHA-256; anything else is refused. One stream per pool at a time; a desktop that takes nothing for two minutes is cut off and the next one may go. The desktop's side is the same script's `run`, hourly from -`t3code-fleet-seed.timer` (the desk kit installs both). Each new seed is -unpacked with Python's tar data filter, checked against the trailer, and -swapped in by a rename; jobs see `/var/lib/fork-fleet/seeds/NAME` read-only -(the launcher's `ALLOW_PATHS`), copy what they need from it when it is there, -and run cold when it is not. The state table (a torn download, the host down, +`t3code-fleet-seed.timer` (the desk kit installs both). It takes only the +names its own installed fleet config lists for its pool, and holds them to +its own `every_hours` and `max_age_hours` and the smaller of its own and the +host's `max_mb`: a host cannot make a desktop take more. Each new seed is +unpacked with Python's tar data filter (members' sizes and count limited +too), checked against the trailer, checked again for links that lead out of +it, made readable by every job (a cache's 0600 files too), and swapped in by a +rename; ssh's stderr and the list are read with a limit. Jobs see +`/var/lib/fork-fleet/seeds/NAME` read-only (the launcher's `ALLOW_PATHS`; +the unpacking and old copies beside it, `.new` and `.v`, are visible too and +hold the same data), copy what they need from it when it is there, and run +cold when it is not. A replaced copy is kept 24 hours for a job still reading +it; there is no free-space check before unpacking (a full disk ends that seed's +fetch and keeps the old one). The state table (a torn download, the host down, clocks that disagree, a run that stopped half-way) is in the script's header. ## Owner steps on the repository diff --git a/.github/scripts/fork-fleet-desk-seed.py b/.github/scripts/fork-fleet-desk-seed.py index f4b2a4327a26..15ad764cc0cc 100644 --- a/.github/scripts/fork-fleet-desk-seed.py +++ b/.github/scripts/fork-fleet-desk-seed.py @@ -15,8 +15,10 @@ stderr line is the trailer `seed NAME BYTES SHA256 MADE` (MADE: the file's mtime, epoch seconds) python3 -I fork-fleet-desk-seed.py [--config FILE] run the desktop, as root, hourly (t3code-fleet-seed.timer), with the agent's settings (FILE, default - its CONF). Each new seed is unpacked with tarfile's data filter into a folder of its own, checked - against the trailer, flushed, and swapped in by rename: SEEDS/NAME is a link to + its CONF). Only names its own installed fleet config lists for its POOL, held to its own + every_hours and max_age_hours and the smaller of its own and the host's max_mb. Each new seed is + unpacked with tarfile's data filter into a folder of its own, checked against the trailer, walked + (a link out of it is refused; all made readable), flushed, and swapped in by rename: SEEDS/NAME is a link to .v/NAME.STAMP/data, root's (folders 0755, files no one else may write), unpacked in .new (one folder: a rename never crosses a mount of the unit's sandbox). Jobs see SEEDS read-only (the launcher's ALLOW_PATHS): a job uses /var/lib/fork-fleet/seeds/NAME when it is there and runs @@ -42,7 +44,7 @@ Standard library only. """ import argparse, fcntl, hashlib, importlib.util, json, os, re, shutil, signal, stat, subprocess, sys, tarfile -import tempfile, threading, time +import threading, time sys.dont_write_bytecode = True HERE = os.path.dirname(os.path.realpath(__file__)) @@ -61,6 +63,7 @@ def module(name, file): NAME = re.compile(r'[a-z0-9][a-z0-9_-]{0,39}') LINK = re.compile(r'\.v/((?P[a-z0-9][a-z0-9_-]{0,39})\.[0-9]{1,20})/data') LIST_WAIT, FETCH_WAIT = 30, 30 * 60 # seconds for the list, and for one seed's whole stream +LIST_MOST, ERR_KEEP, MEMBERS = 1 << 20, 4096, 200_000 # the list's bytes, ssh's stderr kept, a seed's tar members KEEP_OLD, SKEW = 24 * 3600, 3600 # SKEW: how far the host's clock may be ahead of the desktop's PIECE, STALL, AHEAD = 1 << 20, 120, 300 # AHEAD: seconds a file's date may be ahead of now (a clock set back) ITEM = {'everyHours': 'every_hours', 'maxAgeHours': 'max_age_hours', 'maxMb': 'max_mb'} @@ -214,20 +217,52 @@ def ssh(agent, conf, *words): return agent.ssh_argv(conf)[:-2] + list(words) # the agent's ssh, another command than `serve POOL` +class Tail(threading.Thread): + """Reads a pipe to its end and keeps only its last ERR_KEEP bytes (ssh's stderr: the host may write without + end; the trailer is its last line).""" + + def __init__(self, f): + super().__init__(daemon=True) + self.f, self.data = f, b'' + self.start() + + def run(self): + while chunk := os.read(self.f.fileno(), 65536): + self.data = (self.data + chunk)[-ERR_KEEP:] + + def text(self): + self.join(5) + return ' '.join(self.data.decode('utf-8', 'replace').split())[-300:] or 'nothing on stderr' + + def listing(agent, conf): - """{name: item} of the host's list; Torn when it cannot be had or is not well formed.""" - with tempfile.TemporaryFile() as err: - try: - r = subprocess.run(ssh(agent, conf, 'seeds'), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=err, - env=agent.ENV, timeout=LIST_WAIT, start_new_session=True) - except subprocess.TimeoutExpired: - raise Torn(f'no list in {LIST_WAIT} s') from None - except OSError as e: - raise Torn(f'no ssh: {e}') from None - if r.returncode: - raise Torn(f'no list (exit {r.returncode}): {tail(err)}') + """{name: item} of the host's list (at most LIST_MOST bytes); Torn when it cannot be had or is not well formed.""" + try: + p = subprocess.Popen(ssh(agent, conf, 'seeds'), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=agent.ENV, start_new_session=True) + except OSError as e: + raise Torn(f'no ssh: {e}') from None + late = threading.Event() + timer = threading.Timer(LIST_WAIT, lambda: (late.set(), p.kill())) + timer.start() + out = b'' try: - got = json.loads(r.stdout) + err = Tail(p.stderr) + while len(out) <= LIST_MOST and (chunk := os.read(p.stdout.fileno(), 65536)): + out += chunk + finally: + timer.cancel() + p.kill() + code = p.wait() + p.stdout.close() + if late.is_set(): + raise Torn(f'no list in {LIST_WAIT} s') + if len(out) > LIST_MOST: + raise Torn(f'the host\'s list is above {LIST_MOST} bytes') + if code: + raise Torn(f'no list (exit {code}): {err.text()}') + try: + got = json.loads(out, parse_constant=no_constant) assert isinstance(got, dict) and got.get('v') == 1 and isinstance(got.get('seeds'), list) out = {} for s in got['seeds']: @@ -240,43 +275,78 @@ def listing(agent, conf): raise Torn('the host\'s list is not well formed') from None +def no_constant(name): + raise ValueError(f'{name} is not JSON') + + def whole(x): return isinstance(x, int) and not isinstance(x, bool) and x >= 0 -def tail(f): - f.seek(0) - return ' '.join(f.read()[-2000:].decode('utf-8', 'replace').split())[-300:] or 'nothing on stderr' +def guarded(most): + """tarfile's data filter, and a limit on what the tar says it holds: members' sizes together at most MOST + bytes (a sparse member is small in the stream, not on disk) and at most MEMBERS members (each is kept in memory).""" + seen = [0, 0] + + def check(member, path): + seen[0] += 1 + seen[1] += member.size + if seen[0] > MEMBERS or seen[1] > most: + raise Torn(f'the tar holds more than {MEMBERS} members or {most >> 20} MB (max_mb)') + return tarfile.data_filter(member, path) + return check + + +def settle(data): + """After unpacking: every entry a folder, a plain file or a link that stays inside DATA (a check of its own, + besides the data filter, which some Python releases let links slip past); folders and files readable by all + (jobs run as other users; a cache may hold 0600 files), writable by root only. Torn otherwise.""" + top = os.path.realpath(data) + for d, dirs, names in os.walk(data): + for n in dirs + names: + path = os.path.join(d, n) + st = os.lstat(path) + if stat.S_ISLNK(st.st_mode): + if os.path.commonpath([top, os.path.realpath(path)]) != top: + raise Torn(f'{os.path.relpath(path, data)} is a link that leads out of the seed') + elif stat.S_ISDIR(st.st_mode): + os.chmod(path, (st.st_mode & 0o755) | 0o555) + elif stat.S_ISREG(st.st_mode): + os.chmod(path, (st.st_mode & 0o755) | 0o444) + else: + raise Torn(f'{os.path.relpath(path, data)} is not a folder, a file or a link') def fetch(agent, conf, item, root, now): - """The seed `item` into a new version, swapped in; its label. Torn (or OSError, TarError) leaves the old one.""" - name, stamp = item['name'], time.time_ns() + """The seed `item` (its maxMb: the smaller of the host's and the desktop's own) into a new version, swapped in; + its label. Torn (or OSError, TarError) leaves the old one.""" + name, stamp, most = item['name'], time.time_ns(), item['maxMb'] << 20 new = os.path.join(root + '/.new', f'{name}.{stamp}') os.mkdir(new, 0o755) os.mkdir(os.path.join(new, 'data'), 0o755) - with tempfile.TemporaryFile() as err: - p = subprocess.Popen(ssh(agent, conf, 'seed', name), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, - stderr=err, env=agent.ENV, start_new_session=True) - timer = threading.Timer(FETCH_WAIT, p.kill) - timer.start() - try: - got = Summed(p.stdout, item['maxMb'] << 20) - with tarfile.open(fileobj=got, mode='r|') as t: - t.extractall(os.path.join(new, 'data'), filter='data') - while got.read(PIECE): # the tar's end blocks and padding count in the trailer's size and sha256 - pass - code = p.wait(timeout=30) - finally: - timer.cancel() - p.kill() - p.wait() - p.stdout.close() - err.seek(0) - last = (err.read()[-4096:].decode('utf-8', 'replace').strip().splitlines() or [''])[-1].split() - if code or last[:4] != ['seed', name, str(got.size), got.sha.hexdigest()] or len(last) != 5 or \ - not last[4].isdigit(): - raise Torn(f'exit {code}, {got.size} bytes, the trailer does not match: {tail(err)}') + p = subprocess.Popen(ssh(agent, conf, 'seed', name), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=agent.ENV, start_new_session=True) + timer = threading.Timer(FETCH_WAIT, p.kill) + timer.start() + try: + err = Tail(p.stderr) + got = Summed(p.stdout, most) + with tarfile.open(fileobj=got, mode='r|') as t: + t.extractall(os.path.join(new, 'data'), filter=guarded(most)) + while got.read(PIECE): # the tar's end blocks and padding count in the trailer's size and sha256 + pass + code = p.wait(timeout=30) + finally: + timer.cancel() + p.kill() + p.wait() + p.stdout.close() + err.join(5) + last = (err.data.decode('utf-8', 'replace').strip().splitlines() or [''])[-1].split() + if code or len(last) != 5 or last[:4] != ['seed', name, str(got.size), got.sha.hexdigest()] or \ + not re.fullmatch(r'[0-9]{1,20}', last[4]): + raise Torn(f'exit {code}, {got.size} bytes, the trailer does not match: {err.text()}') + settle(os.path.join(new, 'data')) label = {'v': 1, 'name': name, 'bytes': got.size, 'sha256': got.sha.hexdigest(), 'made': int(last[4]), 'fetched': int(now()), 'maxAgeHours': item['maxAgeHours']} with open(os.path.join(new, 'label.json'), 'x', encoding='utf-8') as f: @@ -330,9 +400,22 @@ def drop(root, name, version, now): remove(os.path.join(root, name)) -def run(agent, conf, root=SEEDS, now=time.time, say=print): - if not hasattr(tarfile, 'data_filter'): +def own_seeds(conf, config): + """{name: seed} of this desktop's pool in its own installed fleet config: the only names it takes, and the + every_hours, max_age_hours and max_mb it holds them to (the host's list never raises them).""" + try: + return pool_seeds(conf['POOL'], config) + except Refused as e: + raise Torn(f'its own {e}') from None + + +def run(agent, conf, root=SEEDS, now=time.time, say=print, config=CONFIG): + if not hasattr(tarfile, 'data_filter'): # releases with known filter bypasses pass this: settle() checks again raise agent.Bad('this Python has no tarfile data filter (3.12, or 3.8.17 and later): no seeds') + try: + own = own_seeds(conf, config) + except Torn as e: + raise agent.Bad(f'{e}: no seeds') from None os.umask(0o022) for d in (root, root + '/.new', root + '/.v'): # root: made by the unit (StateDirectory) or here os.makedirs(d, 0o755, exist_ok=True) @@ -345,8 +428,13 @@ def run(agent, conf, root=SEEDS, now=time.time, say=print): for e in os.listdir(root + '/.new'): # a run that stopped half-way if e != '.lock': remove(os.path.join(root + '/.new', e)) + for e in os.listdir(root): # a link made but not yet swapped in + if e.startswith('.') and e.endswith('.link'): + remove(os.path.join(root, e)) try: - items = listing(agent, conf) + host = listing(agent, conf) + items = {n: {**host[n], 'everyHours': s['every_hours'], 'maxAgeHours': s['max_age_hours'], + 'maxMb': min(host[n]['maxMb'], s['max_mb'])} for n, s in own.items() if n in host} except Torn as e: items = None say(f'seeds: {e}: each seed stays until its age drops it') @@ -354,11 +442,11 @@ def run(agent, conf, root=SEEDS, now=time.time, say=print): version, label = current(root, name) if label is None: why = 'its label is missing or cannot be read (fetched again now)' - elif items is not None and name not in items: + elif name not in own: why = 'no longer one of the pool\'s seeds' else: age = now() - label['made'] - hours = items[name]['maxAgeHours'] if items else label['maxAgeHours'] + hours = own[name]['max_age_hours'] why = (f'older than {hours} h (max_age_hours)' if age > hours * 3600 else 'made in the future (the clocks disagree)' if age < -SKEW else None) if why: @@ -380,7 +468,7 @@ def run(agent, conf, root=SEEDS, now=time.time, say=print): try: got = fetch(agent, conf, item, root, now) say(f'seed {name}: {got["bytes"] >> 20} MB, sha256 checked, swapped in') - except (Torn, OSError, tarfile.TarError, subprocess.SubprocessError) as e: + except (Torn, OSError, ValueError, tarfile.TarError, subprocess.SubprocessError) as e: for e2 in os.listdir(root + '/.new'): if e2 != '.lock': remove(os.path.join(root + '/.new', e2)) diff --git a/.github/scripts/fork-fleet-desk-seed.test.py b/.github/scripts/fork-fleet-desk-seed.test.py index 6ab15a803180..5b05dbf3bf21 100644 --- a/.github/scripts/fork-fleet-desk-seed.test.py +++ b/.github/scripts/fork-fleet-desk-seed.test.py @@ -44,7 +44,12 @@ def load(name, file): print('ssh: connect to host h port 22: No route to host', file=sys.stderr) sys.exit(255) kw = dict(config={t!r} + '/fork-fleet.json', files={t!r} + '/files') -if words[0] == 'seed' and mode in ('cut', 'trailer'): +if words[0] == 'seeds' and mode == 'biglist': + os.write(1, b' ' * (2 << 20)) +if mode == 'noisy': + for _ in range(64): + os.write(2, b'x' * (1 << 16) + b'\n') +if words[0] == 'seed' and mode in ('cut', 'trailer', 'odd'): with open({t!r} + '/sent', 'w+b') as out: err = io.StringIO() code = seed.offer({pool!r}, ' '.join(words), {t!r} + '/run', out=out.fileno(), err=err, **kw) @@ -54,7 +59,10 @@ def load(name, file): os.write(1, data[:len(data) // 2]) sys.exit(255) os.write(1, data) - print(err.getvalue().rsplit(' ', 2)[0] + ' ' + '0' * 64 + ' 1', file=sys.stderr) # a wrong sha256 + if mode == 'odd': # a MADE that str.isdigit() takes but int() does not + print(err.getvalue().rsplit(' ', 1)[0] + ' \u00b2', file=sys.stderr) + else: + print(err.getvalue().rsplit(' ', 2)[0] + ' ' + '0' * 64 + ' 1', file=sys.stderr) # a wrong sha256 sys.exit(code) sys.exit(seed.offer({pool!r}, ' '.join(words), {t!r} + '/run', **kw)) ''' @@ -74,13 +82,13 @@ def config(self, seeds): with open(os.path.join(self.t, 'fork-fleet.json'), 'w', encoding='utf-8') as f: json.dump(cfg, f) - def make(self, name, members, mtime=None): + def make(self, name, members, mtime=None, mode=0o644): """FILES/POOL/NAME.tar of `members` ({path: bytes}), made as a project would: a temp file, then a rename.""" tmp = os.path.join(self.t, 'files', POOL, f'.{name}.tmp') with tarfile.open(tmp, 'w', format=tarfile.PAX_FORMAT) as tar: for path, data in members.items(): info = tarfile.TarInfo(path) - info.size, info.mode = len(data), 0o644 + info.size, info.mode = len(data), mode tar.addfile(info, io.BytesIO(data)) if mtime is not None: os.utime(tmp, (mtime, mtime)) @@ -296,13 +304,22 @@ def setUp(self): self.conf = {'SSH': ssh, 'KEY': '/nonexistent/key', 'KNOWN_HOSTS': '/nonexistent/known', 'PORT': '22', 'USER': 'fleet-desk', 'HOST': 'fleet-host.example', 'POOL': POOL} self.said = [] + self.own(SEEDS) sync = mock.patch.object(seed.os, 'sync') sync.start() self.addCleanup(sync.stop) + def own(self, seeds): + """The desktop's own installed fleet config (/opt/fork-fleet/fork-fleet.json); self.config is the host's.""" + cfg = json.loads(json.dumps(CONFIG)) + cfg['pools'][0]['seeds'] = seeds + with open(os.path.join(self.t, 'own.json'), 'w', encoding='utf-8') as f: + json.dump(cfg, f) + def run_seed(self, now=None): self.said = [] - return seed.run(agent, self.conf, self.root, now=now or time.time, say=self.said.append) + return seed.run(agent, self.conf, self.root, now=now or time.time, say=self.said.append, + config=os.path.join(self.t, 'own.json')) def mode(self, text): with open(os.path.join(self.t, 'mode'), 'w') as f: @@ -340,7 +357,7 @@ def test_fetched_checked_and_swapped_in_then_not_again(self): if not stat.S_ISLNK(st.st_mode): # a link's own mode is always 0777 and means nothing self.assertFalse(st.st_mode & 0o022, os.path.join(d, n)) label = json.load(open(os.path.join(seed.current(self.root, 'm2')[0], 'label.json'))) - self.assertEqual(label['bytes'], os.path.getsize(os.path.join(self.t, 'files', 'm2.tar'))) + self.assertEqual(label['bytes'], os.path.getsize(self.tar('m2'))) self.assertEqual(self.run_seed(), 0) # the same seed on the host: nothing fetched self.assertEqual(self.fetches(), ['seed m2']) self.assertEqual(os.listdir(os.path.join(self.root, '.new')), ['.lock']) @@ -394,6 +411,78 @@ def bigger(): self.assertEqual(self.files('m2'), {'a': b'one'}) self.assertTrue(any('max_mb' in s for s in self.said), self.said) + def test_max_mb_is_the_desktops_not_the_hosts(self): + self.config([{**SEEDS[0], 'max_mb': 3}, SEEDS[1]]) # the host lists 3; the desktop's own copy says 2 + self.make('m2', {'a': os.urandom(5 << 19)}) # 2.5 MB + self.run_seed() + self.assertIsNone(self.files('m2'), self.said) + self.assertTrue(any('max_mb' in s for s in self.said), self.said) + + def test_every_and_max_age_hours_are_the_desktops_and_unknown_names_are_ignored(self): + self.config([{**SEEDS[0], 'every_hours': 1, 'max_age_hours': 100000}, SEEDS[1], + {'name': 'extra', 'every_hours': 1, 'max_age_hours': 2, 'max_mb': 1}]) + self.make('m2', {'a': b'one'}) + self.make('extra', {'a': b'x'}) + self.run_seed() + self.assertIsNone(self.files('extra')) # not in the desktop's own list: never fetched + self.assertNotIn('seed extra', self.fetches()) + self.make('m2', {'a': b'two'}, mtime=time.time() + 5) + self.run_seed(now=lambda: time.time() + 2 * 3600) # the host says every 1 h; the desktop's own, 6 h + self.assertEqual(self.files('m2'), {'a': b'one'}) + self.mode('down') + self.run_seed(now=lambda: time.time() + 1001 * 3600) # the desktop's own max_age_hours (1000), not 100000 + self.assertIsNone(self.files('m2')) + + def test_stderr_is_never_kept_whole_and_a_huge_list_is_refused(self): + self.mode('noisy') # 4 MB on stderr before the trailer + self.make('m2', {'a': b'one'}) + with mock.patch('tempfile.TemporaryFile', side_effect=AssertionError('stderr kept on disk')): + self.assertEqual(self.run_seed(), 0) + self.assertEqual(self.files('m2'), {'a': b'one'}, self.said) + self.mode('biglist') + self.make('m2', {'a': b'two'}, mtime=time.time() + 5) + self.run_seed(now=lambda: time.time() + 7 * 3600) + self.assertTrue(any('above' in s and 'bytes' in s for s in self.said), self.said) + self.assertEqual(self.files('m2'), {'a': b'one'}) + + def test_a_0600_file_is_readable_by_jobs(self): + self.make('m2', {'cache/entry': b'x', 'cache/tool': b'#!/bin/sh\n'}, mode=0o600) + self.run_seed() + top = os.path.realpath(os.path.join(self.root, 'm2')) + for d, dirs, names in os.walk(top): + for n in dirs + names: + mode = os.lstat(os.path.join(d, n)).st_mode + self.assertEqual(mode & 0o444, 0o444, (n, oct(mode))) + self.assertFalse(mode & 0o022, (n, oct(mode))) + if stat.S_ISDIR(mode): + self.assertEqual(mode & 0o111, 0o111, (n, oct(mode))) + + def test_a_link_out_of_the_seed_is_refused_even_past_the_filter(self): + self.make('m2', {'a': b'one'}) + self.run_seed() + tmp = os.path.join(self.t, 'files', POOL, '.m2.tmp') + with tarfile.open(tmp, 'w') as tar: + for name, target in (('inside', 'a'), ('escaped', '/')): + info = tarfile.TarInfo(name) + info.type, info.linkname = tarfile.SYMTYPE, target + tar.addfile(info) + os.utime(tmp, (time.time() + 5,) * 2) + os.replace(tmp, self.tar('m2')) + with mock.patch.object(seed.tarfile, 'data_filter', lambda member, path: member): # a filter bypass + self.run_seed(now=lambda: time.time() + 7 * 3600) + self.assertEqual(self.files('m2'), {'a': b'one'}) + self.assertTrue(any('leads out of the seed' in s for s in self.said), self.said) + for d, dirs, names in os.walk(self.root): + self.assertNotIn('escaped', dirs + names, d) + + def test_an_odd_trailer_keeps_the_old_seed_and_the_run_goes_on(self): + self.torn(lambda: (self.make('m2', {'a': b'two'}, mtime=time.time() + 5), self.mode('odd')), + 'trailer does not match') + + def test_too_many_members_keep_the_old_seed(self): + with mock.patch.object(seed, 'MEMBERS', 2): + self.torn(lambda: self.make('m2', {'a': b'1', 'b': b'2', 'c': b'3'}, mtime=time.time() + 5), 'members') + def test_a_tar_that_would_leave_its_folder_writes_nothing_outside(self): bad = {'a ../ path': ('../../escaped', tarfile.REGTYPE, ''), 'a device': ('escaped', tarfile.CHRTYPE, ''), 'an absolute link': ('escaped', tarfile.SYMTYPE, '/'), @@ -401,7 +490,7 @@ def test_a_tar_that_would_leave_its_folder_writes_nothing_outside(self): 'a hard link out': ('escaped', tarfile.LNKTYPE, '/etc/hostname')} for why, (name, kind, target) in bad.items(): def evil(): - tmp = os.path.join(self.t, 'files', '.m2.tmp') + tmp = os.path.join(self.t, 'files', POOL, '.m2.tmp') with tarfile.open(tmp, 'w') as tar: # one bad member each: the filter stops at the first ok = tarfile.TarInfo('ok') ok.size = 1 @@ -410,7 +499,7 @@ def evil(): info.type, info.linkname = kind, target tar.addfile(info) os.utime(tmp, (time.time() + 5,) * 2) - os.replace(tmp, os.path.join(self.t, 'files', 'm2.tar')) + os.replace(tmp, self.tar('m2')) with self.subTest(why): shutil.rmtree(self.root, True) self.torn(evil, 'Error') @@ -431,7 +520,10 @@ def test_dropped_when_no_longer_listed_or_too_old_or_from_the_future(self): self.assertTrue(os.path.isdir(old)) # dropped, but a job still reading it keeps it self.assertTrue(any('older than 1000 h' in s for s in self.said), self.said) self.run_seed() - self.config([SEEDS[1]]) + self.config([SEEDS[1]]) # the host no longer lists it: the desktop's own list still does, it stays + self.run_seed() + self.assertIsNotNone(self.files('m2')) + self.own([SEEDS[1]]) # the desktop's own list no longer does: dropped self.run_seed() self.assertIsNone(self.files('m2')) self.assertTrue(any('no longer one of the pool' in s for s in self.said), self.said) @@ -451,11 +543,13 @@ def test_a_run_that_stopped_half_way_and_bad_links_or_labels(self): self.run_seed() os.makedirs(os.path.join(self.root, '.new', 'm2.123', 'data')) # a torn unpack os.symlink('.v/m2.999/data', os.path.join(self.root, '.m2.link')) # a link made, not yet swapped in + os.symlink('.v/npm.999/data', os.path.join(self.root, '.npm.link')) # one of a seed not fetched now os.symlink('/etc', os.path.join(self.root, 'npm')) # not a link this program makes os.unlink(os.path.join(seed.current(self.root, 'm2')[0], 'label.json')) self.run_seed() self.assertEqual(os.listdir(os.path.join(self.root, '.new')), ['.lock']) self.assertFalse(os.path.lexists(os.path.join(self.root, 'npm'))) + self.assertFalse(os.path.lexists(os.path.join(self.root, '.npm.link'))) self.assertTrue(os.path.isdir('/etc')) self.assertEqual(self.files('m2'), {'a': b'one'}) # fetched again in the same run self.assertEqual(len(self.fetches()), 2) diff --git a/.github/scripts/fork-fleet-seed.service b/.github/scripts/fork-fleet-seed.service index 901eee69a986..e1380376cfaf 100644 --- a/.github/scripts/fork-fleet-seed.service +++ b/.github/scripts/fork-fleet-seed.service @@ -6,6 +6,8 @@ # (StateDirectory makes it, 0755; new copies are unpacked inside it, so a rename never crosses a mount), reads the # desk link's settings and key in /etc/fork-fleet, reaches the network (ssh to the always-on host only, with the # agent's pinned host key) and has a memory, CPU and task limit. +# The sandbox lines below are safety-critical: they bound what a tar that slips past Python's data filter (some +# releases have known bypasses; the script also checks every link after unpacking) could reach. Do not loosen them. [Unit] Description=t3code fork fleet: warm caches for a desk pool's jobs Wants=network-online.target From 8b687649b3ec883bbe09b1fc1b10df48a618af87 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sun, 15 Nov 2026 21:35:19 +0000 Subject: [PATCH 3/6] fork fleet: seed tests: the noisy fake ssh compiles, the upgrade stops the seed units (D11b) The fake ssh template wrote a real newline inside a bytes literal, so every desktop run got a syntax error instead of a list. D8's upgrade test now names the stop call, which on this branch also stops the seed timer and service. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-fleet-desk-seed.test.py | 2 +- fork-kit/desk/tests/install.test.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/scripts/fork-fleet-desk-seed.test.py b/.github/scripts/fork-fleet-desk-seed.test.py index f6720da1d0cf..a0e73e359840 100644 --- a/.github/scripts/fork-fleet-desk-seed.test.py +++ b/.github/scripts/fork-fleet-desk-seed.test.py @@ -48,7 +48,7 @@ def load(name, file): os.write(1, b' ' * (2 << 20)) if mode == 'noisy': for _ in range(64): - os.write(2, b'x' * (1 << 16) + b'\n') + os.write(2, b'x' * (1 << 16) + b'\\n') if words[0] == 'seed' and mode in ('cut', 'trailer', 'odd'): with open({t!r} + '/sent', 'w+b') as out: err = io.StringIO() diff --git a/fork-kit/desk/tests/install.test.py b/fork-kit/desk/tests/install.test.py index 766954d627e1..90109fcc9632 100644 --- a/fork-kit/desk/tests/install.test.py +++ b/fork-kit/desk/tests/install.test.py @@ -298,7 +298,8 @@ def test_upgrade_replaces_changed_files_and_removes_gone_ones(self): self.assertIn('# new', self.read('/opt/fork-fleet/hook.sh')) self.assertEqual(os.stat(self.path('/opt/fork-fleet/fork-fleet-launch.sh')).st_ino, launcher) # unchanged calls = self.calls() - stop = 'systemctl stop t3code-fleet-refresh.timer t3code-fleet-refresh.service fork-fleet-desk-agent.service' + stop = ('systemctl stop t3code-fleet-refresh.timer t3code-fleet-refresh.service t3code-fleet-seed.timer ' + 't3code-fleet-seed.service fork-fleet-desk-agent.service') self.assertLess(calls.index(stop), calls.index('systemctl restart fork-fleet-desk-agent.service')) self.assertEqual(json.loads(self.read('/etc/fork-fleet/desk-kit.json'))['sha'], SHA2) From 223795c267837ca1bbddb9d4ce22acb44508f25c Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sun, 15 Nov 2026 23:05:59 +0000 Subject: [PATCH 4/6] fork fleet: seed fetch round 2: entries count against max_mb, one seed never stops the run (D11b) - guarded(): each member's size plus 4 KiB for it and each folder it makes, within max_mb; at most 200,000 members and folders; no member over 64 folders deep or with a name/link over 1 KiB - run(): any error in one seed is said and the run goes on; prune runs in a finally, per version - remove(): folder by folder through dir fds, no recursion (a tree of any depth) - the trailer's bytes and MADE must be the list's - a broken or pool-less own config still drops seeds past their label's maxAgeHours, then exits 2 - an older or half-installed host (no list) means no seeds: tested - desk-install check: a row for the seed service's last result - ssh's stderr pipe closed; the 0600 test fails when nothing was fetched Co-Authored-By: Claude Opus 5.5 --- .github/fork-fleet.md | 16 +- .github/scripts/fork-fleet-desk-seed.py | 247 +++++++++++++------ .github/scripts/fork-fleet-desk-seed.test.py | 169 ++++++++++++- fork-kit/desk/README.md | 3 +- fork-kit/desk/sbin/desk-install | 3 + fork-kit/desk/tests/install.test.py | 9 +- 6 files changed, 364 insertions(+), 83 deletions(-) diff --git a/.github/fork-fleet.md b/.github/fork-fleet.md index dd94a0ce8deb..819c8a6fa4c6 100644 --- a/.github/fork-fleet.md +++ b/.github/fork-fleet.md @@ -721,10 +721,20 @@ The desktop's side is the same script's `run`, hourly from names its own installed fleet config lists for its pool, and holds them to its own `every_hours` and `max_age_hours` and the smaller of its own and the host's `max_mb`: a host cannot make a desktop take more. Each new seed is -unpacked with Python's tar data filter (members' sizes and count limited -too), checked against the trailer, checked again for links that lead out of +unpacked with Python's tar data filter and a limit on what it would make on +disk (each member's size plus 4 KiB for it and for each folder it makes, all +within `max_mb`; at most 200,000 members and folders; no member more than 64 +folders deep or with a name over 1 KiB), checked against the trailer (its size +and date must be the list's too), checked again for links that lead out of it, made readable by every job (a cache's 0600 files too), and swapped in by a -rename; ssh's stderr and the list are read with a limit. Jobs see +rename; ssh's stderr and the list are read with a limit. A seed that fails in +any way is named in the log and the run goes on with the next; old copies +are pruned in every run. A host that gives no list (down, an older kit, a +half-done install) means no new seeds: each one here stays until its age +drops it. A desktop whose own fleet config is broken or lacks its pool fetches +nothing and exits 2 (`desk-install check` shows the seed service's last +result), but still drops each seed past the `max_age_hours` it was fetched +with. Jobs see `/var/lib/fork-fleet/seeds/NAME` read-only (the launcher's `ALLOW_PATHS`; the unpacking and old copies beside it, `.new` and `.v`, are visible too and hold the same data), copy what they need from it when it is there, and run diff --git a/.github/scripts/fork-fleet-desk-seed.py b/.github/scripts/fork-fleet-desk-seed.py index 5df3b3364ae3..7eac817cb4f0 100644 --- a/.github/scripts/fork-fleet-desk-seed.py +++ b/.github/scripts/fork-fleet-desk-seed.py @@ -17,7 +17,9 @@ the desktop, as root, hourly (t3code-fleet-seed.timer), with the agent's settings (FILE, default its CONF). Only names its own installed fleet config lists for its POOL, held to its own every_hours and max_age_hours and the smaller of its own and the host's max_mb. Each new seed is - unpacked with tarfile's data filter into a folder of its own, checked against the trailer, walked + unpacked with tarfile's data filter into a folder of its own (what it would make on disk held + to max_mb, ENTRY bytes counted for each member and folder; no member over DEPTH folders deep or + a name over NAME_MOST bytes), checked against the trailer (bytes and MADE the list's too), walked (a link out of it is refused; all made readable), flushed, and swapped in by rename: SEEDS/NAME is a link to .v/NAME.STAMP/data, root's (folders 0755, files no one else may write), unpacked in .new (one folder: a rename never crosses a mount of the unit's sandbox). Jobs see SEEDS read-only @@ -27,13 +29,20 @@ Each run (one at a time: a lock) meets this state: the same seed on the host (bytes and MADE) nothing is fetched a new one under every_hours after the last nothing yet - a torn copy (the stream cut or past FETCH_WAIT, a trailer that does not match, more than max_mb, a tar member - that would leave its folder or is a device) the copy is removed, the old seed stays, the next hour tries again + a torn copy (the stream cut or past FETCH_WAIT, a trailer that does not match it or the list, more than max_mb, + a tar member that would leave its folder, is a device, too deep or too long, or any other error) + the copy is removed, the old seed stays, the next hour tries again; + the other seeds go on, old versions are still pruned no seed (never fetched, the host down) jobs run cold + no list (the host down, an older kit that answers with a session or not JSON, exit 1 of a half-done install) + nothing fetched; each seed stays until its age drops it + its own fleet config broken or without POOL nothing fetched, exit 2; a seed older than its label's maxAgeHours + (its own max_age_hours when it was fetched) is still dropped older than max_age_hours, made over SKEW in the future (the clocks disagree), or no longer in the pool's list the link is removed: jobs run cold a label that is missing or unreadable the link is removed and the seed fetched again in the same run - a run that stopped half-way SEEDS/.new is emptied by the next run; a link is the old one or the new + a run that stopped half-way SEEDS/.new is emptied by the next run (folder by folder, a tree of + any depth); a link is the old one or the new a replaced version kept KEEP_OLD after it was replaced, for a job still reading it (a job runs at most the launcher's MAX_SECONDS, 1 h by default), then removed @@ -45,7 +54,7 @@ MIN_RATE bytes a second): the pool's lock is then let go. Standard library only. """ -import argparse, fcntl, hashlib, importlib.util, json, math, os, re, shutil, signal, stat, subprocess, sys +import argparse, fcntl, hashlib, importlib.util, json, math, os, re, signal, stat, subprocess, sys import tarfile, threading, time sys.dont_write_bytecode = True @@ -69,6 +78,7 @@ def module(name, file): KEEP_OLD, SKEW = 24 * 3600, 3600 # SKEW: how far the host's clock may be ahead of the desktop's PIECE, WRITE, STALL, AHEAD = 1 << 20, 64 << 10, 120, 300 # AHEAD: seconds a file's date may be ahead of now GRACE, MIN_RATE = 600, 256 << 10 # a stream's deadline: GRACE seconds plus one second per MIN_RATE bytes +DEPTH, NAME_MOST, ENTRY = 64, 1024, 4096 # a member's most folders deep and name bytes; the disk an entry costs ITEM = {'everyHours': 'every_hours', 'maxAgeHours': 'max_age_hours', 'maxMb': 'max_mb'} @@ -243,6 +253,12 @@ def run(self): while chunk := os.read(self.f.fileno(), 65536): self.data = (self.data + chunk)[-ERR_KEEP:] + def close(self): + """The pipe closed once its reader is done (a child that keeps it open past 5 s keeps it, and the thread).""" + self.join(5) + if not self.is_alive(): + self.f.close() + def text(self): self.join(5) return ' '.join(self.data.decode('utf-8', 'replace').split())[-300:] or 'nothing on stderr' @@ -259,8 +275,8 @@ def listing(agent, conf): timer = threading.Timer(LIST_WAIT, lambda: (late.set(), p.kill())) timer.start() out = b'' + err = Tail(p.stderr) try: - err = Tail(p.stderr) while len(out) <= LIST_MOST and (chunk := os.read(p.stdout.fileno(), 65536)): out += chunk finally: @@ -268,6 +284,7 @@ def listing(agent, conf): p.kill() code = p.wait() p.stdout.close() + err.close() if late.is_set(): raise Torn(f'no list in {LIST_WAIT} s') if len(out) > LIST_MOST: @@ -297,15 +314,31 @@ def whole(x): def guarded(most): - """tarfile's data filter, and a limit on what the tar says it holds: members' sizes together at most MOST - bytes (a sparse member is small in the stream, not on disk) and at most MEMBERS members (each is kept in memory).""" - seen = [0, 0] + """tarfile's data filter, and a limit on what the tar would make on disk: each member's size (a sparse member is + small in the stream, not on disk) plus ENTRY bytes for it and for each folder it makes, together at most MOST + bytes (a stream of empty files or deep paths is small but would fill the disk with entries); at most MEMBERS + members and MEMBERS folders (each is kept in memory); no member more than DEPTH folders deep (unpacking and + removing a tree goes folder by folder) or with a name or link target over NAME_MOST bytes.""" + seen, folders = [0, 0], {} # folders: {(its parent's number, its name): its number}, a name kept once def check(member, path): + parts = [x for x in member.name.split('/') if x not in ('', '.')] + if len(parts) > DEPTH: + raise Torn(f'{"/".join(parts[:3])}/... is more than {DEPTH} folders deep') + for text in (member.name, member.linkname or ''): + if len(text.encode('utf-8', 'surrogateescape')) > NAME_MOST: + raise Torn(f'a member\'s name or link target is over {NAME_MOST} bytes: {text[:80]!r}...') + new = up = 0 + for i, part in enumerate(parts if member.isdir() else parts[:-1], 1): # the folders it makes + if (up, part) not in folders: + folders[up, part] = len(folders) + 1 + new += i < len(parts) # a folder member pays ENTRY once, as a member + up = folders[up, part] seen[0] += 1 - seen[1] += member.size - if seen[0] > MEMBERS or seen[1] > most: - raise Torn(f'the tar holds more than {MEMBERS} members or {most >> 20} MB (max_mb)') + seen[1] += member.size + ENTRY * (1 + new) + if seen[0] > MEMBERS or len(folders) > MEMBERS or seen[1] > most: + raise Torn(f'the tar holds more than {MEMBERS} members or folders, or more than {most >> 20} MB ' + f'(max_mb; {ENTRY} bytes counted for each member and folder)') return tarfile.data_filter(member, path) return check @@ -341,8 +374,8 @@ def fetch(agent, conf, item, root, now): stderr=subprocess.PIPE, env=agent.ENV, start_new_session=True) timer = threading.Timer(FETCH_WAIT, p.kill) timer.start() + err = Tail(p.stderr) try: - err = Tail(p.stderr) got = Summed(p.stdout, most) with tarfile.open(fileobj=got, mode='r|') as t: t.extractall(os.path.join(new, 'data'), filter=guarded(most)) @@ -354,11 +387,14 @@ def fetch(agent, conf, item, root, now): p.kill() p.wait() p.stdout.close() - err.join(5) + err.close() last = (err.data.decode('utf-8', 'replace').strip().splitlines() or [''])[-1].split() if code or len(last) != 5 or last[:4] != ['seed', name, str(got.size), got.sha.hexdigest()] or \ not re.fullmatch(r'[0-9]{1,20}', last[4]): raise Torn(f'exit {code}, {got.size} bytes, the trailer does not match: {err.text()}') + if (got.size, int(last[4])) != (item['bytes'], item['made']): # MADE decides the age drop and every_hours + raise Torn(f'the trailer does not match the list ({got.size} bytes made {last[4]}, the list said ' + f'{item["bytes"]} made {item["made"]}): the host made a new one since') settle(os.path.join(new, 'data')) label = {'v': 1, 'name': name, 'bytes': got.size, 'sha256': got.sha.hexdigest(), 'made': int(last[4]), 'fetched': int(now()), 'maxAgeHours': item['maxAgeHours']} @@ -397,14 +433,41 @@ def current(root, name): def remove(path): + """PATH, and when it is a folder all it holds, folder by folder without recursion: a tree of any depth (one an + older release unpacked, before DEPTH) never stops a run with RecursionError. Links are removed, never followed.""" try: st = os.lstat(path) except FileNotFoundError: return - if stat.S_ISDIR(st.st_mode): - shutil.rmtree(path) - else: + if not stat.S_ISDIR(st.st_mode): os.unlink(path) + return + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC + below, fd = [], os.open(path, flags) + try: + while True: + sub = None + with os.scandir(fd) as entries: + for e in entries: + if e.is_dir(follow_symlinks=False): + sub = e.name + break + os.unlink(e.name, dir_fd=fd) + if sub is not None: # down into it; back up once it is empty + child = os.open(sub, flags, dir_fd=fd) + os.close(fd) + fd = child + below.append(sub) + elif below: + up = os.open('..', flags, dir_fd=fd) + os.close(fd) + fd = up + os.rmdir(below.pop(), dir_fd=fd) + else: + break + finally: + os.close(fd) + os.rmdir(path) def drop(root, name, version, now): @@ -420,15 +483,22 @@ def own_seeds(conf, config): return pool_seeds(conf['POOL'], config) except Refused as e: raise Torn(f'its own {e}') from None + except Exception as e: # a config the fleet's own loader trips on + raise Torn(f'its own fleet config: {e.__class__.__name__}: {e}') from None def run(agent, conf, root=SEEDS, now=time.time, say=print, config=CONFIG): + """One run (the module's docstring). A seed that fails in any way, a deep tree included, is said and the run + goes on with the next; old versions are pruned in every run. When it may fetch nothing (no data filter, its own + fleet config broken or without its pool) it still drops each seed past its age, by its label's max_age_hours, + and then says why (agent.Bad: exit 2, the unit's last result shows it).""" + cannot, own = None, None if not hasattr(tarfile, 'data_filter'): # releases with known filter bypasses pass this: settle() checks again - raise agent.Bad('this Python has no tarfile data filter (3.12, or 3.8.17 and later): no seeds') + cannot = 'this Python has no tarfile data filter (3.12, or 3.8.17 and later)' try: own = own_seeds(conf, config) except Torn as e: - raise agent.Bad(f'{e}: no seeds') from None + cannot = cannot or str(e) os.umask(0o022) for d in (root, root + '/.new', root + '/.v'): # root: made by the unit (StateDirectory) or here os.makedirs(d, 0o755, exist_ok=True) @@ -438,66 +508,99 @@ def run(agent, conf, root=SEEDS, now=time.time, say=print, config=CONFIG): except OSError: say('another seed run is running') return 1 - for e in os.listdir(root + '/.new'): # a run that stopped half-way - if e != '.lock': - remove(os.path.join(root + '/.new', e)) - for e in os.listdir(root): # a link made but not yet swapped in - if e.startswith('.') and e.endswith('.link'): - remove(os.path.join(root, e)) try: - host = listing(agent, conf) - items = {n: {**host[n], 'everyHours': s['every_hours'], 'maxAgeHours': s['max_age_hours'], - 'maxMb': min(host[n]['maxMb'], s['max_mb'])} for n, s in own.items() if n in host} - except Torn as e: - items = None - say(f'seeds: {e}: each seed stays until its age drops it') - for name in sorted(n for n in os.listdir(root) if NAME.fullmatch(n)): - version, label = current(root, name) - if label is None: - why = 'its label is missing or cannot be read (fetched again now)' - elif name not in own: - why = 'no longer one of the pool\'s seeds' - else: - age = now() - label['made'] - hours = own[name]['max_age_hours'] - why = (f'older than {hours} h (max_age_hours)' if age > hours * 3600 - else 'made in the future (the clocks disagree)' if age < -SKEW else None) - if why: - drop(root, name, version, now) - say(f'seed {name}: dropped, {why}; jobs run cold until a new one comes') - for name, item in sorted((items or {}).items()): - label = current(root, name)[1] - if 'made' not in item: - say(f'seed {name}: the host offers none ({item.get("why", "no reason")}): ' - f'{"the one here stays" if label else "jobs run cold"}') - continue - if label and (label['bytes'], label['made']) == (item['bytes'], item['made']): - continue - if label and 0 <= now() - label['fetched'] < item['everyHours'] * 3600: - continue - if not -SKEW <= now() - item['made'] <= item['maxAgeHours'] * 3600: - say(f'seed {name}: the host\'s is too old or made in the future (the clocks disagree): not fetched') - continue + clean(root, say) + for e in os.listdir(root): # a link made but not yet swapped in + if e.startswith('.') and e.endswith('.link'): + each(say, f'seed link {e}', remove, os.path.join(root, e)) + for name in sorted(n for n in os.listdir(root) if NAME.fullmatch(n)): + each(say, f'seed {name}', age_out, root, name, own, now, say) + if cannot: + raise agent.Bad(f'{cannot}: no seeds fetched (each seed here stays until its age drops it)') try: - got = fetch(agent, conf, item, root, now) - say(f'seed {name}: {got["bytes"] >> 20} MB, sha256 checked, swapped in') - except (Torn, OSError, ValueError, tarfile.TarError, subprocess.SubprocessError) as e: - for e2 in os.listdir(root + '/.new'): - if e2 != '.lock': - remove(os.path.join(root + '/.new', e2)) - say(f'seed {name}: keeps {"the old one" if label else "none"}: {e.__class__.__name__}: ' - f'{str(e)[:300]}; tried again next hour') - prune(root, now) + host = listing(agent, conf) + items = {n: {**host[n], 'everyHours': s['every_hours'], 'maxAgeHours': s['max_age_hours'], + 'maxMb': min(host[n]['maxMb'], s['max_mb'])} for n, s in own.items() if n in host} + except Torn as e: # the host down, or a host on an older kit (no list, or not one this reads) + items = {} + say(f'seeds: {e}: each seed stays until its age drops it') + for name, item in sorted(items.items()): + if not each(say, f'seed {name}', take, agent, conf, item, root, now, say): + clean(root, say) + finally: + prune(root, now, say) return 0 -def prune(root, now): - """Versions no link names, KEEP_OLD after they were replaced or dropped.""" - linked = {current(root, n)[0] for n in os.listdir(root) if NAME.fullmatch(n)} - for e in os.listdir(root + '/.v'): - path = os.path.join(root, '.v', e) +def each(say, what, step, *args): + """step(*args); any Exception is said, not raised: one seed never stops the others. False when it failed.""" + try: + return step(*args) is not False + except Exception as e: + say(f'{what}: {e.__class__.__name__}: {str(e)[:300]}; tried again next hour') + return False + + +def clean(root, say): + """SEEDS/.new emptied but for the lock: a run (or a fetch) that stopped half-way.""" + for e in os.listdir(root + '/.new'): + if e != '.lock': + each(say, f'seeds: removing .new/{e}', remove, os.path.join(root + '/.new', e)) + + +def age_out(root, name, own, now, say): + version, label = current(root, name) + if label is None: + why = 'its label is missing or cannot be read (fetched again now)' + elif own is not None and name not in own: + why = 'no longer one of the pool\'s seeds' + else: + age = now() - label['made'] + hours = own[name]['max_age_hours'] if own is not None else label['maxAgeHours'] # own config broken + why = (f'older than {hours} h (max_age_hours)' if age > hours * 3600 + else 'made in the future (the clocks disagree)' if age < -SKEW else None) + if why: + drop(root, name, version, now) + say(f'seed {name}: dropped, {why}; jobs run cold until a new one comes') + + +def take(agent, conf, item, root, now, say): + """The listed seed `item`, fetched when it is new and due. False when its fetch failed (the old one stays).""" + name = item['name'] + label = current(root, name)[1] + if 'made' not in item: + say(f'seed {name}: the host offers none ({item.get("why", "no reason")}): ' + f'{"the one here stays" if label else "jobs run cold"}') + return + if label and (label['bytes'], label['made']) == (item['bytes'], item['made']): + return + if label and 0 <= now() - label['fetched'] < item['everyHours'] * 3600: + return + if not -SKEW <= now() - item['made'] <= item['maxAgeHours'] * 3600: + say(f'seed {name}: the host\'s is too old or made in the future (the clocks disagree): not fetched') + return + try: + got = fetch(agent, conf, item, root, now) + except Exception as e: # any: a torn stream, a full disk, a tar this Python cannot unpack + say(f'seed {name}: keeps {"the old one" if label else "none"}: {e.__class__.__name__}: {str(e)[:300]}; ' + f'tried again next hour') + return False + say(f'seed {name}: {got["bytes"] >> 20} MB, sha256 checked, swapped in') + + +def prune(root, now, say=print): + """Versions no link names, KEEP_OLD after they were replaced or dropped; one that cannot go is said.""" + def old(path): if path not in linked and now() - os.lstat(path).st_mtime > KEEP_OLD: remove(path) + try: + linked = {current(root, n)[0] for n in os.listdir(root) if NAME.fullmatch(n)} + names = os.listdir(root + '/.v') + except OSError as e: + say(f'seeds: no prune: {e}') + return + for e in names: + each(say, f'seeds: old version {e}', old, os.path.join(root, '.v', e)) def main(argv=None): diff --git a/.github/scripts/fork-fleet-desk-seed.test.py b/.github/scripts/fork-fleet-desk-seed.test.py index 822f4f33e85b..5aa1464ed2a5 100644 --- a/.github/scripts/fork-fleet-desk-seed.test.py +++ b/.github/scripts/fork-fleet-desk-seed.test.py @@ -2,7 +2,7 @@ """Tests of fork-fleet-desk-seed.py (a desk pool's warm caches) and the `seeds` check of fork-fleet.json: the host's offer in-process, the desktop's run end to end through a fake ssh that runs the host's side, and the forced command's divert in fork-fleet-desk.py. Temp folders only; no root, no ssh, no network. Run: python3 -I -B . Linux only.""" -import fcntl, hashlib, importlib.util, io, json, os, shutil, stat, subprocess, sys, tarfile, tempfile, threading +import contextlib, fcntl, hashlib, importlib.util, io, json, os, shutil, stat, subprocess, sys, tarfile, tempfile, threading import time, unittest from unittest import mock @@ -44,13 +44,19 @@ def load(name, file): if mode == 'down': print('ssh: connect to host h port 22: No route to host', file=sys.stderr) sys.exit(255) +if words[0] == 'seeds' and mode == 'oldhost': # a host on an older kit: a serve session, not a list + print('fork-fleet-desk: serving') + sys.exit(0) +if words[0] == 'seeds' and mode == 'exit1': # a host stopped half-way through its install + print('fork-fleet-desk: no config', file=sys.stderr) + sys.exit(1) kw = dict(config={t!r} + '/fork-fleet.json', files={t!r} + '/files') if words[0] == 'seeds' and mode == 'biglist': os.write(1, b' ' * (2 << 20)) if mode == 'noisy': for _ in range(64): os.write(2, b'x' * (1 << 16) + b'\\n') -if words[0] == 'seed' and mode in ('cut', 'trailer', 'odd'): +if words[0] == 'seed' and mode in ('cut', 'trailer', 'odd', 'made1'): with open({t!r} + '/sent', 'w+b') as out: err = io.StringIO() code = seed.offer({pool!r}, ' '.join(words), {t!r} + '/run', out=out.fileno(), err=err, **kw) @@ -62,6 +68,8 @@ def load(name, file): os.write(1, data) if mode == 'odd': # a MADE that str.isdigit() takes but int() does not print(err.getvalue().rsplit(' ', 1)[0] + ' \u00b2', file=sys.stderr) + elif mode == 'made1': # bytes and sha256 right, MADE not the list's + print(err.getvalue().rsplit(' ', 1)[0] + ' 1', file=sys.stderr) else: print(err.getvalue().rsplit(' ', 2)[0] + ' ' + '0' * 64 + ' 1', file=sys.stderr) # a wrong sha256 sys.exit(code) @@ -72,7 +80,7 @@ def load(name, file): class Base(unittest.TestCase): def setUp(self): self.t = os.path.realpath(tempfile.mkdtemp(prefix='seed-test-')) - self.addCleanup(shutil.rmtree, self.t, True) + self.addCleanup(seed.remove, self.t) # a tree of any depth for d in ('files', f'files/{POOL}', 'run', 'src'): os.mkdir(os.path.join(self.t, d), 0o755) self.config(SEEDS) @@ -422,7 +430,8 @@ def test_fetched_checked_and_swapped_in_then_not_again(self): st = os.lstat(os.path.join(d, n)) if not stat.S_ISLNK(st.st_mode): # a link's own mode is always 0777 and means nothing self.assertFalse(st.st_mode & 0o022, os.path.join(d, n)) - label = json.load(open(os.path.join(seed.current(self.root, 'm2')[0], 'label.json'))) + with open(os.path.join(seed.current(self.root, 'm2')[0], 'label.json')) as f: + label = json.load(f) self.assertEqual(label['bytes'], os.path.getsize(self.tar('m2'))) self.assertEqual(self.run_seed(), 0) # the same seed on the host: nothing fetched self.assertEqual(self.fetches(), ['seed m2']) @@ -514,6 +523,7 @@ def test_stderr_is_never_kept_whole_and_a_huge_list_is_refused(self): def test_a_0600_file_is_readable_by_jobs(self): self.make('m2', {'cache/entry': b'x', 'cache/tool': b'#!/bin/sh\n'}, mode=0o600) self.run_seed() + self.assertEqual(self.files('m2'), {'cache/entry': b'x', 'cache/tool': b'#!/bin/sh\n'}, self.said) top = os.path.realpath(os.path.join(self.root, 'm2')) for d, dirs, names in os.walk(top): for n in dirs + names: @@ -572,6 +582,157 @@ def evil(): for d, dirs, names in os.walk(self.t): self.assertNotIn('escaped', dirs + names, d) + def deep_tar(self, name, depth, count=1, size=0): + """FILES/POOL/NAME.tar of COUNT members d{i}/x/x/.../f, DEPTH folders deep, newer than any made before.""" + tmp = os.path.join(self.t, 'files', POOL, f'.{name}.tmp') + with tarfile.open(tmp, 'w', format=tarfile.GNU_FORMAT) as tar: + for i in range(count): + info = tarfile.TarInfo(f'd{i}/' + 'x/' * depth + 'f') + info.size = size + tar.addfile(info, io.BytesIO(b'z' * size)) + os.utime(tmp, (time.time() + 5,) * 2) + os.chmod(tmp, 0o644) + os.replace(tmp, self.tar(name)) + + def deep_folder(self, top, depth): + """TOP/x/x/.../f, DEPTH folders deep, made folder by folder (a path that long is past PATH_MAX).""" + os.makedirs(top) + fd = os.open(top, os.O_RDONLY | os.O_DIRECTORY) + try: + for _ in range(depth): + os.mkdir('x', dir_fd=fd) + child = os.open('x', os.O_RDONLY | os.O_DIRECTORY, dir_fd=fd) + os.close(fd) + fd = child + os.close(os.open('f', os.O_WRONLY | os.O_CREAT, 0o600, dir_fd=fd)) + finally: + os.close(fd) + + def test_a_deep_path_does_not_stop_the_run(self): + self.make('npm', {'a': b'n'}) + self.deep_tar('m2', 1500) + self.assertEqual(self.run_seed(), 0) + self.assertEqual(self.files('npm'), {'a': b'n'}) + self.assertIsNone(self.files('m2')) + self.assertTrue(any('m2' in s and 'folders deep' in s for s in self.said), self.said) + self.assertEqual(os.listdir(os.path.join(self.root, '.new')), ['.lock']) + + def test_any_failure_of_one_seed_never_stops_the_run_and_old_versions_are_still_pruned(self): + os.makedirs(os.path.join(self.root, '.v', 'm2.1', 'data')) # a version no link names, long replaced + os.utime(os.path.join(self.root, '.v', 'm2.1'), (1, 1)) + self.make('npm', {'a': b'n'}) + self.deep_tar('m2', 1500) + real = seed.fetch + + def broken(agent_, conf, item, root, now): + if item['name'] == 'm2': + raise RuntimeError('anything at all') + return real(agent_, conf, item, root, now) + with mock.patch.object(seed, 'fetch', broken): + self.assertEqual(self.run_seed(), 0) + self.assertEqual(self.files('npm'), {'a': b'n'}) + self.assertTrue(any('m2' in s and 'RuntimeError' in s for s in self.said), self.said) + self.assertFalse(os.path.exists(os.path.join(self.root, '.v', 'm2.1'))) + # a path deeper than Python's recursion allows, past the depth check: said, cleaned up, the run goes on + shutil.rmtree(self.root) + with mock.patch.object(seed, 'DEPTH', 10 ** 6), mock.patch.object(seed, 'NAME_MOST', 10 ** 6), \ + mock.patch.object(seed, 'ENTRY', 0): + self.assertEqual(self.run_seed(), 0) + self.assertTrue(any('m2' in s and 'RecursionError' in s for s in self.said), self.said) + self.assertEqual(self.files('npm'), {'a': b'n'}) + self.assertIsNone(self.files('m2')) + self.assertEqual(os.listdir(os.path.join(self.root, '.new')), ['.lock']) + + def test_a_tree_deeper_than_recursion_left_by_an_older_run_is_removed(self): + self.make('m2', {'a': b'one'}) + self.run_seed() + self.deep_folder(os.path.join(self.root, '.new', 'm2.5'), 1500) # a torn unpack of an older release + self.deep_folder(os.path.join(self.root, '.v', 'm2.6', 'data'), 1500) # a version it left, long replaced + os.utime(os.path.join(self.root, '.v', 'm2.6'), (1, 1)) + self.assertEqual(self.run_seed(), 0) + self.assertEqual(os.listdir(os.path.join(self.root, '.new')), ['.lock']) + self.assertEqual(os.listdir(os.path.join(self.root, '.v')), + [os.path.basename(seed.current(self.root, 'm2')[0])]) + self.assertEqual(self.files('m2'), {'a': b'one'}) + + def test_folders_count_against_max_mb(self): + self.deep_tar('m2', 900, count=150) + self.run_seed() + self.assertLess(sum(len(d) for _, d, _ in os.walk(self.root)), 100000) + self.assertIsNone(self.files('m2')) + cases = [('1220 folders', 'max_mb', lambda: self.deep_tar('m2', 60, count=20)), + ('600 empty files', 'max_mb', lambda: self.make('m2', {f'f{i}': b'' for i in range(600)}, + mtime=time.time() + 5)), + ('a long name', 'over 1024 bytes', lambda: self.make('m2', {'/'.join(['y' * 120] * 10): b'x'}, + mtime=time.time() + 5))] + for case, why, setup in cases: + with self.subTest(case): + shutil.rmtree(self.root, True) + self.torn(setup, why) + + def test_the_trailers_bytes_and_made_must_be_the_lists(self): + self.torn(lambda: (self.make('m2', {'a': b'two'}, mtime=time.time() + 5), self.mode('made1')), + 'does not match the list') + t0 = time.time() + 7 * 3600 + for h in range(5): # each hour tries again; a copy dated 1 is never swapped in, so none piles up in .v + self.run_seed(now=lambda: t0 + h * 3600) + self.assertEqual(self.files('m2'), {'a': b'one'}) + self.assertEqual(len(os.listdir(os.path.join(self.root, '.v'))), 1) + os.unlink(os.path.join(self.t, 'mode')) + shutil.rmtree(self.root) + real = seed.listing + + def lying(*a): + out = real(*a) + out['m2'] = {**out['m2'], 'bytes': out['m2']['bytes'] + 512} + return out + lie = mock.patch.object(seed, 'listing', lying) + self.addCleanup(lie.stop) + self.torn(lambda: (self.make('m2', {'a': b'two'}, mtime=time.time() + 5), lie.start()), + 'does not match the list') + + def test_a_broken_or_pool_less_own_config_still_drops_a_too_old_seed(self): + for why in ('broken', 'pool-less'): + with self.subTest(why): + shutil.rmtree(self.root, True) + self.own(SEEDS) + self.make('m2', {'a': b'one'}) + self.run_seed() + with open(os.path.join(self.t, 'own.json'), 'w') as f: + if why == 'broken': + f.write('{') + else: + cfg = json.loads(json.dumps(CONFIG)) + cfg['pools'][0]['name'] = 'not-this-pool' + json.dump(cfg, f) + fetched = len(self.fetches()) + with self.assertRaisesRegex(agent.Bad, 'no seeds fetched'): + self.run_seed(now=lambda: time.time() + 999 * 3600) + self.assertEqual(self.files('m2'), {'a': b'one'}) # under its label's max_age_hours: kept + with self.assertRaises(agent.Bad): + self.run_seed(now=lambda: time.time() + 1001 * 3600) + self.assertIsNone(self.files('m2')) + self.assertTrue(any('older than 1000 h' in s for s in self.said), self.said) + self.assertEqual(len(self.fetches()), fetched) + + def test_an_older_or_half_installed_host_means_no_seeds(self): + for mode in ('oldhost', 'exit1'): + with self.subTest(mode): + shutil.rmtree(self.root, True) + with contextlib.suppress(FileNotFoundError): + os.unlink(os.path.join(self.t, 'mode')) + self.make('m2', {'a': b'one'}) + self.run_seed() + fetched = len(self.fetches()) + self.mode(mode) + self.make('m2', {'a': b'two'}, mtime=time.time() + 5) + self.assertEqual(self.run_seed(now=lambda: time.time() + 999 * 3600), 0) + self.assertEqual(self.files('m2'), {'a': b'one'}) + self.assertTrue(any('stays until its age drops it' in s for s in self.said), self.said) + self.assertEqual(self.run_seed(now=lambda: time.time() + 1001 * 3600), 0) + self.assertIsNone(self.files('m2')) + self.assertEqual(len(self.fetches()), fetched) + def test_dropped_when_no_longer_listed_or_too_old_or_from_the_future(self): self.make('m2', {'a': b'one'}) self.run_seed() diff --git a/fork-kit/desk/README.md b/fork-kit/desk/README.md index 6e23a0e7d88e..acd39316b788 100644 --- a/fork-kit/desk/README.md +++ b/fork-kit/desk/README.md @@ -51,7 +51,8 @@ the refresh timer; `check` says FAIL for `template` until it is there. `check` only reads: the installed files against their manifest, each unit against the release's (header plus `/opt/fork-fleet/units/UNIT`), a restart a stopped install still owes, systemd drop-ins that change the kit's units, -every wall of this OS (`fork-fleet-walls.py describe`), the agent's settings, the units, the template, the settings +every wall of this OS (`fork-fleet-walls.py describe`), the agent's settings, the units (and the seed service's +last result), the template, the settings files against the machine file, the runner users (and names a user the kit made that a smaller `slots` no longer uses) and IPv4 forwarding. diff --git a/fork-kit/desk/sbin/desk-install b/fork-kit/desk/sbin/desk-install index 1ad128b9a7a1..495feb15643d 100755 --- a/fork-kit/desk/sbin/desk-install +++ b/fork-kit/desk/sbin/desk-install @@ -507,6 +507,9 @@ def check(): on = run('systemctl', 'is-active', '--quiet', unit, check=False).returncode == 0 row(on and run('systemctl', 'is-enabled', '--quiet', unit, check=False).returncode == 0, 'unit', f'{unit} enabled and active') + last = run('systemctl', 'show', '-p', 'Result', '--value', SEED, check=False).stdout.strip() or 'unknown' + row(last == 'success', 'unit', f'{SEED} last run: {last}' + ('' if last == 'success' else + f' (journalctl -u {SEED} says why)')) row(os.path.exists(p(OPT + '/template/runner-template/.fork-fleet-version')), 'template', 'the runner template is there (the refresh timer makes it)') try: diff --git a/fork-kit/desk/tests/install.test.py b/fork-kit/desk/tests/install.test.py index 2fbe79f593d6..5626fd96aa02 100644 --- a/fork-kit/desk/tests/install.test.py +++ b/fork-kit/desk/tests/install.test.py @@ -47,6 +47,8 @@ def put(n, items): open(key + '.pub', 'w').write('ssh-ed25519 AAAAfake desk-x\n') if name == 'systemctl' and args[0] in ('is-active', 'is-enabled'): sys.exit(1 if args[-1] in lines('down') else 0) +if name == 'systemctl' and args[:3] == ['show', '-p', 'Result']: # a unit's last result: the file `result` + print((lines('result') or ['success'])[0]) if name == 'systemctl' and args[0] == 'list-units': # runner units: the file `units`; `stuck` ones never stop print(''.join(f'{u} loaded active running runner\n' for u in lines('units')), end='') if name == 'systemctl' and args[0] == 'stop': @@ -515,6 +517,7 @@ def test_check_names_each_failure(self): self.write(os.path.join(self.fakes, 'wall-fails'), '') self.write(os.path.join(self.fakes, 'agent-bad'), '') self.write(os.path.join(self.fakes, 'down'), 'fork-fleet-desk-agent.service\n') + self.write(os.path.join(self.fakes, 'result'), 'exit-code\n') # the seed service failed its last run self.write(self.path('/opt/fork-fleet/hook.sh'), '#!/bin/bash\nevil\n', 0o755) self.write(self.path('/proc/sys/net/ipv4/ip_forward'), '0\n') os.unlink(os.path.join(self.fakes, 'users')) @@ -523,10 +526,10 @@ def test_check_names_each_failure(self): for line in ('FAIL kit /opt/fork-fleet/hook.sh: differs', 'FAIL vsock closed: open', 'FAIL agent-settings bad settings', 'FAIL unit fork-fleet-desk-agent.service', 'PASS unit t3code-fleet-refresh.timer', 'PASS unit t3code-fleet-seed.timer', - 'FAIL template', 'FAIL runner-user fork-runner-1', - 'FAIL ip-forward'): + 'FAIL unit t3code-fleet-seed.service last run: exit-code', 'FAIL template', + 'FAIL runner-user fork-runner-1', 'FAIL ip-forward'): self.assertIn(line, out) - self.assertTrue(out.endswith('8 FAIL\n'), out) + self.assertTrue(out.endswith('9 FAIL\n'), out) def test_check_fails_on_a_drop_in_and_on_settings_the_machine_file_no_longer_gives(self): self.assertEqual(self.install()[0], 0) From 4812b17d85334efa4c974beee5e29fbc2e3d3b04 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 01:18:53 +0000 Subject: [PATCH 5/6] fleet seeds: prune first, bound unpack memory, back off torn seeds - Prune old versions right after the start-of-run clean, before any fetch, so a run killed in every fetch (MemoryMax, TimeoutStartSec) cannot let .v fill up. - Unpack member by member and let each go; extended (pax, GNU long name) headers at most 1 MiB per member and globally; sparse members refused; folder-name bytes capped; a hard link must name a regular file already unpacked. - A deeply nested list is no list (RecursionError); a folder that cannot be listed tears the seed. - One run fetches for at most RUN_WAIT (2 h), each seed at most the time left, well inside the unit's 3 h TimeoutStartSec. - .torn.json: a torn seed (or a fetch the run was killed in) is tried again after 1, 2, 4, 8, 16, then 24 h; torn on every try for over a day makes the run exit 1, so desk-install check shows FAIL. - Docs: memory limits, prune order, run deadline, backoff, disk budget per seed. Co-Authored-By: Claude Opus 5.5 --- .github/fork-fleet.md | 21 +- .github/scripts/fork-fleet-desk-seed.py | 190 ++++++++++++++++-- .github/scripts/fork-fleet-desk-seed.test.py | 200 ++++++++++++++++++- fork-kit/desk/README.md | 3 + 4 files changed, 386 insertions(+), 28 deletions(-) diff --git a/.github/fork-fleet.md b/.github/fork-fleet.md index 2a7f3780f90c..05f6a1f36398 100644 --- a/.github/fork-fleet.md +++ b/.github/fork-fleet.md @@ -737,12 +737,23 @@ host's `max_mb`: a host cannot make a desktop take more. Each new seed is unpacked with Python's tar data filter and a limit on what it would make on disk (each member's size plus 4 KiB for it and for each folder it makes, all within `max_mb`; at most 200,000 members and folders; no member more than 64 -folders deep or with a name over 1 KiB), checked against the trailer (its size +folders deep or with a name over 1 KiB) and on memory (members are let go as +they are unpacked; the extended pax or long-name headers before one member, and +the global ones of the whole tar, at most 1 MiB; sparse members are refused, so +make a seed without `--sparse`; a hard link must name a file unpacked before +it), checked against the trailer (its size and date must be the list's too), checked again for links that lead out of it, made readable by every job (a cache's 0600 files too), and swapped in by a rename; ssh's stderr and the list are read with a limit. A seed that fails in any way is named in the log and the run goes on with the next; old copies -are pruned in every run. A host that gives no list (down, an older kit, a +are pruned at the start and the end of every run (so a run killed by its unit's +memory or time limit cannot pile them up). A run fetches for at most two hours +in all (the unit allows three), each seed at most 30 minutes. A seed torn again +and again is tried again after 1, 2, 4, 8, 16 and then every 24 hours (a link +too slow for it does not spend every hour on it; a fetch that kills the run +counts as torn), and one torn on every try for more than a day makes the run +exit 1, so `desk-install check` shows FAIL until it is fetched +(`/var/lib/fork-fleet/seeds/.torn.json` names it and why). A host that gives no list (down, an older kit, a half-done install) means no new seeds: each one here stays until its age drops it. A desktop whose own fleet config is broken or lacks its pool fetches nothing and exits 2 (`desk-install check` shows the seed service's last @@ -753,7 +764,11 @@ the unpacking and old copies beside it, `.new` and `.v`, are visible too and hold the same data), copy what they need from it when it is there, and run cold when it is not. A replaced copy is kept 24 hours for a job still reading it; there is no free-space check before unpacking (a full disk ends that seed's -fetch and keeps the old one). The state table (a torn download, the host down, +fetch and keeps the old one). **Disk budget per seed:** up to `max_mb` x +(24 / `every_hours` + 2): the copy in use, the copies replaced in the last 24 +hours, and one being unpacked (`every_hours` 1 and `max_mb` 4096: up to about +104 GiB; `every_hours` 6: about 24 GiB). Size `max_mb` and `every_hours` to the +desktop's free disk. The state table (a torn download, the host down, clocks that disagree, a run that stopped half-way) is in the script's header. ## Owner steps on the repository diff --git a/.github/scripts/fork-fleet-desk-seed.py b/.github/scripts/fork-fleet-desk-seed.py index 563594ba3d3c..432426950943 100644 --- a/.github/scripts/fork-fleet-desk-seed.py +++ b/.github/scripts/fork-fleet-desk-seed.py @@ -19,8 +19,9 @@ every_hours and max_age_hours and the smaller of its own and the host's max_mb. Each new seed is unpacked with tarfile's data filter into a folder of its own (what it would make on disk held to max_mb, ENTRY bytes counted for each member and folder; no member over DEPTH folders deep or - a name over NAME_MOST bytes), checked against the trailer (bytes and MADE the list's too), walked - (a link out of it is refused; all made readable), flushed, and swapped in by rename: SEEDS/NAME is a link to + a name over NAME_MOST bytes; memory held too: see Info and unpack), checked against the + trailer (bytes and MADE the list's too), walked (a link out of it is refused; all made + readable), flushed, and swapped in by rename: SEEDS/NAME is a link to .v/NAME.STAMP/data, root's (folders 0755, files no one else may write), unpacked in .new (one folder: a rename never crosses a mount of the unit's sandbox). Jobs see SEEDS read-only (the launcher's ALLOW_PATHS): a job uses /var/lib/fork-fleet/seeds/NAME when it is there and runs @@ -31,8 +32,12 @@ a new one under every_hours after the last nothing yet a torn copy (the stream cut or past FETCH_WAIT, a trailer that does not match it or the list, more than max_mb, a tar member that would leave its folder, is a device, too deep or too long, or any other error) - the copy is removed, the old seed stays, the next hour tries again; - the other seeds go on, old versions are still pruned + the copy is removed, the old seed stays, the other seeds go on, old + versions are still pruned; tried again after 1, 2, 4, 8, 16, then 24 + hours (TornLog); torn on every try for over TORN_DAY: the run exits 1 + a fetch that kills the run (MemoryMax, TimeoutStartSec) + counted as torn before it began; the next run prunes first + the run's RUN_WAIT used the seeds left are tried next hour no seed (never fetched, the host down) jobs run cold no list (the host down, an older kit that answers with a session or not JSON, exit 1 of a half-done install) nothing fetched; each seed stays until its age drops it @@ -74,11 +79,14 @@ def module(name, file): NAME = re.compile(r'[a-z0-9][a-z0-9_-]{0,39}') LINK = re.compile(r'\.v/((?P[a-z0-9][a-z0-9_-]{0,39})\.[0-9]{1,20})/data') LIST_WAIT, FETCH_WAIT = 30, 30 * 60 # seconds for the list, and for one seed's whole stream +RUN_WAIT = 2 * 3600 # seconds a run may spend fetching, all seeds together (the unit's TimeoutStartSec is 3 h) +TORN_DAY = 24 * 3600 # a seed torn on every try for this long makes the run exit 1 (the unit's result: `check` FAIL) LIST_MOST, ERR_KEEP, MEMBERS = 1 << 20, 4096, 200_000 # the list's bytes, ssh's stderr kept, a seed's tar members KEEP_OLD, SKEW = 24 * 3600, 3600 # SKEW: how far the host's clock may be ahead of the desktop's PIECE, WRITE, STALL, AHEAD = 1 << 20, 64 << 10, 120, 300 # AHEAD: seconds a file's date may be ahead of now GRACE, MIN_RATE = 600, 256 << 10 # a stream's deadline: GRACE seconds plus one second per MIN_RATE bytes DEPTH, NAME_MOST, ENTRY = 64, 1024, 4096 # a member's most folders deep and name bytes; the disk an entry costs +HEADER_MOST, FOLDER_NAMES = 1 << 20, 16 << 20 # extended header bytes (see Info); folder name bytes kept ITEM = {'everyHours': 'every_hours', 'maxAgeHours': 'max_age_hours', 'maxMb': 'max_mb'} @@ -309,7 +317,7 @@ def listing(agent, conf): and whole(s['made'])) out[s['name']] = s return out - except (ValueError, AssertionError): + except (ValueError, AssertionError, RecursionError): # RecursionError: a list nested deeper than json can go raise Torn('the host\'s list is not well formed') from None @@ -321,13 +329,65 @@ def whole(x): return isinstance(x, int) and not isinstance(x, bool) and x >= 0 +EXTENDED = (tarfile.XHDTYPE, tarfile.XGLTYPE, tarfile.SOLARIS_XHDTYPE, tarfile.GNUTYPE_LONGNAME, + tarfile.GNUTYPE_LONGLINK) + + +class Info(tarfile.TarInfo): + """A tar header read with limits tarfile has not: it reads an extended header (pax, or a GNU long name) into + memory whole and its parsed form takes many times its size, and a sparse member's map can grow without end. So the + extended headers before one member may hold at most HEADER_MOST bytes together, the global pax headers of the + whole tar too, and a sparse member is refused (make a seed without --sparse). Torn before tarfile reads one.""" + + def _proc_member(self, tarfile_): + used = tarfile_.__dict__.setdefault('fork_headers', [0, 0]) # [before this member, global in all] + if self.type in EXTENDED: + used[0] += self.size + used[1] += self.size if self.type == tarfile.XGLTYPE else 0 + if max(used) > HEADER_MOST: + raise Torn(f'the tar\'s extended (pax or long-name) headers hold more than {HEADER_MOST >> 20} MB') + else: + used[0] = 0 + if self.type == tarfile.GNUTYPE_SPARSE: + self.sparse_refused() + return super()._proc_member(tarfile_) + + def sparse_refused(self, member=None, *_): + raise Torn(f'{(member or self).name[:80]!r} is a sparse member (make the seed without --sparse)') + + _proc_gnusparse_00 = _proc_gnusparse_01 = _proc_gnusparse_10 = sparse_refused # pax sparse forms + + +def unpack(t, data, most): + """The members of the open stream T into DATA, one at a time: tarfile keeps every member it has read in + `members` (and extractall each folder besides), so they are let go as they are made; only `guarded`'s count of + folders stays. A hard link must name a file already unpacked (otherwise tarfile would read the rest of the + stream into memory to find it).""" + check = guarded(most) + + def linked(member, path): + got = check(member, path) + if got is not None and got.islnk(): + try: + plain = stat.S_ISREG(os.lstat(os.path.join(path, got.linkname)).st_mode) + except OSError: + plain = False + if not plain: + raise Torn(f'{got.name[:80]!r} is a hard link to {got.linkname[:80]!r}, not a file unpacked before it') + return got + while (member := t.next()) is not None: + t.members.clear() + t.extract(member, data, set_attrs=not member.isdir(), filter=linked) # data_filter keeps no folder mode + + def guarded(most): """tarfile's data filter, and a limit on what the tar would make on disk: each member's size (a sparse member is small in the stream, not on disk) plus ENTRY bytes for it and for each folder it makes, together at most MOST bytes (a stream of empty files or deep paths is small but would fill the disk with entries); at most MEMBERS members and MEMBERS folders (each is kept in memory); no member more than DEPTH folders deep (unpacking and - removing a tree goes folder by folder) or with a name or link target over NAME_MOST bytes.""" - seen, folders = [0, 0], {} # folders: {(its parent's number, its name): its number}, a name kept once + removing a tree goes folder by folder) or with a name or link target over NAME_MOST bytes; the folder names + kept to count them at most FOLDER_NAMES bytes.""" + seen, folders = [0, 0, 0], {} # folders: {(its parent's number, its name): its number}, a name kept once def check(member, path): parts = [x for x in member.name.split('/') if x not in ('', '.')] @@ -340,13 +400,15 @@ def check(member, path): for i, part in enumerate(parts if member.isdir() else parts[:-1], 1): # the folders it makes if (up, part) not in folders: folders[up, part] = len(folders) + 1 + seen[2] += len(part) new += i < len(parts) # a folder member pays ENTRY once, as a member up = folders[up, part] seen[0] += 1 seen[1] += member.size + ENTRY * (1 + new) - if seen[0] > MEMBERS or len(folders) > MEMBERS or seen[1] > most: - raise Torn(f'the tar holds more than {MEMBERS} members or folders, or more than {most >> 20} MB ' - f'(max_mb; {ENTRY} bytes counted for each member and folder)') + if seen[0] > MEMBERS or len(folders) > MEMBERS or seen[1] > most or seen[2] > FOLDER_NAMES: + raise Torn(f'the tar holds more than {MEMBERS} members or folders, folder names over ' + f'{FOLDER_NAMES >> 20} MB, or more than {most >> 20} MB (max_mb; {ENTRY} bytes counted for ' + f'each member and folder)') return tarfile.data_filter(member, path) return check @@ -356,7 +418,10 @@ def settle(data): besides the data filter, which some Python releases let links slip past); folders and files readable by all (jobs run as other users; a cache may hold 0600 files), writable by root only. Torn otherwise.""" top = os.path.realpath(data) - for d, dirs, names in os.walk(data): + + def unlisted(e): # a folder os.walk cannot list (a path past PATH_MAX, say) would go unchecked + raise Torn(f'{os.path.relpath(e.filename or data, data)[:80]} cannot be listed: {e.strerror}') + for d, dirs, names in os.walk(data, onerror=unlisted): for n in dirs + names: path = os.path.join(d, n) st = os.lstat(path) @@ -371,22 +436,22 @@ def settle(data): raise Torn(f'{os.path.relpath(path, data)} is not a folder, a file or a link') -def fetch(agent, conf, item, root, now): +def fetch(agent, conf, item, root, now, wait=FETCH_WAIT): """The seed `item` (its maxMb: the smaller of the host's and the desktop's own) into a new version, swapped in; - its label. Torn (or OSError, TarError) leaves the old one.""" + its label. The stream is cut after WAIT seconds. Torn (or OSError, TarError) leaves the old one.""" name, stamp, most = item['name'], time.time_ns(), item['maxMb'] << 20 new = os.path.join(root + '/.new', f'{name}.{stamp}') os.mkdir(new, 0o755) os.mkdir(os.path.join(new, 'data'), 0o755) p = subprocess.Popen(ssh(agent, conf, 'seed', name), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=agent.ENV, start_new_session=True) - timer = threading.Timer(FETCH_WAIT, p.kill) + timer = threading.Timer(wait, p.kill) timer.start() err = Tail(p.stderr) try: got = Summed(p.stdout, most) - with tarfile.open(fileobj=got, mode='r|') as t: - t.extractall(os.path.join(new, 'data'), filter=guarded(most)) + with tarfile.open(fileobj=got, mode='r|', tarinfo=Info) as t: + unpack(t, os.path.join(new, 'data'), most) while got.read(PIECE): # the tar's end blocks and padding count in the trailer's size and sha256 pass code = p.wait(timeout=30) @@ -521,6 +586,7 @@ def run(agent, conf, root=SEEDS, now=time.time, say=print, config=CONFIG): for e in os.listdir(root): # a link made but not yet swapped in if e.startswith('.') and e.endswith('.link'): each(say, f'seed link {e}', remove, os.path.join(root, e)) + prune(root, now, say) # first too: a run killed later (its memory or time limit) skips the finally for name in sorted(n for n in os.listdir(root) if NAME.fullmatch(n)): each(say, f'seed {name}', age_out, root, name, own, now, say) if cannot: @@ -532,14 +598,78 @@ def run(agent, conf, root=SEEDS, now=time.time, say=print, config=CONFIG): except Torn as e: # the host down, or a host on an older kit (no list, or not one this reads) items = {} say(f'seeds: {e}: each seed stays until its age drops it') + torn, end = TornLog(root, now), time.monotonic() + RUN_WAIT + torn.keep(items) for name, item in sorted(items.items()): - if not each(say, f'seed {name}', take, agent, conf, item, root, now, say): + if not each(say, f'seed {name}', take, agent, conf, item, root, now, say, torn, end): clean(root, say) + each(say, 'seeds: .torn.json', torn.save) + if late := torn.late(): + say(f'seeds: {", ".join(late)} torn on every try for more than {TORN_DAY // 3600} h: exit 1 ' + f'(desk-install check shows it)') + return 1 finally: prune(root, now, say) return 0 +class TornLog: + """SEEDS/.torn.json, {name: {"first", "last", "tries", "why"}}: the seeds whose last fetch was torn, written + before each fetch (so a fetch that kills the run, by its memory or time limit, counts as torn too) and cleared + when the seed is fetched or the same as the host's. A seed torn TRIES times in a row is tried again only + min(2 ** (TRIES - 1), 24) hours after the last try (a link too slow for it, a host that cuts it: not every + hour); one torn on every try since more than TORN_DAY is `late`. Only seeds in the host's list are kept. A + missing or unreadable file is an empty record.""" + + def __init__(self, root, now): + self.path, self.now, self.seeds = root and os.path.join(root, '.torn.json'), now, {} + try: + with open(self.path, encoding='utf-8') as f: + got = json.load(f) + self.seeds = {n: r for n, r in got.items() if NAME.fullmatch(n) and isinstance(r, dict) and all( + whole(r.get(k)) for k in ('first', 'last', 'tries'))} + except (OSError, TypeError, ValueError, AttributeError, RecursionError): # TypeError: no path + pass + + def keep(self, items): + self.seeds = {n: r for n, r in self.seeds.items() if n in items} + + def due(self, name): + """None when the seed may be tried now, else the hours until it may.""" + r = self.seeds.get(name) + if not r: + return None + wait = min(2 ** (min(r['tries'], 6) - 1), 24) * 3600 - 300 # 5 min early: the hourly timer drifts + left = r['last'] + wait - self.now() + return math.ceil(left / 3600) if 0 < left <= wait else None # a date ahead of the clock: try now + + def trying(self, name): + """Counted as torn before the fetch, saved: `why` says so until the fetch ends.""" + r = self.seeds.get(name) or {'first': int(self.now()), 'tries': 0} + self.seeds[name] = {**r, 'last': int(self.now()), 'tries': r['tries'] + 1, + 'why': 'the run stopped during the fetch (its memory or time limit?)'} + self.save() + + def torn(self, name, why): + self.seeds[name]['why'] = why[:300] + self.save() + + def fine(self, name): + if self.seeds.pop(name, None): + self.save() + + def late(self): + return sorted(n for n, r in self.seeds.items() if self.now() - r['first'] > TORN_DAY) + + def save(self): + if self.path is None: + return + tmp = self.path + '.tmp' + with open(tmp, 'w', encoding='utf-8') as f: + json.dump(self.seeds, f) + os.replace(tmp, self.path) + + def each(say, what, step, *args): """step(*args); any Exception is said, not raised: one seed never stops the others. False when it failed.""" try: @@ -572,27 +702,43 @@ def age_out(root, name, own, now, say): say(f'seed {name}: dropped, {why}; jobs run cold until a new one comes') -def take(agent, conf, item, root, now, say): - """The listed seed `item`, fetched when it is new and due. False when its fetch failed (the old one stays).""" +def take(agent, conf, item, root, now, say, torn=None, end=None): + """The listed seed `item`, fetched when it is new and due (and not held back after torn tries, and while the + run has time left: END, monotonic). False when its fetch failed (the old one stays).""" name = item['name'] + torn = torn or TornLog(None, now) # one take() alone: nothing saved + end = time.monotonic() + RUN_WAIT if end is None else end label = current(root, name)[1] if 'made' not in item: say(f'seed {name}: the host offers none ({item.get("why", "no reason")}): ' f'{"the one here stays" if label else "jobs run cold"}') + torn.fine(name) # nothing to fetch is not a torn fetch return if label and (label['bytes'], label['made']) == (item['bytes'], item['made']): + torn.fine(name) return if label and 0 <= now() - label['fetched'] < item['everyHours'] * 3600: return if not -SKEW <= now() - item['made'] <= item['maxAgeHours'] * 3600: say(f'seed {name}: the host\'s is too old or made in the future (the clocks disagree): not fetched') return + if (hours := torn.due(name)) is not None: + say(f'seed {name}: torn {torn.seeds[name]["tries"]} times in a row; tried again in about {hours} h') + return + left = end - time.monotonic() + if left < 60: + say(f'seed {name}: this run has used its {RUN_WAIT // 60} minutes; tried again next hour') + return + torn.trying(name) try: - got = fetch(agent, conf, item, root, now) + got = fetch(agent, conf, item, root, now, wait=min(FETCH_WAIT, left)) except Exception as e: # any: a torn stream, a full disk, a tar this Python cannot unpack - say(f'seed {name}: keeps {"the old one" if label else "none"}: {e.__class__.__name__}: {str(e)[:300]}; ' - f'tried again next hour') + why = f'{e.__class__.__name__}: {str(e)[:300]}' + torn.torn(name, why) + hours = min(2 ** (min(torn.seeds[name]['tries'], 6) - 1), 24) + say(f'seed {name}: keeps {"the old one" if label else "none"}: {why}; tried again in {hours} h') return False + torn.fine(name) say(f'seed {name}: {got["bytes"] >> 20} MB, sha256 checked, swapped in') diff --git a/.github/scripts/fork-fleet-desk-seed.test.py b/.github/scripts/fork-fleet-desk-seed.test.py index 6ac1e5e919e4..60c585a60b64 100644 --- a/.github/scripts/fork-fleet-desk-seed.test.py +++ b/.github/scripts/fork-fleet-desk-seed.test.py @@ -3,7 +3,7 @@ offer in-process, the desktop's run end to end through a fake ssh that runs the host's side, and the forced command's divert in fork-fleet-desk.py. Temp folders only; no root, no ssh, no network. Run: python3 -I -B . Linux only.""" import contextlib, fcntl, hashlib, importlib.util, io, json, os, shutil, stat, subprocess, sys, tarfile, tempfile, threading -import time, unittest +import signal, time, unittest from unittest import mock sys.dont_write_bytecode = True @@ -47,6 +47,9 @@ def load(name, file): if words[0] == 'seeds' and mode == 'oldhost': # a host on an older kit: a serve session, not a list print('fork-fleet-desk: serving') sys.exit(0) +if words[0] == 'seeds' and mode == 'deep': # a list nested deeper than json can go + print('[' * 200000) + sys.exit(0) if words[0] == 'seeds' and mode == 'exit1': # a host stopped half-way through its install print('fork-fleet-desk: no config', file=sys.stderr) sys.exit(1) @@ -651,10 +654,10 @@ def test_any_failure_of_one_seed_never_stops_the_run_and_old_versions_are_still_ self.deep_tar('m2', 1500) real = seed.fetch - def broken(agent_, conf, item, root, now): + def broken(agent_, conf, item, root, now, **kw): if item['name'] == 'm2': raise RuntimeError('anything at all') - return real(agent_, conf, item, root, now) + return real(agent_, conf, item, root, now, **kw) with mock.patch.object(seed, 'fetch', broken): self.assertEqual(self.run_seed(), 0) self.assertEqual(self.files('npm'), {'a': b'n'}) @@ -670,6 +673,197 @@ def broken(agent_, conf, item, root, now): self.assertIsNone(self.files('m2')) self.assertEqual(os.listdir(os.path.join(self.root, '.new')), ['.lock']) + def raw(self, name, blob): + """FILES/POOL/NAME.tar holding BLOB (tar blocks made by hand), newer than any made before.""" + tmp = os.path.join(self.t, 'files', POOL, f'.{name}.tmp') + with open(tmp, 'wb') as f: + f.write(blob + b'\0' * 1024) + os.chmod(tmp, 0o644) + os.utime(tmp, (time.time() + 5,) * 2) + os.replace(tmp, self.tar(name)) + + @staticmethod + def block(info, data=b'', fmt=tarfile.GNU_FORMAT): + """One member: INFO's header (its size is DATA's) and DATA padded to whole blocks.""" + info.size = len(data) + return info.tobuf(format=fmt) + data + b'\0' * (-len(data) % 512) + + @staticmethod + def pax(**records): + out = b'' + for k, v in records.items(): + line = f' {k}={v}\n'.encode() + n = len(line) + 1 + while n != len(line) + len(str(n)): # the length counts its own digits + n = len(line) + len(str(n)) + out += str(n).encode() + line + return out + + def test_a_run_killed_in_every_fetch_still_prunes(self): # lens237sr2: the MUST + os.makedirs(os.path.join(self.root, '.v', 'm2.1', 'data')) # long replaced, no link names it + os.utime(os.path.join(self.root, '.v', 'm2.1'), (1, 1)) + self.make('m2', {'a': b'm'}) + self.make('npm', {'a': b'n'}) + real = seed.fetch + + def oom(agent_, conf, item, root, now, **kw): # MemoryMax or TimeoutStartSec: SIGKILL, no finally runs + if item['name'] == 'npm': + os.kill(os.getpid(), signal.SIGKILL) + return real(agent_, conf, item, root, now, **kw) + pid = os.fork() + if pid == 0: + try: + with mock.patch.object(seed, 'fetch', oom): + self.run_seed() + finally: + os._exit(0) + self.assertEqual(os.waitpid(pid, 0)[1] & 0x7f, signal.SIGKILL) + self.assertEqual(self.files('m2'), {'a': b'm'}) + self.assertFalse(os.path.exists(os.path.join(self.root, '.v', 'm2.1'))) + with open(os.path.join(self.root, '.torn.json')) as f: # the killed fetch counts as torn + self.assertEqual(json.load(f)['npm']['tries'], 1) + + def test_a_pax_header_seed_stays_under_the_units_memory(self): # lens237sr2: the MUST + big = [{**SEEDS[0], 'max_mb': 64}, SEEDS[1]] # a realistic max_mb (fork-fleet.md shows 4096) + self.config(big) + self.own(big) + body = b''.join(b'%d a.%07d=x\n' % (len(b' a.0000000=x\n') + 2, i) for i in range(2_500_000)) + info = tarfile.TarInfo('pax') + info.type = tarfile.XGLTYPE + self.raw('m2', self.block(info, body, tarfile.USTAR_FORMAT)) + del body + r, w = os.pipe() + pid = os.fork() + if pid == 0: + import resource + code = 2 + try: + os.close(r) + self.run_seed() + os.write(w, json.dumps(self.said).encode()) + code = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss >= 512 << 10 # MemoryMax=512M, in KB + finally: + os._exit(code) + os.close(w) + with os.fdopen(r) as f: + said = f.read() + self.assertEqual(os.waitpid(pid, 0)[1] >> 8, 0, said) # a ~35 MB stream (under max_mb) within 512 MB + self.assertIn('extended (pax or long-name) headers hold more than 1 MB', said) + + def test_extended_headers_and_sparse_members_are_held_to_limits(self): + long_name = tarfile.TarInfo('././@LongLink') + long_name.type = tarfile.GNUTYPE_LONGNAME + chain = tarfile.TarInfo('pax') + chain.type = tarfile.XHDTYPE + old_sparse = tarfile.TarInfo('s') + old_sparse.type = tarfile.GNUTYPE_SPARSE + pax_sparse = tarfile.TarInfo('pax') + pax_sparse.type = tarfile.XHDTYPE + cases = { + 'a GNU long name of 1.5 MB': (self.block(long_name, b'n' * (3 << 19) + b'\0') + + self.block(tarfile.TarInfo('f'), b'x'), 'extended'), + 'pax headers of 400 KB each, three before one member': ( + self.block(chain, self.pax(comment='c' * (400 << 10))) * 3 + self.block(tarfile.TarInfo('f'), b'x'), + 'extended'), + 'an old GNU sparse member': (self.block(old_sparse), 'sparse member'), + 'a pax (1.0) sparse member': ( + self.block(pax_sparse, self.pax(**{'GNU.sparse.major': 1, 'GNU.sparse.minor': 0, + 'GNU.sparse.name': 's', 'GNU.sparse.realsize': 0})) + + self.block(tarfile.TarInfo('s'), b'0\n'.ljust(512, b'\0')), 'sparse member'), + } + for what, (blob, why) in cases.items(): + with self.subTest(what): + self.raw('m2', blob) + self.run_seed() + self.assertIsNone(self.files('m2')) + self.assertTrue(any('m2' in x and why in x for x in self.said), self.said) + with contextlib.suppress(FileNotFoundError): # each case tried at once, not after a wait + os.unlink(os.path.join(self.root, '.torn.json')) + for k in ('_proc_member', '_proc_sparse', '_proc_gnusparse_00', '_proc_gnusparse_01', '_proc_gnusparse_10'): + self.assertTrue(callable(getattr(tarfile.TarInfo, k, None)), f'tarfile.TarInfo.{k} is gone: see Info') + + def test_members_are_let_go_as_they_are_unpacked(self): + self.make('m2', {f'd/{i}': b'x' for i in range(300)}) + most, real = [0], tarfile.TarFile.next + + def counted(t): + got = real(t) + most[0] = max(most[0], len(t.members)) + return got + with mock.patch.object(tarfile.TarFile, 'next', counted): + self.run_seed() + self.assertEqual(len(self.files('m2')), 300) + self.assertLessEqual(most[0], 2) + + def test_a_hard_link_must_name_a_file_unpacked_before_it(self): + def hard(name, target): + info = tarfile.TarInfo(name) + info.type, info.linkname = tarfile.LNKTYPE, target + return self.block(info) + self.raw('m2', self.block(tarfile.TarInfo('a'), b'one') + hard('b', 'a')) + self.run_seed() + self.assertEqual(self.files('m2'), {'a': b'one', 'b': b'one'}) + self.raw('m2', self.block(tarfile.TarInfo('a'), b'two') + hard('b', 'later') + + self.block(tarfile.TarInfo('later'), b'x')) + self.run_seed(now=lambda: time.time() + 7 * 3600) + self.assertEqual(self.files('m2'), {'a': b'one', 'b': b'one'}) # the old one stays + self.assertTrue(any('hard link' in x for x in self.said), self.said) + + def test_a_deeply_nested_list_means_no_list(self): # lens237cr2 + self.make('m2', {'a': b'one'}) + self.run_seed() + self.mode('deep') + self.assertEqual(self.run_seed(now=lambda: time.time() + 7 * 3600), 0) + self.assertEqual(self.files('m2'), {'a': b'one'}) + self.assertTrue(any('not well formed' in x for x in self.said), self.said) + + def test_a_folder_that_cannot_be_listed_tears_the_seed(self): + self.make('m2', {'a/b/c': b'one'}) + real = os.scandir + + def scandir(path='.'): + if isinstance(path, str) and path.endswith(os.sep + 'b'): + raise OSError(36, 'File name too long', path) + return real(path) + with mock.patch.object(os, 'scandir', scandir): + self.run_seed() + self.assertIsNone(self.files('m2')) + self.assertTrue(any('cannot be listed' in x for x in self.said), self.said) + + def test_a_run_stops_fetching_when_its_time_is_used(self): + self.make('m2', {'a': b'one'}) + with mock.patch.object(seed, 'RUN_WAIT', 30): + self.assertEqual(self.run_seed(), 0) + self.assertIsNone(self.files('m2')) + self.assertEqual(self.fetches(), []) + self.assertTrue(any('this run has used its' in x for x in self.said), self.said) + waits = [] + real = seed.fetch + + def timed(agent_, conf, item, root, now, wait=None): + waits.append(wait) + return real(agent_, conf, item, root, now, wait=wait) + with mock.patch.object(seed, 'RUN_WAIT', 600), mock.patch.object(seed, 'fetch', timed): + self.run_seed() + self.assertEqual(self.files('m2'), {'a': b'one'}) + self.assertTrue(waits and all(500 < w <= 600 for w in waits), waits) # each fetch within what is left + + def test_a_seed_torn_every_time_backs_off_then_fails_the_run_after_a_day(self): + self.make('m2', {'a': b'one'}) + start, tried = int(time.time()), [] + + def torn(agent_, conf, item, root, now, **kw): + tried.append(now() - start) + raise seed.Torn('cut off') + with mock.patch.object(seed, 'fetch', torn): + codes = {h: self.run_seed(now=lambda h=h: start + h * 3600) for h in range(0, 33)} + self.assertEqual([round(x / 3600) for x in tried], [0, 1, 3, 7, 15, 31]) # 1, 2, 4, 8, 16 h apart + self.assertEqual({h for h, c in codes.items() if c}, set(range(25, 33))) # torn for more than 24 h: exit 1 + self.assertTrue(any('torn on every try' in x for x in self.said), self.said) + self.assertEqual(self.run_seed(now=lambda: start + 63 * 3600), 0) # fetched at last: the record goes + self.assertEqual(self.files('m2'), {'a': b'one'}) + self.assertEqual(self.run_seed(now=lambda: start + 64 * 3600), 0) + def test_a_tree_deeper_than_recursion_left_by_an_older_run_is_removed(self): self.make('m2', {'a': b'one'}) self.run_seed() diff --git a/fork-kit/desk/README.md b/fork-kit/desk/README.md index b5c4f9c476ba..17e58d046b3a 100644 --- a/fork-kit/desk/README.md +++ b/fork-kit/desk/README.md @@ -65,6 +65,9 @@ names the fix (`usermod -s /usr/sbin/nologin`, `passwd -l`). One install or unin The seed timer (`t3code-fleet-seed.timer`) fetches the pool's warm caches, if its fleet config lists any, into `/var/lib/fork-fleet/seeds`, which jobs see read-only ([fork-fleet.md](../../.github/fork-fleet.md), "Seeds"). +A torn seed is tried again after 1, 2, 4, 8, 16, then 24 hours; one torn on every try for more than a day makes each +run exit 1, so `check` shows the seed service's last run as failed. `/var/lib/fork-fleet/seeds/.torn.json` names it +and the host's reason (jobs can read that file too). ## Windows (WSL2) From c68366bb302ea096e3338c5d34ba82789e434dc8 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 02:09:24 +0000 Subject: [PATCH 6/6] fleet seeds: links never load the stream, sizes never lower the count, torn record kept - A link member whose path is already there is refused, and a Tar subclass turns tarfile's fallback (_find_link_target, which reads the rest of the stream into memory) into Torn: a link that cannot be made never copies its target. - A member with a size below 0, or a non-file with a size, is refused: the disk count can no longer be lowered. - A run without the host's list leaves .torn.json as it is (exit 0). - A torn record that cannot be written (full disk) never stops the fetch: the try counts in memory and the run exits 1. - A torn seed is tried at once when the host offers another copy. - Docs: retry times are approximate; a replaced copy lives about 25 h; budget. - Test that tarfile's private parts the limits rely on are there. Co-Authored-By: Claude Opus 5.5 --- .github/fork-fleet.md | 33 +++-- .github/scripts/fork-fleet-desk-seed.py | 84 +++++++---- .github/scripts/fork-fleet-desk-seed.test.py | 146 ++++++++++++++++++- fork-kit/desk/README.md | 8 +- 4 files changed, 228 insertions(+), 43 deletions(-) diff --git a/.github/fork-fleet.md b/.github/fork-fleet.md index 05f6a1f36398..8e2d5e2c0477 100644 --- a/.github/fork-fleet.md +++ b/.github/fork-fleet.md @@ -736,12 +736,15 @@ its own `every_hours` and `max_age_hours` and the smaller of its own and the host's `max_mb`: a host cannot make a desktop take more. Each new seed is unpacked with Python's tar data filter and a limit on what it would make on disk (each member's size plus 4 KiB for it and for each folder it makes, all -within `max_mb`; at most 200,000 members and folders; no member more than 64 +within `max_mb`; a size below 0, or a size on a member that is not a file, is +refused; at most 200,000 members and folders; no member more than 64 folders deep or with a name over 1 KiB) and on memory (members are let go as they are unpacked; the extended pax or long-name headers before one member, and the global ones of the whole tar, at most 1 MiB; sparse members are refused, so make a seed without `--sparse`; a hard link must name a file unpacked before -it), checked against the trailer (its size +it, and a link whose path is already there, or that cannot be made, is refused, +never copied from its target, which tarfile would look for in the rest of the +stream), checked against the trailer (its size and date must be the list's too), checked again for links that lead out of it, made readable by every job (a cache's 0600 files too), and swapped in by a rename; ssh's stderr and the list are read with a limit. A seed that fails in @@ -749,13 +752,16 @@ any way is named in the log and the run goes on with the next; old copies are pruned at the start and the end of every run (so a run killed by its unit's memory or time limit cannot pile them up). A run fetches for at most two hours in all (the unit allows three), each seed at most 30 minutes. A seed torn again -and again is tried again after 1, 2, 4, 8, 16 and then every 24 hours (a link -too slow for it does not spend every hour on it; a fetch that kills the run -counts as torn), and one torn on every try for more than a day makes the run -exit 1, so `desk-install check` shows FAIL until it is fetched -(`/var/lib/fork-fleet/seeds/.torn.json` names it and why). A host that gives no list (down, an older kit, a -half-done install) means no new seeds: each one here stays until its age -drops it. A desktop whose own fleet config is broken or lacks its pool fetches +and again is tried again after about 1, 2, 4, 8, 16 and then every 24 hours (a +link too slow for it does not spend every hour on it; a fetch that kills the +run counts as torn; the times are approximate: the timer's 10 minute random +delay and a 5 minute margin can make a wait up to an hour longer), and at once +when the host offers another copy of it. One torn on every try for more than a +day makes the run exit 1, so `desk-install check` shows FAIL until it is +fetched (`/var/lib/fork-fleet/seeds/.torn.json` names it and why); so does a +run that cannot write that file (a full disk). A host that gives no list (down, +an older kit, a half-done install) means no new seeds: each one here stays until +its age drops it, and the torn record is left as it is. A desktop whose own fleet config is broken or lacks its pool fetches nothing and exits 2 (`desk-install check` shows the seed service's last result), but still drops each seed past the `max_age_hours` it was fetched with. Jobs see @@ -763,11 +769,12 @@ with. Jobs see the unpacking and old copies beside it, `.new` and `.v`, are visible too and hold the same data), copy what they need from it when it is there, and run cold when it is not. A replaced copy is kept 24 hours for a job still reading -it; there is no free-space check before unpacking (a full disk ends that seed's +it and removed by the next hourly prune, so it lives up to about 25 hours; +there is no free-space check before unpacking (a full disk ends that seed's fetch and keeps the old one). **Disk budget per seed:** up to `max_mb` x -(24 / `every_hours` + 2): the copy in use, the copies replaced in the last 24 -hours, and one being unpacked (`every_hours` 1 and `max_mb` 4096: up to about -104 GiB; `every_hours` 6: about 24 GiB). Size `max_mb` and `every_hours` to the +(25 / `every_hours`, rounded up, + 2): the copy in use, the copies replaced in +the last 25 hours, and one being unpacked (`every_hours` 1 and `max_mb` 4096: +up to about 108 GiB; `every_hours` 6: about 28 GiB). Size `max_mb` and `every_hours` to the desktop's free disk. The state table (a torn download, the host down, clocks that disagree, a run that stopped half-way) is in the script's header. diff --git a/.github/scripts/fork-fleet-desk-seed.py b/.github/scripts/fork-fleet-desk-seed.py index 432426950943..f4aa3f3bf947 100644 --- a/.github/scripts/fork-fleet-desk-seed.py +++ b/.github/scripts/fork-fleet-desk-seed.py @@ -31,16 +31,21 @@ the same seed on the host (bytes and MADE) nothing is fetched a new one under every_hours after the last nothing yet a torn copy (the stream cut or past FETCH_WAIT, a trailer that does not match it or the list, more than max_mb, - a tar member that would leave its folder, is a device, too deep or too long, or any other error) + a tar member that would leave its folder, is a device, too deep or too long, has a size below 0 (or one that is + not a file has a size), a link whose path is already there or that cannot be made, or any other error) the copy is removed, the old seed stays, the other seeds go on, old - versions are still pruned; tried again after 1, 2, 4, 8, 16, then 24 - hours (TornLog); torn on every try for over TORN_DAY: the run exits 1 + versions are still pruned; tried again after about 1, 2, 4, 8, 16, then + 24 hours (TornLog; the hourly timer's RandomizedDelaySec and a 5 min + margin make each wait up to an hour longer), at once when the host + offers another copy; torn on every try for over TORN_DAY: exit 1 + .torn.json cannot be written (a full disk) the run goes on, counts its tries in memory, and exits 1 a fetch that kills the run (MemoryMax, TimeoutStartSec) counted as torn before it began; the next run prunes first the run's RUN_WAIT used the seeds left are tried next hour no seed (never fetched, the host down) jobs run cold no list (the host down, an older kit that answers with a session or not JSON, exit 1 of a half-done install) - nothing fetched; each seed stays until its age drops it + nothing fetched, exit 0; each seed stays until its age drops it; the + torn record (.torn.json) is left as it is its own fleet config broken or without POOL nothing fetched, exit 2; a seed older than its label's maxAgeHours (its own max_age_hours when it was fetched) is still dropped older than max_age_hours, made over SKEW in the future (the clocks disagree), or no longer in the pool's list @@ -50,6 +55,7 @@ any depth); a link is the old one or the new a replaced version kept KEEP_OLD after it was replaced, for a job still reading it (a job runs at most the launcher's MAX_SECONDS, 1 h by default), then removed + by the next hourly prune: it lives up to about 25 h The forced command sends every command that starts with `seed` here. The host side exits 2 for any other command, 3 while another stream for the pool runs, 4 when the file may not be sent (missing, a link, in a folder or a file @@ -358,15 +364,25 @@ def sparse_refused(self, member=None, *_): _proc_gnusparse_00 = _proc_gnusparse_01 = _proc_gnusparse_10 = sparse_refused # pax sparse forms +class Tar(tarfile.TarFile): + """A link that cannot be made (its path is there, or os.link fails) is never copied from its target: tarfile + would read the whole rest of the stream into memory to find it (_find_link_target), past every limit here.""" + + def _find_link_target(self, tarinfo): + raise Torn(f'{tarinfo.name[:80]!r} is a link that cannot be made here (its target is not copied instead)') + + def unpack(t, data, most): - """The members of the open stream T into DATA, one at a time: tarfile keeps every member it has read in - `members` (and extractall each folder besides), so they are let go as they are made; only `guarded`'s count of - folders stays. A hard link must name a file already unpacked (otherwise tarfile would read the rest of the - stream into memory to find it).""" + """The members of the open stream T (a Tar) into DATA, one at a time: tarfile keeps every member it has read + in `members` (and extractall each folder besides), so they are let go as they are made; only `guarded`'s count + of folders stays. A link's path must not be there yet (tarfile would unlink a file, fail on a folder) and a hard + link must name a file already unpacked (otherwise tarfile would look for it in the rest of the stream).""" check = guarded(most) def linked(member, path): got = check(member, path) + if got is not None and (got.islnk() or got.issym()) and os.path.lexists(os.path.join(path, got.name)): + raise Torn(f'{got.name[:80]!r} is a link to {got.linkname[:80]!r} where a member is already there') if got is not None and got.islnk(): try: plain = stat.S_ISREG(os.lstat(os.path.join(path, got.linkname)).st_mode) @@ -390,6 +406,8 @@ def guarded(most): seen, folders = [0, 0, 0], {} # folders: {(its parent's number, its name): its number}, a name kept once def check(member, path): + if member.size < 0 or (member.size and not member.isreg()): # a size below 0 would lower the count + raise Torn(f'{member.name[:80]!r} has the size {member.size} (a member that is not a file has 0)') parts = [x for x in member.name.split('/') if x not in ('', '.')] if len(parts) > DEPTH: raise Torn(f'{"/".join(parts[:3])}/... is more than {DEPTH} folders deep') @@ -450,7 +468,7 @@ def fetch(agent, conf, item, root, now, wait=FETCH_WAIT): err = Tail(p.stderr) try: got = Summed(p.stdout, most) - with tarfile.open(fileobj=got, mode='r|', tarinfo=Info) as t: + with Tar.open(fileobj=got, mode='r|', tarinfo=Info) as t: unpack(t, os.path.join(new, 'data'), most) while got.read(PIECE): # the tar's end blocks and padding count in the trailer's size and sha256 pass @@ -596,17 +614,21 @@ def run(agent, conf, root=SEEDS, now=time.time, say=print, config=CONFIG): items = {n: {**host[n], 'everyHours': s['every_hours'], 'maxAgeHours': s['max_age_hours'], 'maxMb': min(host[n]['maxMb'], s['max_mb'])} for n, s in own.items() if n in host} except Torn as e: # the host down, or a host on an older kit (no list, or not one this reads) - items = {} say(f'seeds: {e}: each seed stays until its age drops it') + return 0 # the torn record stays as it is: no list says nothing about a seed torn, end = TornLog(root, now), time.monotonic() + RUN_WAIT torn.keep(items) for name, item in sorted(items.items()): if not each(say, f'seed {name}', take, agent, conf, item, root, now, say, torn, end): clean(root, say) - each(say, 'seeds: .torn.json', torn.save) + torn.save() + if torn.unsaved: + say(f'seeds: .torn.json cannot be written ({torn.unsaved}; a full disk?): exit 1 (desk-install ' + f'check shows it; the tries of this run are not counted for the next)') if late := torn.late(): say(f'seeds: {", ".join(late)} torn on every try for more than {TORN_DAY // 3600} h: exit 1 ' f'(desk-install check shows it)') + if torn.unsaved or late: return 1 finally: prune(root, now, say) @@ -614,15 +636,18 @@ def run(agent, conf, root=SEEDS, now=time.time, say=print, config=CONFIG): class TornLog: - """SEEDS/.torn.json, {name: {"first", "last", "tries", "why"}}: the seeds whose last fetch was torn, written - before each fetch (so a fetch that kills the run, by its memory or time limit, counts as torn too) and cleared - when the seed is fetched or the same as the host's. A seed torn TRIES times in a row is tried again only + """SEEDS/.torn.json, {name: {"first", "last", "tries", "why", "copy"}}: the seeds whose last fetch was torn, + written before each fetch (so a fetch that kills the run, by its memory or time limit, counts as torn too) and + cleared when the seed is fetched or the same as the host's. A seed torn TRIES times in a row is tried again only min(2 ** (TRIES - 1), 24) hours after the last try (a link too slow for it, a host that cuts it: not every - hour); one torn on every try since more than TORN_DAY is `late`. Only seeds in the host's list are kept. A - missing or unreadable file is an empty record.""" + hour), or at once when the host offers another copy (`copy`: its [bytes, made] when last tried); one torn on + every try since more than TORN_DAY is `late`. Only seeds in the host's list are kept, and only in a run that + read the list. A missing or unreadable file is an empty record; one that cannot be written (a full disk) is + kept in memory for this run and said in `unsaved`.""" def __init__(self, root, now): self.path, self.now, self.seeds = root and os.path.join(root, '.torn.json'), now, {} + self.unsaved = None try: with open(self.path, encoding='utf-8') as f: got = json.load(f) @@ -634,20 +659,23 @@ def __init__(self, root, now): def keep(self, items): self.seeds = {n: r for n, r in self.seeds.items() if n in items} - def due(self, name): - """None when the seed may be tried now, else the hours until it may.""" + def due(self, name, item=None): + """None when the seed may be tried now (ITEM, the host's offer, another copy than the one last tried + included), else the hours until it may.""" r = self.seeds.get(name) - if not r: + if not r or item and r.get('copy') not in (None, [item['bytes'], item['made']]): return None wait = min(2 ** (min(r['tries'], 6) - 1), 24) * 3600 - 300 # 5 min early: the hourly timer drifts left = r['last'] + wait - self.now() return math.ceil(left / 3600) if 0 < left <= wait else None # a date ahead of the clock: try now - def trying(self, name): + def trying(self, name, item=None): """Counted as torn before the fetch, saved: `why` says so until the fetch ends.""" r = self.seeds.get(name) or {'first': int(self.now()), 'tries': 0} self.seeds[name] = {**r, 'last': int(self.now()), 'tries': r['tries'] + 1, 'why': 'the run stopped during the fetch (its memory or time limit?)'} + if item: + self.seeds[name]['copy'] = [item['bytes'], item['made']] self.save() def torn(self, name, why): @@ -662,12 +690,18 @@ def late(self): return sorted(n for n, r in self.seeds.items() if self.now() - r['first'] > TORN_DAY) def save(self): + """Written by tmp + rename; an OSError (a full disk) is kept in `unsaved` (cleared by a save that works), + never raised: the run goes on, counts in memory and exits 1.""" if self.path is None: return tmp = self.path + '.tmp' - with open(tmp, 'w', encoding='utf-8') as f: - json.dump(self.seeds, f) - os.replace(tmp, self.path) + try: + with open(tmp, 'w', encoding='utf-8') as f: + json.dump(self.seeds, f) + os.replace(tmp, self.path) + self.unsaved = None + except OSError as e: + self.unsaved = f'{e.__class__.__name__}: {e.strerror or e}' def each(say, what, step, *args): @@ -722,14 +756,14 @@ def take(agent, conf, item, root, now, say, torn=None, end=None): if not -SKEW <= now() - item['made'] <= item['maxAgeHours'] * 3600: say(f'seed {name}: the host\'s is too old or made in the future (the clocks disagree): not fetched') return - if (hours := torn.due(name)) is not None: + if (hours := torn.due(name, item)) is not None: say(f'seed {name}: torn {torn.seeds[name]["tries"]} times in a row; tried again in about {hours} h') return left = end - time.monotonic() if left < 60: say(f'seed {name}: this run has used its {RUN_WAIT // 60} minutes; tried again next hour') return - torn.trying(name) + torn.trying(name, item) try: got = fetch(agent, conf, item, root, now, wait=min(FETCH_WAIT, left)) except Exception as e: # any: a torn stream, a full disk, a tar this Python cannot unpack diff --git a/.github/scripts/fork-fleet-desk-seed.test.py b/.github/scripts/fork-fleet-desk-seed.test.py index f5607843f7e3..53982322e984 100644 --- a/.github/scripts/fork-fleet-desk-seed.test.py +++ b/.github/scripts/fork-fleet-desk-seed.test.py @@ -795,8 +795,6 @@ def test_extended_headers_and_sparse_members_are_held_to_limits(self): self.assertTrue(any('m2' in x and why in x for x in self.said), self.said) with contextlib.suppress(FileNotFoundError): # each case tried at once, not after a wait os.unlink(os.path.join(self.root, '.torn.json')) - for k in ('_proc_member', '_proc_sparse', '_proc_gnusparse_00', '_proc_gnusparse_01', '_proc_gnusparse_10'): - self.assertTrue(callable(getattr(tarfile.TarInfo, k, None)), f'tarfile.TarInfo.{k} is gone: see Info') def test_members_are_let_go_as_they_are_unpacked(self): self.make('m2', {f'd/{i}': b'x' for i in range(300)}) @@ -825,6 +823,150 @@ def hard(name, target): self.assertEqual(self.files('m2'), {'a': b'one', 'b': b'one'}) # the old one stays self.assertTrue(any('hard link' in x for x in self.said), self.said) + def test_the_private_tarfile_parts_the_limits_rely_on_are_there(self): # lens237sr3: fail loud, not silently + for k in ('_proc_member', '_proc_sparse', '_proc_gnusparse_00', '_proc_gnusparse_01', '_proc_gnusparse_10'): + self.assertTrue(callable(getattr(tarfile.TarInfo, k, None)), f'tarfile.TarInfo.{k} is gone: see Info') + self.assertTrue(callable(getattr(tarfile.TarFile, '_find_link_target', None)), + 'tarfile.TarFile._find_link_target is gone: see Tar') + self.assertIs(seed.Tar._find_link_target, seed.Tar.__dict__['_find_link_target']) + + def forked(self): + """run_seed() in a child: 0 when it stayed under the unit's MemoryMax=512M, 1 when not (its peak RSS, or a + MemoryError: the child may map 1 GB at most, so the test never takes the machine's memory).""" + r, w = os.pipe() + pid = os.fork() + if pid == 0: + import resource + code = 2 + try: + os.close(r) + resource.setrlimit(resource.RLIMIT_AS, (1 << 30, 1 << 30)) + self.run_seed() + os.write(w, json.dumps(self.said).encode()) + code = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss >= 512 << 10 + or any('MemoryError' in x for x in self.said)) + finally: + os._exit(code) + os.close(w) + with os.fdopen(r) as f: + self.said = json.loads(f.read() or '[]') + return os.waitpid(pid, 0)[1] >> 8 + + def test_a_link_that_cannot_be_made_never_loads_the_stream(self): # lens237sr3: the MUST + big = [{**SEEDS[0], 'max_mb': 128}, SEEDS[1]] + self.config(big) + self.own(big) + x = tarfile.TarInfo('pax') + x.type = tarfile.XHDTYPE + body = b''.join(b'15 a.%07d=x\n' % i for i in range(65_000)) # ~0.98 MB, under HEADER_MOST + rest = b''.join(self.block(x, body) + self.block(tarfile.TarInfo(f'f{i}')) for i in range(110)) # ~108 MB + del body + with open(os.path.join(self.t, 'rest'), 'wb') as f: # kept on disk: the child must not inherit it + f.write(rest) + del rest + + def rest(): + with open(os.path.join(self.t, 'rest'), 'rb') as f: + return f.read() + + def link(name, target, kind): + info = tarfile.TarInfo(name) + info.type, info.linkname = kind, target + return self.block(info) + cases = { # a link whose path is there: os.link fails; os.unlink of a folder fails + 'a hard link over a file': self.block(tarfile.TarInfo('a'), b'1') + self.block(tarfile.TarInfo('b'), b'2') + + link('b', 'a', tarfile.LNKTYPE), + 'a symlink over a folder': self.block(tarfile.TarInfo('d/x'), b'1') + link('d', 'x', tarfile.SYMTYPE), + } + for i, (what, head) in enumerate(cases.items()): + with self.subTest(what): + self.raw('m2', head + rest()) + self.assertEqual(self.forked(), 0, self.said[-3:]) # hard link: over 512 MB before round 4 + self.assertIsNone(self.files('m2')) + self.assertTrue(any('m2' in x and 'already there' in x for x in self.said), self.said) + with contextlib.suppress(FileNotFoundError): + os.unlink(os.path.join(self.root, '.torn.json')) + self.raw('m2', self.block(tarfile.TarInfo('a'), b'1') + link('b', 'a', tarfile.LNKTYPE)) + with mock.patch.object(os, 'link', side_effect=OSError(18, 'Invalid cross-device link')): # any other + self.run_seed() # link failure + self.assertIsNone(self.files('m2')) + self.assertTrue(any('m2' in x and 'cannot be made' in x for x in self.said), self.said) + + def test_a_negative_size_never_lowers_the_disk_count(self): # lens237sr3: the MUST + d = bytearray(tarfile.TarInfo('z').tobuf(format=tarfile.GNU_FORMAT)) + d[156:157] = tarfile.DIRTYPE + d[124:136] = b'\xff' + b'\0' * 11 # a base-256 size below zero + d[148:156] = b' ' * 8 + d[148:156] = b'%06o\0 ' % sum(d[:512]) + self.raw('m2', bytes(d) + b''.join(self.block(tarfile.TarInfo(f'e{i}')) for i in range(1500))) + self.run_seed() + self.assertIsNone(self.files('m2')) # before round 4: 1,500 files (6 MB of entries) under max_mb 2 + self.assertTrue(any('m2' in x and 'size' in x for x in self.said), self.said) + f = bytearray(tarfile.TarInfo('y').tobuf(format=tarfile.GNU_FORMAT)) # a folder with a size: refused too + f[156:157] = tarfile.DIRTYPE + f[124:136] = b'%011o\0' % 512 + f[148:156] = b' ' * 8 + f[148:156] = b'%06o\0 ' % sum(f[:512]) + self.raw('m2', bytes(f) + b'\0' * 512 + self.block(tarfile.TarInfo('e'), b'x')) + self.run_seed(now=lambda: time.time() + 2 * 3600) + self.assertIsNone(self.files('m2')) + self.assertTrue(any('m2' in x and 'size' in x for x in self.said), self.said) + + def test_one_lost_list_does_not_reset_the_torn_record(self): # lens237cr3 + self.make('m2', {'a': b'one'}) + start, codes, real, tried = int(time.time()), {}, seed.listing, [] + + def torn(agent_, conf, item, root, now, **kw): + tried.append(round((now() - start) / 3600)) + raise seed.Torn('cut off') + + def lost(h): + def listing(a, c): + if h % 12 == 11: # the 30 s list lost twice a day + raise seed.Torn('no list in 30 s') + return real(a, c) + return listing + with mock.patch.object(seed, 'fetch', torn): + for h in range(49): + with mock.patch.object(seed, 'listing', lost(h)): + codes[h] = self.run_seed(now=lambda h=h: start + h * 3600) + self.assertEqual(tried, [0, 1, 3, 7, 15, 31]) # the back-off goes on + self.assertEqual({h for h, c in codes.items() if c}, set(range(25, 49)) - {35, 47}) # late, list or not + + def test_a_torn_record_that_cannot_be_saved_still_counts_and_fails_the_run(self): # lens237cr3: disk full + self.make('m2', {'a': b'one'}) + os.makedirs(os.path.join(self.root, '.torn.json.tmp')) # the record cannot be written, as on a full disk + tried = [] + + def torn(agent_, conf, item, root, now, **kw): + tried.append(item['name']) + raise seed.Torn('No space left on device') + with mock.patch.object(seed, 'fetch', torn): + self.assertEqual(self.run_seed(), 1) + self.assertEqual(tried, ['m2']) # tried, and the try counted in memory + self.assertTrue(any('.torn.json' in x and 'cannot be written' in x for x in self.said), self.said) + os.rmdir(os.path.join(self.root, '.torn.json.tmp')) + with mock.patch.object(seed, 'fetch', torn): + self.assertEqual(self.run_seed(now=lambda: time.time() + 3600), 0) + with open(os.path.join(self.root, '.torn.json')) as f: + self.assertEqual(json.load(f)['m2']['tries'], 1) # the lost record starts again, no worse + + def test_a_new_copy_on_the_host_is_tried_at_once_after_torn_tries(self): # lens237cr3 nit + start, tried = int(time.time()), [] + self.make('m2', {'a': b'one'}, mtime=start - 3600) + + def torn(agent_, conf, item, root, now, **kw): + tried.append(round((now() - start) / 3600)) + raise seed.Torn('cut off') + with mock.patch.object(seed, 'fetch', torn): + for h in (0, 1, 2): + self.run_seed(now=lambda h=h: start + h * 3600) + self.make('m2', {'a': b'two!'}, mtime=start - 60) # the host made a fixed one + self.run_seed(now=lambda: start + 2 * 3600 + 120) + self.run_seed(now=lambda: start + 2 * 3600 + 180) + self.assertEqual(tried, [0, 1, 2]) # at once, then the back-off again (now 4 h) + self.assertTrue(any('torn 3 times' in x for x in self.said), self.said) + def test_a_deeply_nested_list_means_no_list(self): # lens237cr2 self.make('m2', {'a': b'one'}) self.run_seed() diff --git a/fork-kit/desk/README.md b/fork-kit/desk/README.md index 9e0b34c37a55..67b78733d835 100644 --- a/fork-kit/desk/README.md +++ b/fork-kit/desk/README.md @@ -69,9 +69,11 @@ hand setup) is refused the same way, not followed or fixed. One install or unins The seed timer (`t3code-fleet-seed.timer`) fetches the pool's warm caches, if its fleet config lists any, into `/var/lib/fork-fleet/seeds`, which jobs see read-only ([fork-fleet.md](../../.github/fork-fleet.md), "Seeds"). -A torn seed is tried again after 1, 2, 4, 8, 16, then 24 hours; one torn on every try for more than a day makes each -run exit 1, so `check` shows the seed service's last run as failed. `/var/lib/fork-fleet/seeds/.torn.json` names it -and the host's reason (jobs can read that file too). +A torn seed is tried again after about 1, 2, 4, 8, 16, then 24 hours (up to an hour later: the timer's random delay), +or at once when the host offers another copy; one torn on every try for more than a day makes each run that reads the +host's list exit 1, so `check` shows the seed service's last run as failed (a run that cannot write the record, a full +disk, exits 1 too). `/var/lib/fork-fleet/seeds/.torn.json` names it and the host's reason (jobs can read that file +too); a run without the host's list leaves it as it is. ## Windows (WSL2)