From c8f43ebce73071d7fbc6f570e35c02c526ac3ebd Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:15:54 -0500 Subject: [PATCH 01/84] feat(k3s-p1): add inert deterministic worker manifest/lease preview --- examples/k3s-p1/plan.py | 184 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 examples/k3s-p1/plan.py diff --git a/examples/k3s-p1/plan.py b/examples/k3s-p1/plan.py new file mode 100644 index 0000000000000..8af66bd933be3 --- /dev/null +++ b/examples/k3s-p1/plan.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""P1 preview: deterministic, fail-closed T3 worker plan, with NO cluster writes. + +A stored local lease-plan is NOT a running lease: there is no Kubernetes apply, +lease renewal, expiry controller, peer OAuth grant, or T3 task submission here. +P0 live acceptance remains a prerequisite to any P1 rollout. +""" + +import argparse +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import sys + +import yaml + +KEY = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}\Z") +IMAGE = re.compile(r"(?=.{1,255}$)[a-z0-9][a-z0-9._:/-]*/[a-z0-9][a-z0-9._/-]*@sha256:[a-f0-9]{64}\Z") +UTC = re.compile(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ\Z") + + +class PlanError(ValueError): + """Invalid plan or conflicting immutable lease request.""" + + +def build(lease_key: str, image: str, expires_at: str) -> tuple[dict, list[dict]]: + if not isinstance(lease_key, str) or not KEY.fullmatch(lease_key) or '..' in lease_key: + raise PlanError('lease key must be a stable, bounded opaque ASCII identifier') + if not isinstance(image, str) or not IMAGE.fullmatch(image) or '..' in image or '//' in image: + raise PlanError('image must use an immutable registry/repository@sha256 digest') + if not isinstance(expires_at, str) or not UTC.fullmatch(expires_at): + raise PlanError('expires-at must be an exact UTC timestamp YYYY-MM-DDTHH:MM:SSZ') + try: + timestamp = datetime.strptime(expires_at, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc) + except ValueError as exc: + raise PlanError('invalid UTC expiration timestamp') from exc + if timestamp.year < 2020 or timestamp.year > 2100: + raise PlanError('expiration outside supported planning bounds') + + full_hash = hashlib.sha256(lease_key.encode('ascii')).hexdigest() + suffix = full_hash[:24] + ns = f't3-p1-{suffix}' + ingress_host = f't3-p1-{suffix}' + labels = {'app': 't3-worker'} + annotations = {'t3code.dev/lease-key-sha256': full_hash, + 't3code.dev/lease-expires-at': expires_at, + 't3code.dev/expiry-enforced': 'false', + 't3code.dev/retention': 'retain-until-owner-decision'} + plan = {'format': 't3-k3s-p1-plan-v1', 'leaseHash': full_hash, 'namespace': ns, + 'image': image, 'expiresAt': expires_at, 'retention': 'retain'} + + def obj(api: str, kind: str, name: str, spec: dict | None = None) -> dict: + doc = {'apiVersion': api, 'kind': kind, 'metadata': {'name': name, 'namespace': ns}} + if spec is not None: + doc['spec'] = spec + return doc + + namespace = {'apiVersion': 'v1', 'kind': 'Namespace', 'metadata': { + 'name': ns, 'annotations': annotations, + 'labels': {'pod-security.kubernetes.io/enforce': 'restricted', + 'pod-security.kubernetes.io/enforce-version': 'latest'}}} + quota = obj('v1', 'ResourceQuota', 'worker-quota', {'hard': { + 'pods': '1', 'persistentvolumeclaims': '1', 'requests.cpu': '2', + 'requests.memory': '4Gi', 'limits.cpu': '4', 'limits.memory': '8Gi', + 'requests.storage': '20Gi', 'requests.ephemeral-storage': '2Gi', + 'limits.ephemeral-storage': '4Gi'}}) + sa = obj('v1', 'ServiceAccount', 't3-worker') + sa['automountServiceAccountToken'] = False + pvc = obj('v1', 'PersistentVolumeClaim', 't3-worker-data', { + 'accessModes': ['ReadWriteOnce'], 'resources': {'requests': {'storage': '10Gi'}}}) + container = { + 'name': 't3', 'image': image, 'imagePullPolicy': 'IfNotPresent', + 'workingDir': '/data', 'ports': [{'name': 'http', 'containerPort': 3773}], + 'resources': {'requests': {'cpu': '500m', 'memory': '1Gi', 'ephemeral-storage': '512Mi'}, + 'limits': {'cpu': '2', 'memory': '4Gi', 'ephemeral-storage': '2Gi'}}, + 'securityContext': {'allowPrivilegeEscalation': False, 'readOnlyRootFilesystem': True, + 'capabilities': {'drop': ['ALL']}}, + 'startupProbe': {'tcpSocket': {'port': 'http'}, 'periodSeconds': 5, 'failureThreshold': 60}, + 'readinessProbe': {'tcpSocket': {'port': 'http'}, 'periodSeconds': 5}, + 'volumeMounts': [{'name': 'data', 'mountPath': '/data'}, + {'name': 'tmp', 'mountPath': '/tmp'}], + } + pod = { + 'serviceAccountName': 't3-worker', 'automountServiceAccountToken': False, + 'hostname': 't3-worker', 'terminationGracePeriodSeconds': 90, + 'securityContext': {'runAsNonRoot': True, 'runAsUser': 10001, + 'runAsGroup': 10001, 'fsGroup': 10001, + 'seccompProfile': {'type': 'RuntimeDefault'}}, + 'containers': [container], + 'volumes': [{'name': 'data', 'persistentVolumeClaim': {'claimName': 't3-worker-data'}}, + {'name': 'tmp', 'emptyDir': {'sizeLimit': '1Gi'}}], + } + deployment = obj('apps/v1', 'Deployment', 't3-worker', { + 'replicas': 1, 'strategy': {'type': 'Recreate'}, 'selector': {'matchLabels': labels}, + 'template': {'metadata': {'labels': labels}, 'spec': pod}}) + service = obj('v1', 'Service', 't3-worker', { + 'type': 'ClusterIP', 'selector': labels, + 'ports': [{'name': 'http', 'port': 80, 'targetPort': 'http', 'protocol': 'TCP'}]}) + ingress = obj('networking.k8s.io/v1', 'Ingress', 't3-worker', { + 'ingressClassName': 'tailscale', 'tls': [{'hosts': [ingress_host]}], + 'defaultBackend': {'service': {'name': 't3-worker', 'port': {'name': 'http'}}}}) + policy = obj('networking.k8s.io/v1', 'NetworkPolicy', 'worker-ingress', { + 'podSelector': {'matchLabels': labels}, 'policyTypes': ['Ingress'], + 'ingress': [{'from': [{'namespaceSelector': {'matchLabels': { + 'kubernetes.io/metadata.name': 'tailscale'}}}], + 'ports': [{'protocol': 'TCP', 'port': 3773}]}]}) + return plan, [namespace, quota, sa, pvc, deployment, service, ingress, policy] + + +def lock(state_dir: Path, plan: dict) -> bool: + """Create-once 0600 local receipt; retries match exactly or fail closed. + + Requires owner-created private absolute directory. Never overwrites a + conflicting or symlinked plan; never contacts the cluster. + """ + if not state_dir.is_absolute(): + raise PlanError('state directory must be an absolute owner-private path') + flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) | getattr(os, 'O_NOFOLLOW', 0) + try: + directory = os.open(state_dir, flags) + except OSError as exc: + raise PlanError('state directory unavailable') from exc + try: + st = os.fstat(directory) + if not stat.S_ISDIR(st.st_mode) or (st.st_mode & 0o077) or st.st_uid != os.getuid(): + raise PlanError('state directory must be owned by caller with mode 0700') + filename = plan['leaseHash'] + '.json' + data = (json.dumps(plan, sort_keys=True, separators=(',', ':')) + '\n').encode('utf-8') + file_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_NOFOLLOW', 0) + try: + file = os.open(filename, file_flags, 0o600, dir_fd=directory) + except FileExistsError: + file = None + except OSError as exc: + raise PlanError('cannot establish local lease receipt') from exc + if file is not None: + with os.fdopen(file, 'wb') as output: + output.write(data) + output.flush() + os.fsync(output.fileno()) + return True + read_flags = os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) + try: + file = os.open(filename, read_flags, dir_fd=directory) + with os.fdopen(file, 'rb') as source: + info = os.fstat(source.fileno()) + if (not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077 + or info.st_uid != os.getuid() or info.st_size > 4096): + raise PlanError('existing lease receipt is unsafe') + previous = source.read(4097) + except OSError as exc: + raise PlanError('existing lease receipt unavailable or unsafe') from exc + if previous != data: + raise PlanError('same lease key has conflicting image, expiry, or plan contract') + return False + finally: + os.close(directory) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--lease-key', required=True, help='opaque stable idempotency key (not printed)') + parser.add_argument('--image', required=True, help='T3-compatible worker image@sha256 digest') + parser.add_argument('--expires-at', required=True, help='UTC expiry metadata; NOT enforced') + parser.add_argument('--state-dir', required=True, type=Path, help='pre-existing private (0700) directory') + args = parser.parse_args() + try: + plan, documents = build(args.lease_key, args.image, args.expires_at) + created = lock(args.state_dir, plan) + except PlanError as exc: + parser.exit(1, f'FAIL: {exc}\n') + # stdout contains declarative YAML only; operator must separately inspect, + # authorize, and apply. No automatic time-based retention/deletion. + print(yaml.safe_dump_all(documents, sort_keys=False, explicit_start=True), end='') + print('P1 PREVIEW ONLY: no cluster write, OAuth pairing, TTL action, or worker execution; ' + f'local receipt {"created" if created else "reused"}', file=sys.stderr) + + +if __name__ == '__main__': + main() From d58987bebfadc865717b44013342feebdfbf04ce Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:16:10 -0500 Subject: [PATCH 02/84] test(k3s-p1): exercise deterministic plans and conflict-safe lease receipts --- examples/k3s-p1/test_plan.py | 117 +++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 examples/k3s-p1/test_plan.py diff --git a/examples/k3s-p1/test_plan.py b/examples/k3s-p1/test_plan.py new file mode 100644 index 0000000000000..fd407acfd99b4 --- /dev/null +++ b/examples/k3s-p1/test_plan.py @@ -0,0 +1,117 @@ +"""P1 manifest/lease contract: deterministic, no network, kubectl or scheduler.""" +import importlib.util +from pathlib import Path +import json +import os +import tempfile +import unittest + +import yaml + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('p1_plan', HERE / 'plan.py') +assert spec and spec.loader +p = importlib.util.module_from_spec(spec) +spec.loader.exec_module(p) + +IMAGE = 'ghcr.io/owned/t3-worker@sha256:' + 'a' * 64 +EXPIRY = '2026-11-01T00:00:00Z' + + +class P1PlanTests(unittest.TestCase): + def test_deterministic_names_and_same_input(self): + a, objs = p.build('work-item:abc-23', IMAGE, EXPIRY) + b, again = p.build('work-item:abc-23', IMAGE, EXPIRY) + self.assertEqual(a, b) + self.assertEqual(yaml.safe_dump_all(objs), yaml.safe_dump_all(again)) + c, _ = p.build('work-item:abc-24', IMAGE, EXPIRY) + self.assertNotEqual(a['namespace'], c['namespace']) + self.assertNotIn('work-item:abc', yaml.safe_dump_all(objs)) + self.assertNotIn('work-item:abc', json.dumps(a)) + + def test_only_bounded_private_k8s_resources(self): + plan, docs = p.build('work:1', IMAGE, EXPIRY) + self.assertEqual([d['kind'] for d in docs], ['Namespace', 'ResourceQuota', + 'ServiceAccount', 'PersistentVolumeClaim', 'Deployment', + 'Service', 'Ingress', 'NetworkPolicy']) + self.assertTrue(all(d['metadata']['namespace'] == plan['namespace'] + for d in docs[1:])) + self.assertEqual(docs[0]['metadata']['labels']['pod-security.kubernetes.io/enforce'], 'restricted') + self.assertEqual(docs[0]['metadata']['annotations']['t3code.dev/expiry-enforced'], 'false') + self.assertEqual(docs[1]['spec']['hard']['pods'], '1') + self.assertFalse(docs[2]['automountServiceAccountToken']) + pod = docs[4]['spec']['template']['spec'] + self.assertFalse(pod['automountServiceAccountToken']) + self.assertTrue(pod['securityContext']['runAsNonRoot']) + self.assertEqual(docs[4]['spec']['strategy']['type'], 'Recreate') + self.assertEqual(pod['containers'][0]['image'], IMAGE) + self.assertEqual(pod['containers'][0]['securityContext']['capabilities']['drop'], ['ALL']) + self.assertTrue(pod['containers'][0]['securityContext']['readOnlyRootFilesystem']) + self.assertTrue(any(v.get('persistentVolumeClaim', {}).get('claimName') == 't3-worker-data' + for v in pod['volumes'])) + self.assertFalse(any('hostPath' in v for v in pod['volumes'])) + self.assertEqual(docs[5]['spec']['type'], 'ClusterIP') + self.assertEqual(docs[6]['spec']['ingressClassName'], 'tailscale') + self.assertNotIn('rules', docs[6]['spec']) + self.assertNotIn('tailscale.com/funnel', docs[6]['metadata'].get('annotations', {})) + self.assertEqual(docs[7]['spec']['policyTypes'], ['Ingress']) + self.assertEqual(docs[7]['spec']['ingress'][0]['ports'][0]['port'], 3773) + self.assertEqual(docs[7]['spec']['ingress'][0]['from'][0]['namespaceSelector'] + ['matchLabels']['kubernetes.io/metadata.name'], 'tailscale') + self.assertEqual([d['kind'] for d in yaml.safe_load_all(yaml.safe_dump_all(docs))], + [d['kind'] for d in docs]) + + def test_bad_keys_images_and_expiries_fail_closed(self): + for key in ('', '/bad', '-bad', '../steal', 'a\nkind:Pod', 'x' * 129): + with self.subTest(key=key), self.assertRaises(p.PlanError): + p.build(key, IMAGE, EXPIRY) + for image in ('t3:latest', 't3@sha256:' + 'a' * 64, + IMAGE.upper(), IMAGE + '\n---\nkind: Pod', + 'ghcr.io/x/../y@sha256:' + 'a' * 64): + with self.subTest(image=image), self.assertRaises(p.PlanError): + p.build('work:1', image, EXPIRY) + for date in ('', 'tomorrow', '2026-02-30T00:00:00Z', + '2026-11-01T00:00:00+01:00', '2000-01-01T00:00:00Z'): + with self.subTest(date=date), self.assertRaises(p.PlanError): + p.build('work:1', IMAGE, date) + + def test_local_receipt_is_0600_and_conflicts_fail(self): + with tempfile.TemporaryDirectory() as temp: + base = Path(temp) / 'private' + base.mkdir(mode=0o700) + plan, docs = p.build('work:1', IMAGE, EXPIRY) + self.assertTrue(p.lock(base, plan)) + self.assertFalse(p.lock(base, plan)) + path = base / (plan['leaseHash'] + '.json') + self.assertEqual(os.stat(path).st_mode & 0o777, 0o600) + self.assertNotIn('work:1', path.read_text()) + changed = dict(plan, image='ghcr.io/other/w@sha256:' + 'b' * 64) + with self.assertRaisesRegex(p.PlanError, 'conflicting'): + p.lock(base, changed) + path.chmod(0o644) + with self.assertRaisesRegex(p.PlanError, 'unsafe'): + p.lock(base, plan) + + def test_rejects_symlinked_or_public_plan_directory(self): + with tempfile.TemporaryDirectory() as temp: + private = Path(temp) / 'private' + private.mkdir(mode=0o700) + public = Path(temp) / 'public' + public.mkdir(mode=0o755) + symlink = Path(temp) / 'alias' + symlink.symlink_to(private) + plan, _ = p.build('work:2', IMAGE, EXPIRY) + with self.assertRaises(p.PlanError): + p.lock(public, plan) + with self.assertRaises(p.PlanError): + p.lock(symlink, plan) + with self.assertRaises(p.PlanError): + p.lock(Path('relative'), plan) + path = private / (plan['leaseHash'] + '.json') + path.symlink_to(public) + with self.assertRaises(p.PlanError): + p.lock(private, plan) + + +if __name__ == '__main__': + unittest.main() From 81aa096641b42fbbb0aadb42cb63171c209ba017 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:17:05 -0500 Subject: [PATCH 03/84] docs(k3s-p1): bound preview, retention and P0 prerequisite --- examples/k3s-p1/README.md | 78 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 examples/k3s-p1/README.md diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md new file mode 100644 index 0000000000000..d39d622737fc6 --- /dev/null +++ b/examples/k3s-p1/README.md @@ -0,0 +1,78 @@ +# P1 preview — deterministic worker plans, no cluster actuation + +**LOCAL CONTRACT ONLY / NOT RUN ON K3s.** This is a stacked preparatory +slice against [P0 draft #46](https://github.com/kvnloo/t3code/pull/46). +No P1 deployment is authorized until the P0 persistent owner passes real +disposable-cluster storage/restart, private HTTPS/ACL, client pairing and +saved-thread acceptance. + +## Scope + +plan.py turns a stable opaque idempotency key, immutable OCI image digest and +explicit UTC expiration into a reproducible **per-worker environment +blueprint**. The key is hashed (full SHA-256 in a private receipt, truncated +only for DNS names). The raw key is neither logged nor written to Kubernetes. +The file outputs exactly eight ordinary resources: restricted Namespace, +ResourceQuota, no-token ServiceAccount, separate PVC, one Recreate Deployment +running a headless T3 server, ClusterIP Service, private Tailscale Ingress and +an ingress-only NetworkPolicy. + +The local private mode-0600 receipt is create-once. A retry with identical +arguments renders identical YAML. Reusing the key with changed image/expiry +fails closed rather than silently rebinding a purported lease. The receipt +is **not** a distributed lock or a cluster reservation. Expiration is advisory +metadata only; **no TTL cleanup or enforcement exists**. PVC retention always +requires a human decision. The CNI and tailnet ACLs still need negative tests; +the manifest does not restrict egress. + +## Offline usage + +Requires Python 3.10+ and PyYAML. In the fork checkout: + + python3 -m pip install 'PyYAML==6.0.2' + python3 -m unittest discover -s examples/k3s-p1 -p 'test_*.py' -v + +Create a private receipt directory locally: + + mkdir -m 700 /tmp/k3s-p1-receipts + +After replacing the placeholder with a **real** verified 64-character +lowercase image digest, run: + + python3 examples/k3s-p1/plan.py \ + --lease-key 'approved-request:example-001' \ + --image 'ghcr.io/OWNER/t3-worker@sha256:REPLACE_WITH_64_HEX' \ + --expires-at '2026-11-01T00:00:00Z' \ + --state-dir /tmp/k3s-p1-receipts > /tmp/k3s-p1-review.yaml + +The output is YAML for review, **not an automatic kubectl command**. Do not +use a credential or private text as a lease key. Directory mode must deny +group and other users. If a receipt exists with altered args, the command +fails instead of overwriting it. + +## Future acceptance — NOT RUN + +1. Complete P0 live acceptance first. Confirm the pinned worker image + actually supports the latest T3 linked-environment beta, provider/runtime + requirements, and a persistent environment ID. +2. Inspect the generated eight-object YAML and chosen *disposable* K3s + context. Verify the computed worker namespace is unused. Any dry run, + image build or cluster apply requires separate owner authorization. +3. Develop an opt-in reconcile/create/retry path that records Kubernetes + object UIDs, a stable worker descriptor, lease identity, expiry and explicit + retention/cancel/resume states; a lost HTTP response must never trigger + duplicate accepted completion. +4. Independently approve OAuth peer linking on both environments, then run + one real remote delegated task against a pinned repository revision. + Client-saved environment and T3 peer OAuth are not interchangeable. + Test restart, route loss, token revocation, cancellation and negative ACLs. +5. Never enroll temporary workers automatically in client load balancing or + assume worker and owner share a filesystem, provider secrets or GPUs. + +The original P0 image is pinned to T3 release v0.1.3; latest-main beta +compatibility is **unverified**. This preview does not install tools, +clone repositories, contact Kubernetes, link agents, grant credentials, +launch workers or alter the current deployment. + +Canonical decisions: [fork #45](https://github.com/kvnloo/t3code/issues/45). +No upstream writes and no READY status. From 97e42b7ba439e2277aa5d68993e2d0eb372966f1 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:17:12 -0500 Subject: [PATCH 04/84] ci(k3s-p1): pin offline lease-plan regression suite --- .github/workflows/k3s-p1-plan.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/k3s-p1-plan.yml diff --git a/.github/workflows/k3s-p1-plan.yml b/.github/workflows/k3s-p1-plan.yml new file mode 100644 index 0000000000000..ae068adff0e3c --- /dev/null +++ b/.github/workflows/k3s-p1-plan.yml @@ -0,0 +1,24 @@ +name: K3s P1 plan contract +on: + pull_request: + paths: + - "examples/k3s-p1/**" + - ".github/workflows/k3s-p1-plan.yml" + push: + branches-ignore: [main] + paths: + - "examples/k3s-p1/**" + - ".github/workflows/k3s-p1-plan.yml" + workflow_dispatch: +jobs: + offline: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install "PyYAML==6.0.2" + - run: python -m unittest discover -s examples/k3s-p1 -p 'test_*.py' -v From d6a66333f80268ed05432e38a7e6a8795ac109b5 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:17:37 -0500 Subject: [PATCH 05/84] test(k3s-p1): add read-only namespace and PVC identity binding --- examples/k3s-p1/observe.py | 110 +++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 examples/k3s-p1/observe.py diff --git a/examples/k3s-p1/observe.py b/examples/k3s-p1/observe.py new file mode 100644 index 0000000000000..81c8f3b4f6774 --- /dev/null +++ b/examples/k3s-p1/observe.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""P1 offline binding check: observed Namespace and PVC UIDs, no K8s calls. + +A matching Kubernetes snapshot is NOT worker readiness, T3 identity, peer +consent, a task completion, or a license to apply/delete any resources. +""" + +import argparse +import json +from pathlib import Path +import re + + +class ObservationError(ValueError): + """Observation cannot be bound safely to the immutable lease plan.""" + + +def _require(value, expected, label): + if value != expected: + raise ObservationError(f'{label} does not match lease plan') + + +def _uid(obj, label): + value = obj.get('metadata', {}).get('uid') + if not isinstance(value, str) or not re.fullmatch(r'[a-zA-Z0-9-]{8,128}', value): + raise ObservationError(f'{label} lacks a valid Kubernetes UID') + return value + + +def inspect(plan, namespace, pvc): + """Verify immutable lease identity and bound storage from API JSON objects. + + Inputs must be collected from one explicitly approved Kubernetes context. + This does not contact or attest that context; it also cannot prove PVC + contents, live Pod ownership, TLS, peer OAuth, or CNI enforcement. + """ + if not all(isinstance(x, dict) for x in (plan, namespace, pvc)): + raise ObservationError('expected JSON objects') + lease_hash = plan.get('leaseHash') + if not isinstance(lease_hash, str) or not re.fullmatch('[0-9a-f]{64}', lease_hash): + raise ObservationError('invalid lease hash') + name = 't3-p1-' + lease_hash[:24] + _require(plan.get('format'), 't3-k3s-p1-plan-v1', 'plan version') + _require(plan.get('namespace'), name, 'planned namespace') + _require(plan.get('retention'), 'retain', 'planned retention') + expiry = plan.get('expiresAt') + if not isinstance(expiry, str) or not re.fullmatch(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z', expiry): + raise ObservationError('invalid lease expiry') + + _require(namespace.get('kind'), 'Namespace', 'namespace kind') + nmeta = namespace.get('metadata', {}) + _require(nmeta.get('name'), name, 'namespace name') + _require(namespace.get('status', {}).get('phase'), 'Active', 'namespace phase') + if nmeta.get('deletionTimestamp'): + raise ObservationError('namespace is deleting') + labels = nmeta.get('labels', {}) + _require(labels.get('pod-security.kubernetes.io/enforce'), 'restricted', 'Pod Security') + annotations = nmeta.get('annotations', {}) + _require(annotations.get('t3code.dev/lease-key-sha256'), lease_hash, 'namespace lease') + _require(annotations.get('t3code.dev/lease-expires-at'), expiry, 'namespace expiry') + _require(annotations.get('t3code.dev/expiry-enforced'), 'false', 'TTL disclaimer') + _require(annotations.get('t3code.dev/retention'), 'retain-until-owner-decision', 'PVC retention') + + _require(pvc.get('kind'), 'PersistentVolumeClaim', 'PVC kind') + pmeta = pvc.get('metadata', {}) + _require(pmeta.get('name'), 't3-worker-data', 'PVC name') + _require(pmeta.get('namespace'), name, 'PVC namespace') + if pmeta.get('deletionTimestamp'): + raise ObservationError('PVC is deleting') + _require(pvc.get('status', {}).get('phase'), 'Bound', 'PVC phase') + pspec = pvc.get('spec', {}) + _require(pspec.get('accessModes'), ['ReadWriteOnce'], 'PVC access mode') + _require(pspec.get('resources', {}).get('requests', {}).get('storage'), '10Gi', 'PVC request') + volume = pspec.get('volumeName') + if not isinstance(volume, str) or not volume or len(volume) > 253: + raise ObservationError('PVC has no bound PersistentVolume name') + + return {'format': 't3-k3s-p1-binding-v1', 'leaseHash': lease_hash, + 'namespace': name, 'namespaceUID': _uid(namespace, 'namespace'), + 'pvcUID': _uid(pvc, 'PVC'), 'volumeName': volume, + 'expiresAt': expiry, 'retention': 'retain'} + + +def compare(current, baseline): + if not isinstance(baseline, dict) or current != baseline: + raise ObservationError('binding changed: refuse to resume or accept a worker') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--plan', type=Path, required=True, help='private immutable lease-plan receipt') + parser.add_argument('--namespace', type=Path, required=True, help='kubectl get namespace -o json') + parser.add_argument('--pvc', type=Path, required=True, help='kubectl get pvc -o json') + parser.add_argument('--baseline', type=Path, help='previously verified private binding JSON') + args = parser.parse_args() + try: + documents = [json.loads(path.read_text(encoding='utf-8')) + for path in (args.plan, args.namespace, args.pvc)] + binding = inspect(*documents) + if args.baseline is not None: + compare(binding, json.loads(args.baseline.read_text(encoding='utf-8'))) + except (OSError, json.JSONDecodeError, ObservationError) as exc: + # Avoid echoing object contents or OS errors with private host paths. + parser.exit(1, f'FAIL: P1 lease/PVC binding could not be verified ({type(exc).__name__})\n') + print(json.dumps(binding, sort_keys=True)) + print('UID BINDING ONLY: NOT worker readiness, HTTPS, OAuth or delegation proof', file=__import__('sys').stderr) + + +if __name__ == '__main__': + main() From 239967d514b262010443f842344b3bcec72425d5 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:17:59 -0500 Subject: [PATCH 06/84] test(k3s-p1): prove UID binding rejects drift and unsafe claims --- examples/k3s-p1/test_observe.py | 120 ++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 examples/k3s-p1/test_observe.py diff --git a/examples/k3s-p1/test_observe.py b/examples/k3s-p1/test_observe.py new file mode 100644 index 0000000000000..b7bcb920eece8 --- /dev/null +++ b/examples/k3s-p1/test_observe.py @@ -0,0 +1,120 @@ +"""Offline P1 lease/storage binding checks; no Kubernetes/network/secret access.""" +import importlib.util +from pathlib import Path +import copy +import json +import subprocess +import sys +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('p1_observe', HERE / 'observe.py') +assert spec and spec.loader +p = importlib.util.module_from_spec(spec) +spec.loader.exec_module(p) + +HASH = 'a' * 64 +NAMESPACE = 't3-p1-' + HASH[:24] +EXPIRY = '2026-11-01T00:00:00Z' +PLAN = {'format': 't3-k3s-p1-plan-v1', 'leaseHash': HASH, 'namespace': NAMESPACE, + 'image': 'ghcr.io/owned/t3-worker@sha256:' + 'b' * 64, + 'expiresAt': EXPIRY, 'retention': 'retain'} +NS = {'kind': 'Namespace', 'metadata': {'name': NAMESPACE, + 'uid': '11111111-aaaa-bbbb-cccc-111111111111', + 'labels': {'pod-security.kubernetes.io/enforce': 'restricted'}, + 'annotations': {'t3code.dev/lease-key-sha256': HASH, + 't3code.dev/lease-expires-at': EXPIRY, + 't3code.dev/expiry-enforced': 'false', + 't3code.dev/retention': 'retain-until-owner-decision'}}, + 'status': {'phase': 'Active'}} +PVC = {'kind': 'PersistentVolumeClaim', 'metadata': { + 'name': 't3-worker-data', 'namespace': NAMESPACE, + 'uid': '22222222-aaaa-bbbb-cccc-222222222222'}, + 'spec': {'accessModes': ['ReadWriteOnce'], + 'resources': {'requests': {'storage': '10Gi'}}, + 'volumeName': 'pvc-22222222-aaaa-bbbb-cccc-222222222222'}, + 'status': {'phase': 'Bound'}} + + +class ObserveTests(unittest.TestCase): + def test_stable_namespace_and_pvc_uid(self): + initial = p.inspect(PLAN, NS, PVC) + self.assertEqual(initial['pvcUID'], PVC['metadata']['uid']) + p.compare(p.inspect(copy.deepcopy(PLAN), copy.deepcopy(NS), copy.deepcopy(PVC)), initial) + self.assertNotIn('image', initial) + self.assertNotIn('credentials', json.dumps(initial)) + + def test_wrong_lease_identity_or_namespace_is_rejected(self): + for item, change in [('ns', {'metadata': {'name': 'other'}}), + ('plan', {'namespace': 'different'}), + ('plan', {'leaseHash': 'x' * 64})]: + with self.subTest(item=item), self.assertRaises(p.ObservationError): + parts = [copy.deepcopy(PLAN), copy.deepcopy(NS), copy.deepcopy(PVC)] + target = {'plan': 0, 'ns': 1}[item] + parts[target].update(change) + p.inspect(*parts) + + def test_missing_uid_and_binding_replacement_fail(self): + for obj in ('ns', 'pvc'): + parts = [copy.deepcopy(PLAN), copy.deepcopy(NS), copy.deepcopy(PVC)] + parts[1 if obj == 'ns' else 2]['metadata']['uid'] = '' + with self.subTest(obj=obj), self.assertRaises(p.ObservationError): + p.inspect(*parts) + baseline = p.inspect(PLAN, NS, PVC) + replaced = copy.deepcopy(PVC) + replaced['metadata']['uid'] = '33333333-aaaa-bbbb-cccc-333333333333' + with self.assertRaisesRegex(p.ObservationError, 'binding changed'): + p.compare(p.inspect(PLAN, NS, replaced), baseline) + + def test_unbound_deleting_or_unsafe_claim_is_rejected(self): + for path, value in [(('status', 'phase'), 'Pending'), + (('metadata', 'deletionTimestamp'), '2026-10-28T01:00:00Z'), + (('spec', 'accessModes'), ['ReadWriteMany']), + (('spec', 'volumeName'), '')]: + with self.subTest(path=path): + bad = copy.deepcopy(PVC) + bad[path[0]][path[1]] = value + with self.assertRaises(p.ObservationError): + p.inspect(PLAN, NS, bad) + + def test_namespace_annotation_or_policy_drift_fails(self): + for key, value in [('t3code.dev/lease-key-sha256', 'b' * 64), + ('t3code.dev/expiry-enforced', 'true'), + ('t3code.dev/retention', 'delete')]: + with self.subTest(key=key): + bad = copy.deepcopy(NS) + bad['metadata']['annotations'][key] = value + with self.assertRaises(p.ObservationError): + p.inspect(PLAN, bad, PVC) + deleted = copy.deepcopy(NS) + deleted['metadata']['deletionTimestamp'] = '2026-10-28T01:00:00Z' + with self.assertRaises(p.ObservationError): + p.inspect(PLAN, deleted, PVC) + + def test_cli_rejects_invalid_binding_without_emitting_ids(self): + with tempfile.TemporaryDirectory() as directory: + paths = [Path(directory) / name for name in ('plan.json', 'ns.json', 'pvc.json')] + for path, value in zip(paths, (PLAN, NS, PVC)): + path.write_text(json.dumps(value)) + argv = [sys.executable, str(HERE / 'observe.py'), '--plan', str(paths[0]), + '--namespace', str(paths[1]), '--pvc', str(paths[2])] + first = subprocess.run(argv, capture_output=True, text=True, check=False) + self.assertEqual(first.returncode, 0, first.stderr) + baseline = Path(directory) / 'binding.json' + baseline.write_text(first.stdout) + okay = subprocess.run(argv + ['--baseline', str(baseline)], capture_output=True, + text=True, check=False) + self.assertEqual(okay.returncode, 0) + altered = copy.deepcopy(PVC) + altered['metadata']['uid'] = '33333333-aaaa-bbbb-cccc-333333333333' + paths[2].write_text(json.dumps(altered)) + rejected = subprocess.run(argv + ['--baseline', str(baseline)], capture_output=True, + text=True, check=False) + self.assertNotEqual(rejected.returncode, 0) + self.assertEqual(rejected.stdout, '') + self.assertNotIn(altered['metadata']['uid'], rejected.stderr) + + +if __name__ == '__main__': + unittest.main() From 814fa6be178a6be70eaa1233d9c7f495af2fec05 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:18:31 -0500 Subject: [PATCH 07/84] docs(k3s-p1): describe gated UID evidence and non-actuating acceptance --- examples/k3s-p1/OBSERVATION.md | 64 ++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 examples/k3s-p1/OBSERVATION.md diff --git a/examples/k3s-p1/OBSERVATION.md b/examples/k3s-p1/OBSERVATION.md new file mode 100644 index 0000000000000..53d1090b1a8c6 --- /dev/null +++ b/examples/k3s-p1/OBSERVATION.md @@ -0,0 +1,64 @@ +# P1 offline binding observation — UID inventory only + +**READ-ONLY SOURCE CHECK; NO K3s EXECUTION OR WORKER AUTHORIZATION.** The +`observe.py` helper compares one immutable P1 plan with API JSON for its +Namespace and PVC, then extracts their stable Kubernetes UIDs. A previous +binding can be replay-checked so a replaced Namespace or PVC **fails closed** +rather than looking like the same worker lease. + +This is **not** a provisioner, cluster lease, service/Pod readiness probe, +T3 environment identity check, HTTPS proof, peer authorization, task +completion, automatic expiry, or deletion approval. The CLI contacts **no** +cluster. JSON files can be stale or fabricated, so source and context must +be independently verified; never promote a local fixture to live evidence. + +## Reproducible offline tests + +From the fork checkout (no kubeconfig or provider token needed): + +```sh +python3 -m unittest discover -s examples/k3s-p1 -p 'test_*.py' -v +python3 -m py_compile examples/k3s-p1/observe.py +``` + +The six focused binding tests check stable UIDs, lease/namespace mismatch, +PVC replacement, invalid/unbound/deleting storage, retention/annotation drift, +and CLI failure without printing private IDs. + +## Human-gated future Kubernetes observation (NOT RUN) + +**Only after P0 real disposable-cluster acceptance and separate approval to +create one P1 worker**, inspect the rendered plan and explicitly confirm the +selected kubeconfig context and physical machine. The following commands are +read-only; they never deploy, pair, cancel, renew, clean up or grant access. +Keep output in an owner-private directory; do not commit snapshots/receipts. + +```sh +umask 077 +mkdir -p /tmp/t3-p1-private +# Use the namespace printed by plan.py, not a guessed or existing namespace. +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT get namespace YOUR_P1_NAMESPACE \ + -o json > /tmp/t3-p1-private/namespace.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE \ + get pvc t3-worker-data -o json > /tmp/t3-p1-private/pvc.json +python3 examples/k3s-p1/observe.py \ + --plan /tmp/k3s-p1-receipts/YOUR_FULL_LEASE_SHA256.json \ + --namespace /tmp/t3-p1-private/namespace.json \ + --pvc /tmp/t3-p1-private/pvc.json \ + > /tmp/t3-p1-private/binding.json +``` + +For a subsequent observation, **refresh both JSON snapshots from the same +approved context**, then pass +`--baseline /tmp/t3-p1-private/binding.json`. A Namespace/PVC UID or PV +binding change now fails. A missing/unbound PVC or lease/retention mismatch +also fails; nothing is automatically repaired or deleted. Store the baseline +safely and never overwrite it merely to make a failing comparison pass. + +**Still required separately:** verify the server's persisted T3 +`environmentId` and actual live Pod UID/image/Service/EndpointSlice, +private TLS and negative ACL tests, consented MCP OAuth link, exact pinned +repository/delegated task result, cancel/restart/TTL behavior and deliberate +PVC retention. An old or expired plan is not a live reservation. Consult +[work order #45](https://github.com/kvnloo/t3code/issues/45) and +[stacked Draft #47](https://github.com/kvnloo/t3code/pull/47) for gates. From b31c7e2b64f89d9ce711eb222eb002ade39b2b80 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:18:46 -0500 Subject: [PATCH 08/84] docs(k3s-p1): distinguish offline binding from runtime reconciliation --- examples/k3s-p1/README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index d39d622737fc6..dd85a44eccc8e 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -17,6 +17,10 @@ ResourceQuota, no-token ServiceAccount, separate PVC, one Recreate Deployment running a headless T3 server, ClusterIP Service, private Tailscale Ingress and an ingress-only NetworkPolicy. +The separate [offline UID observation helper](OBSERVATION.md) checks a +reported Kubernetes Namespace/PVC binding against this plan and an earlier +binding. It never contacts a cluster or proves a live T3 worker. + The local private mode-0600 receipt is create-once. A retry with identical arguments renders identical YAML. Reusing the key with changed image/expiry fails closed rather than silently rebinding a purported lease. The receipt @@ -58,10 +62,11 @@ fails instead of overwriting it. 2. Inspect the generated eight-object YAML and chosen *disposable* K3s context. Verify the computed worker namespace is unused. Any dry run, image build or cluster apply requires separate owner authorization. -3. Develop an opt-in reconcile/create/retry path that records Kubernetes - object UIDs, a stable worker descriptor, lease identity, expiry and explicit - retention/cancel/resume states; a lost HTTP response must never trigger - duplicate accepted completion. +3. The offline UID helper checks supplied Namespace/PVC snapshots, but does + not persist a runtime lease or authorize reuse. Develop an opt-in + reconcile/create/retry path recording verified live Pod/Service ownership, + stable T3 descriptor, lease identity, expiry and explicit retention/cancel/ + resume states; a lost response must never duplicate accepted completion. 4. Independently approve OAuth peer linking on both environments, then run one real remote delegated task against a pinned repository revision. Client-saved environment and T3 peer OAuth are not interchangeable. From 3afdc302e56092b4210179f0f2f0fef1c6f123fe Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:14:51 -0500 Subject: [PATCH 09/84] fix(k3s-p1): bind offline lease evidence to immutable image reference --- examples/k3s-p1/observe.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/examples/k3s-p1/observe.py b/examples/k3s-p1/observe.py index 81c8f3b4f6774..2628318dba12c 100644 --- a/examples/k3s-p1/observe.py +++ b/examples/k3s-p1/observe.py @@ -6,6 +6,7 @@ """ import argparse +import hashlib import json from pathlib import Path import re @@ -46,6 +47,14 @@ def inspect(plan, namespace, pvc): expiry = plan.get('expiresAt') if not isinstance(expiry, str) or not re.fullmatch(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z', expiry): raise ObservationError('invalid lease expiry') + # A stable Namespace/PVC is not enough: the same key must not silently + # rebind to a different image. Hash the *whole* pinned reference so the + # private binding receipt never exposes the registry or image name. + image = plan.get('image') + if (not isinstance(image, str) or not re.fullmatch( + r'(?=.{1,255}$)[a-z0-9][a-z0-9._:/-]*/[a-z0-9][a-z0-9._/-]*@sha256:[a-f0-9]{64}', image) + or '..' in image or '//' in image): + raise ObservationError('invalid immutable image reference') _require(namespace.get('kind'), 'Namespace', 'namespace kind') nmeta = namespace.get('metadata', {}) @@ -75,9 +84,10 @@ def inspect(plan, namespace, pvc): if not isinstance(volume, str) or not volume or len(volume) > 253: raise ObservationError('PVC has no bound PersistentVolume name') - return {'format': 't3-k3s-p1-binding-v1', 'leaseHash': lease_hash, + return {'format': 't3-k3s-p1-binding-v2', 'leaseHash': lease_hash, 'namespace': name, 'namespaceUID': _uid(namespace, 'namespace'), 'pvcUID': _uid(pvc, 'PVC'), 'volumeName': volume, + 'imageRefSha256': hashlib.sha256(image.encode('ascii')).hexdigest(), 'expiresAt': expiry, 'retention': 'retain'} From 15789150233a0f2509568b1dcf23c1a171c15b05 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:15:19 -0500 Subject: [PATCH 10/84] test(k3s-p1): reject altered image and legacy UID-only binding --- examples/k3s-p1/test_observe.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/examples/k3s-p1/test_observe.py b/examples/k3s-p1/test_observe.py index b7bcb920eece8..a6de03d12936a 100644 --- a/examples/k3s-p1/test_observe.py +++ b/examples/k3s-p1/test_observe.py @@ -92,6 +92,34 @@ def test_namespace_annotation_or_policy_drift_fails(self): with self.assertRaises(p.ObservationError): p.inspect(PLAN, deleted, PVC) + def test_changed_pinned_image_invalidates_existing_binding(self): + first = p.inspect(PLAN, NS, PVC) + self.assertEqual(first['format'], 't3-k3s-p1-binding-v2') + self.assertEqual(len(first['imageRefSha256']), 64) + self.assertNotIn(PLAN['image'], json.dumps(first)) + changed = copy.deepcopy(PLAN) + changed['image'] = 'ghcr.io/owned/other@sha256:' + 'c' * 64 + later = p.inspect(changed, NS, PVC) + self.assertNotEqual(first['imageRefSha256'], later['imageRefSha256']) + with self.assertRaisesRegex(p.ObservationError, 'binding changed'): + p.compare(later, first) + + def test_invalid_image_and_legacy_binding_are_rejected(self): + for image in (None, 't3:latest', 'ghcr.io/owned/t3-worker:latest', + 'ghcr.io/owned/t3-worker@sha256:' + 'Z' * 64, + 'ghcr.io/owned/../other@sha256:' + 'a' * 64, + 'ghcr.io//owned/t3@sha256:' + 'a' * 64): + with self.subTest(image=image), self.assertRaisesRegex( + p.ObservationError, 'immutable image'): + bad = copy.deepcopy(PLAN) + bad['image'] = image + p.inspect(bad, NS, PVC) + current = p.inspect(PLAN, NS, PVC) + legacy = dict(current, format='t3-k3s-p1-binding-v1') + legacy.pop('imageRefSha256') + with self.assertRaisesRegex(p.ObservationError, 'binding changed'): + p.compare(current, legacy) + def test_cli_rejects_invalid_binding_without_emitting_ids(self): with tempfile.TemporaryDirectory() as directory: paths = [Path(directory) / name for name in ('plan.json', 'ns.json', 'pvc.json')] From 1b7ec82871662e6d29b47ead29f437d470de6921 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:15:31 -0500 Subject: [PATCH 11/84] docs(k3s-p1): require v2 image-bound observation baseline --- examples/k3s-p1/OBSERVATION.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/k3s-p1/OBSERVATION.md b/examples/k3s-p1/OBSERVATION.md index 53d1090b1a8c6..95d1fc445c578 100644 --- a/examples/k3s-p1/OBSERVATION.md +++ b/examples/k3s-p1/OBSERVATION.md @@ -3,8 +3,11 @@ **READ-ONLY SOURCE CHECK; NO K3s EXECUTION OR WORKER AUTHORIZATION.** The `observe.py` helper compares one immutable P1 plan with API JSON for its Namespace and PVC, then extracts their stable Kubernetes UIDs. A previous -binding can be replay-checked so a replaced Namespace or PVC **fails closed** -rather than looking like the same worker lease. +binding can be replay-checked so a replaced Namespace, PVC **or pinned image** +**fails closed** rather than looking like the same worker lease. The v2 +receipt contains a SHA-256 fingerprint of the full immutable OCI image +reference, never the raw registry/image name. Old v1 binding baselines +cannot be reused: recollect a trusted v2 baseline before an approved rollout. This is **not** a provisioner, cluster lease, service/Pod readiness probe, T3 environment identity check, HTTPS proof, peer authorization, task @@ -21,8 +24,9 @@ python3 -m unittest discover -s examples/k3s-p1 -p 'test_*.py' -v python3 -m py_compile examples/k3s-p1/observe.py ``` -The six focused binding tests check stable UIDs, lease/namespace mismatch, +The eight focused binding tests check stable UIDs, lease/namespace mismatch, PVC replacement, invalid/unbound/deleting storage, retention/annotation drift, +immutable image reference drift/format, rejection of a legacy v1 baseline, and CLI failure without printing private IDs. ## Human-gated future Kubernetes observation (NOT RUN) @@ -49,7 +53,7 @@ python3 examples/k3s-p1/observe.py \ ``` For a subsequent observation, **refresh both JSON snapshots from the same -approved context**, then pass +approved context** and use the **same pinned image receipt**, then pass `--baseline /tmp/t3-p1-private/binding.json`. A Namespace/PVC UID or PV binding change now fails. A missing/unbound PVC or lease/retention mismatch also fails; nothing is automatically repaired or deleted. Store the baseline From 697fefba964600a06ba9dbba60204b0ea39384a2 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:16:08 -0500 Subject: [PATCH 12/84] docs(k3s-p1): describe image-bound baseline contract --- examples/k3s-p1/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index dd85a44eccc8e..3d8f275f4ca7e 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -18,8 +18,9 @@ running a headless T3 server, ClusterIP Service, private Tailscale Ingress and an ingress-only NetworkPolicy. The separate [offline UID observation helper](OBSERVATION.md) checks a -reported Kubernetes Namespace/PVC binding against this plan and an earlier -binding. It never contacts a cluster or proves a live T3 worker. +reported Kubernetes Namespace/PVC binding **and immutable image-reference +fingerprint** against the plan and an earlier v2 baseline. Legacy UID-only +baselines are rejected. It never contacts a cluster or proves a live T3 worker. The local private mode-0600 receipt is create-once. A retry with identical arguments renders identical YAML. Reusing the key with changed image/expiry From 44cb22fc41d1250833df8c376aad27cd6534c9a7 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:12:09 -0500 Subject: [PATCH 13/84] fix(k3s-p1): bind lease to observed Deployment desired image --- examples/k3s-p1/observe.py | 52 +++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/examples/k3s-p1/observe.py b/examples/k3s-p1/observe.py index 2628318dba12c..41065010c0f55 100644 --- a/examples/k3s-p1/observe.py +++ b/examples/k3s-p1/observe.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""P1 offline binding check: observed Namespace and PVC UIDs, no K8s calls. +"""P1 offline binding check: Namespace, PVC and desired Deployment; no K8s calls. A matching Kubernetes snapshot is NOT worker readiness, T3 identity, peer consent, a task completion, or a license to apply/delete any resources. @@ -28,14 +28,14 @@ def _uid(obj, label): return value -def inspect(plan, namespace, pvc): +def inspect(plan, namespace, pvc, deployment): """Verify immutable lease identity and bound storage from API JSON objects. Inputs must be collected from one explicitly approved Kubernetes context. This does not contact or attest that context; it also cannot prove PVC contents, live Pod ownership, TLS, peer OAuth, or CNI enforcement. """ - if not all(isinstance(x, dict) for x in (plan, namespace, pvc)): + if not all(isinstance(x, dict) for x in (plan, namespace, pvc, deployment)): raise ObservationError('expected JSON objects') lease_hash = plan.get('leaseHash') if not isinstance(lease_hash, str) or not re.fullmatch('[0-9a-f]{64}', lease_hash): @@ -56,6 +56,41 @@ def inspect(plan, namespace, pvc): or '..' in image or '//' in image): raise ObservationError('invalid immutable image reference') + # v3 binds the actual observed Deployment's *desired template* to the + # immutable plan. v2 hashed the plan alone and could miss image drift in + # Kubernetes. This still says nothing about which image a live Pod runs. + _require(deployment.get('kind'), 'Deployment', 'deployment kind') + dmeta = deployment.get('metadata', {}) + _require(dmeta.get('name'), 't3-worker', 'deployment name') + _require(dmeta.get('namespace'), name, 'deployment namespace') + if dmeta.get('deletionTimestamp'): + raise ObservationError('deployment is deleting') + deployment_uid = _uid(deployment, 'deployment') + dspec = deployment.get('spec', {}) + _require(dspec.get('replicas'), 1, 'deployment replicas') + _require(dspec.get('strategy', {}).get('type'), 'Recreate', 'deployment strategy') + expected_labels = {'app': 't3-worker'} + _require(dspec.get('selector', {}).get('matchLabels'), expected_labels, 'deployment selector') + template = dspec.get('template', {}) + _require(template.get('metadata', {}).get('labels'), expected_labels, 'deployment Pod labels') + pod = template.get('spec', {}) + _require(pod.get('serviceAccountName'), 't3-worker', 'deployment ServiceAccount') + _require(pod.get('automountServiceAccountToken'), False, 'deployment API token') + containers = pod.get('containers') + if not isinstance(containers, list) or len(containers) != 1 or not isinstance(containers[0], dict): + raise ObservationError('deployment must have exactly one T3 container') + container = containers[0] + _require(container.get('name'), 't3', 'deployment container') + _require(container.get('image'), image, 'observed deployment image') + _require(container.get('volumeMounts'), [ + {'name': 'data', 'mountPath': '/data'}, + {'name': 'tmp', 'mountPath': '/tmp'}, + ], 'deployment volume mounts') + _require(pod.get('volumes'), [ + {'name': 'data', 'persistentVolumeClaim': {'claimName': 't3-worker-data'}}, + {'name': 'tmp', 'emptyDir': {'sizeLimit': '1Gi'}}, + ], 'deployment volumes') + _require(namespace.get('kind'), 'Namespace', 'namespace kind') nmeta = namespace.get('metadata', {}) _require(nmeta.get('name'), name, 'namespace name') @@ -84,9 +119,10 @@ def inspect(plan, namespace, pvc): if not isinstance(volume, str) or not volume or len(volume) > 253: raise ObservationError('PVC has no bound PersistentVolume name') - return {'format': 't3-k3s-p1-binding-v2', 'leaseHash': lease_hash, + return {'format': 't3-k3s-p1-binding-v3', 'leaseHash': lease_hash, 'namespace': name, 'namespaceUID': _uid(namespace, 'namespace'), - 'pvcUID': _uid(pvc, 'PVC'), 'volumeName': volume, + 'pvcUID': _uid(pvc, 'PVC'), 'deploymentUID': deployment_uid, + 'volumeName': volume, 'imageRefSha256': hashlib.sha256(image.encode('ascii')).hexdigest(), 'expiresAt': expiry, 'retention': 'retain'} @@ -101,11 +137,13 @@ def main(): parser.add_argument('--plan', type=Path, required=True, help='private immutable lease-plan receipt') parser.add_argument('--namespace', type=Path, required=True, help='kubectl get namespace -o json') parser.add_argument('--pvc', type=Path, required=True, help='kubectl get pvc -o json') + parser.add_argument('--deployment', type=Path, required=True, + help='kubectl get deployment t3-worker -o json') parser.add_argument('--baseline', type=Path, help='previously verified private binding JSON') args = parser.parse_args() try: documents = [json.loads(path.read_text(encoding='utf-8')) - for path in (args.plan, args.namespace, args.pvc)] + for path in (args.plan, args.namespace, args.pvc, args.deployment)] binding = inspect(*documents) if args.baseline is not None: compare(binding, json.loads(args.baseline.read_text(encoding='utf-8'))) @@ -113,7 +151,7 @@ def main(): # Avoid echoing object contents or OS errors with private host paths. parser.exit(1, f'FAIL: P1 lease/PVC binding could not be verified ({type(exc).__name__})\n') print(json.dumps(binding, sort_keys=True)) - print('UID BINDING ONLY: NOT worker readiness, HTTPS, OAuth or delegation proof', file=__import__('sys').stderr) + print('DESIRED-SPEC BINDING ONLY: NOT live Pod image, readiness, HTTPS, OAuth or delegation proof', file=__import__('sys').stderr) if __name__ == '__main__': From d2644d23e7d030c0d9104d77769c05f69a7dcd1e Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:12:11 -0500 Subject: [PATCH 14/84] fix(k3s-p1): bind lease to observed Deployment desired image --- examples/k3s-p1/test_observe.py | 94 +++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 16 deletions(-) diff --git a/examples/k3s-p1/test_observe.py b/examples/k3s-p1/test_observe.py index a6de03d12936a..099451206359c 100644 --- a/examples/k3s-p1/test_observe.py +++ b/examples/k3s-p1/test_observe.py @@ -36,12 +36,29 @@ 'volumeName': 'pvc-22222222-aaaa-bbbb-cccc-222222222222'}, 'status': {'phase': 'Bound'}} +DEPLOYMENT = {'kind': 'Deployment', 'metadata': { + 'name': 't3-worker', 'namespace': NAMESPACE, + 'uid': '44444444-aaaa-bbbb-cccc-444444444444'}, + 'spec': {'replicas': 1, 'strategy': {'type': 'Recreate'}, + 'selector': {'matchLabels': {'app': 't3-worker'}}, + 'template': {'metadata': {'labels': {'app': 't3-worker'}}, + 'spec': {'serviceAccountName': 't3-worker', + 'automountServiceAccountToken': False, + 'containers': [{'name': 't3', 'image': PLAN['image'], + 'volumeMounts': [ + {'name': 'data', 'mountPath': '/data'}, + {'name': 'tmp', 'mountPath': '/tmp'}]}], + 'volumes': [ + {'name': 'data', 'persistentVolumeClaim': { + 'claimName': 't3-worker-data'}}, + {'name': 'tmp', 'emptyDir': {'sizeLimit': '1Gi'}}]}}}} + class ObserveTests(unittest.TestCase): def test_stable_namespace_and_pvc_uid(self): - initial = p.inspect(PLAN, NS, PVC) + initial = p.inspect(PLAN, NS, PVC, DEPLOYMENT) self.assertEqual(initial['pvcUID'], PVC['metadata']['uid']) - p.compare(p.inspect(copy.deepcopy(PLAN), copy.deepcopy(NS), copy.deepcopy(PVC)), initial) + p.compare(p.inspect(copy.deepcopy(PLAN), copy.deepcopy(NS), copy.deepcopy(PVC), copy.deepcopy(DEPLOYMENT)), initial) self.assertNotIn('image', initial) self.assertNotIn('credentials', json.dumps(initial)) @@ -50,22 +67,22 @@ def test_wrong_lease_identity_or_namespace_is_rejected(self): ('plan', {'namespace': 'different'}), ('plan', {'leaseHash': 'x' * 64})]: with self.subTest(item=item), self.assertRaises(p.ObservationError): - parts = [copy.deepcopy(PLAN), copy.deepcopy(NS), copy.deepcopy(PVC)] + parts = [copy.deepcopy(PLAN), copy.deepcopy(NS), copy.deepcopy(PVC), copy.deepcopy(DEPLOYMENT)] target = {'plan': 0, 'ns': 1}[item] parts[target].update(change) p.inspect(*parts) def test_missing_uid_and_binding_replacement_fail(self): for obj in ('ns', 'pvc'): - parts = [copy.deepcopy(PLAN), copy.deepcopy(NS), copy.deepcopy(PVC)] + parts = [copy.deepcopy(PLAN), copy.deepcopy(NS), copy.deepcopy(PVC), copy.deepcopy(DEPLOYMENT)] parts[1 if obj == 'ns' else 2]['metadata']['uid'] = '' with self.subTest(obj=obj), self.assertRaises(p.ObservationError): p.inspect(*parts) - baseline = p.inspect(PLAN, NS, PVC) + baseline = p.inspect(PLAN, NS, PVC, DEPLOYMENT) replaced = copy.deepcopy(PVC) replaced['metadata']['uid'] = '33333333-aaaa-bbbb-cccc-333333333333' with self.assertRaisesRegex(p.ObservationError, 'binding changed'): - p.compare(p.inspect(PLAN, NS, replaced), baseline) + p.compare(p.inspect(PLAN, NS, replaced, DEPLOYMENT), baseline) def test_unbound_deleting_or_unsafe_claim_is_rejected(self): for path, value in [(('status', 'phase'), 'Pending'), @@ -76,7 +93,7 @@ def test_unbound_deleting_or_unsafe_claim_is_rejected(self): bad = copy.deepcopy(PVC) bad[path[0]][path[1]] = value with self.assertRaises(p.ObservationError): - p.inspect(PLAN, NS, bad) + p.inspect(PLAN, NS, bad, DEPLOYMENT) def test_namespace_annotation_or_policy_drift_fails(self): for key, value in [('t3code.dev/lease-key-sha256', 'b' * 64), @@ -86,20 +103,22 @@ def test_namespace_annotation_or_policy_drift_fails(self): bad = copy.deepcopy(NS) bad['metadata']['annotations'][key] = value with self.assertRaises(p.ObservationError): - p.inspect(PLAN, bad, PVC) + p.inspect(PLAN, bad, PVC, DEPLOYMENT) deleted = copy.deepcopy(NS) deleted['metadata']['deletionTimestamp'] = '2026-10-28T01:00:00Z' with self.assertRaises(p.ObservationError): p.inspect(PLAN, deleted, PVC) def test_changed_pinned_image_invalidates_existing_binding(self): - first = p.inspect(PLAN, NS, PVC) - self.assertEqual(first['format'], 't3-k3s-p1-binding-v2') + first = p.inspect(PLAN, NS, PVC, DEPLOYMENT) + self.assertEqual(first['format'], 't3-k3s-p1-binding-v3') self.assertEqual(len(first['imageRefSha256']), 64) self.assertNotIn(PLAN['image'], json.dumps(first)) changed = copy.deepcopy(PLAN) changed['image'] = 'ghcr.io/owned/other@sha256:' + 'c' * 64 - later = p.inspect(changed, NS, PVC) + redeployed = copy.deepcopy(DEPLOYMENT) + redeployed['spec']['template']['spec']['containers'][0]['image'] = changed['image'] + later = p.inspect(changed, NS, PVC, redeployed) self.assertNotEqual(first['imageRefSha256'], later['imageRefSha256']) with self.assertRaisesRegex(p.ObservationError, 'binding changed'): p.compare(later, first) @@ -113,20 +132,63 @@ def test_invalid_image_and_legacy_binding_are_rejected(self): p.ObservationError, 'immutable image'): bad = copy.deepcopy(PLAN) bad['image'] = image - p.inspect(bad, NS, PVC) - current = p.inspect(PLAN, NS, PVC) + p.inspect(bad, NS, PVC, DEPLOYMENT) + current = p.inspect(PLAN, NS, PVC, DEPLOYMENT) legacy = dict(current, format='t3-k3s-p1-binding-v1') legacy.pop('imageRefSha256') with self.assertRaisesRegex(p.ObservationError, 'binding changed'): p.compare(current, legacy) + + def test_observed_deployment_image_must_match_plan(self): + first = p.inspect(PLAN, NS, PVC, DEPLOYMENT) + self.assertEqual(first['deploymentUID'], DEPLOYMENT['metadata']['uid']) + for image in ('ghcr.io/other/t3@sha256:' + 'c' * 64, 't3:latest'): + bad = copy.deepcopy(DEPLOYMENT) + bad['spec']['template']['spec']['containers'][0]['image'] = image + with self.subTest(image=image), self.assertRaisesRegex( + p.ObservationError, 'observed deployment image'): + p.inspect(PLAN, NS, PVC, bad) + + def test_deployment_identity_and_pvc_mount_drift_fail_closed(self): + mutations = [ + ('namespace', ('metadata', 'namespace'), 'another'), + ('uid', ('metadata', 'uid'), ''), + ('replicas', ('spec', 'replicas'), 2), + ('strategy', ('spec', 'strategy'), {'type': 'RollingUpdate'}), + ('selector', ('spec', 'selector'), {'matchLabels': {'app': 'other'}}), + ('labels', ('spec', 'template', 'metadata', 'labels'), {'app': 'other'}), + ('sa', ('spec', 'template', 'spec', 'serviceAccountName'), 'default'), + ('token', ('spec', 'template', 'spec', 'automountServiceAccountToken'), True), + ('volume', ('spec', 'template', 'spec', 'volumes'), [ + {'name': 'data', 'hostPath': {'path': '/host'}}, + {'name': 'tmp', 'emptyDir': {'sizeLimit': '1Gi'}}]), + ('mount', ('spec', 'template', 'spec', 'containers'), [ + {'name': 't3', 'image': PLAN['image'], 'volumeMounts': [ + {'name': 'data', 'mountPath': '/home'}, {'name': 'tmp', 'mountPath': '/tmp'}]}]), + ] + for label, path, value in mutations: + bad = copy.deepcopy(DEPLOYMENT) + target = bad + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + with self.subTest(label=label), self.assertRaises(p.ObservationError): + p.inspect(PLAN, NS, PVC, bad) + replaced = copy.deepcopy(DEPLOYMENT) + replaced['metadata']['uid'] = '55555555-aaaa-bbbb-cccc-555555555555' + with self.assertRaisesRegex(p.ObservationError, 'binding changed'): + p.compare(p.inspect(PLAN, NS, PVC, replaced), + p.inspect(PLAN, NS, PVC, DEPLOYMENT)) + def test_cli_rejects_invalid_binding_without_emitting_ids(self): with tempfile.TemporaryDirectory() as directory: - paths = [Path(directory) / name for name in ('plan.json', 'ns.json', 'pvc.json')] - for path, value in zip(paths, (PLAN, NS, PVC)): + paths = [Path(directory) / name for name in ('plan.json', 'ns.json', 'pvc.json', 'deployment.json')] + for path, value in zip(paths, (PLAN, NS, PVC, DEPLOYMENT)): path.write_text(json.dumps(value)) argv = [sys.executable, str(HERE / 'observe.py'), '--plan', str(paths[0]), - '--namespace', str(paths[1]), '--pvc', str(paths[2])] + '--namespace', str(paths[1]), '--pvc', str(paths[2]), + '--deployment', str(paths[3])] first = subprocess.run(argv, capture_output=True, text=True, check=False) self.assertEqual(first.returncode, 0, first.stderr) baseline = Path(directory) / 'binding.json' From 7ffd43bc6a3abbb449202eedda946cc1378df5d0 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:12:14 -0500 Subject: [PATCH 15/84] docs(k3s-p1): require desired Deployment observation v3 --- examples/k3s-p1/OBSERVATION.md | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/examples/k3s-p1/OBSERVATION.md b/examples/k3s-p1/OBSERVATION.md index 95d1fc445c578..533974e0b6db5 100644 --- a/examples/k3s-p1/OBSERVATION.md +++ b/examples/k3s-p1/OBSERVATION.md @@ -1,13 +1,14 @@ -# P1 offline binding observation — UID inventory only +# P1 offline binding observation — UID + desired Deployment contract **READ-ONLY SOURCE CHECK; NO K3s EXECUTION OR WORKER AUTHORIZATION.** The `observe.py` helper compares one immutable P1 plan with API JSON for its -Namespace and PVC, then extracts their stable Kubernetes UIDs. A previous -binding can be replay-checked so a replaced Namespace, PVC **or pinned image** -**fails closed** rather than looking like the same worker lease. The v2 -receipt contains a SHA-256 fingerprint of the full immutable OCI image -reference, never the raw registry/image name. Old v1 binding baselines -cannot be reused: recollect a trusted v2 baseline before an approved rollout. +Namespace, PVC and Deployment, then extracts their stable Kubernetes UIDs. A previous +binding can be replay-checked so a replaced Namespace, PVC, Deployment **or pinned image** +**fails closed** rather than looking like the same worker lease. The v3 receipt contains a SHA-256 fingerprint of the full immutable OCI image +reference, never the raw registry/image name, and checks the observed +Deployment's **desired Pod template** image equals the plan. Old v1/v2 binding +baselines cannot be reused: recollect a trusted v3 baseline from one approved +context. This does **not** check the image actually running in any Pod. This is **not** a provisioner, cluster lease, service/Pod readiness probe, T3 environment identity check, HTTPS proof, peer authorization, task @@ -24,10 +25,11 @@ python3 -m unittest discover -s examples/k3s-p1 -p 'test_*.py' -v python3 -m py_compile examples/k3s-p1/observe.py ``` -The eight focused binding tests check stable UIDs, lease/namespace mismatch, +The ten focused binding tests check stable UIDs, lease/namespace mismatch, PVC replacement, invalid/unbound/deleting storage, retention/annotation drift, immutable image reference drift/format, rejection of a legacy v1 baseline, -and CLI failure without printing private IDs. +desired Deployment image/ServiceAccount/volume/selector drift, and CLI failure +without printing private IDs. ## Human-gated future Kubernetes observation (NOT RUN) @@ -45,22 +47,26 @@ kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT get namespace YOUR_P1_NAMESPA -o json > /tmp/t3-p1-private/namespace.json kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE \ get pvc t3-worker-data -o json > /tmp/t3-p1-private/pvc.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE \ + get deployment t3-worker -o json > /tmp/t3-p1-private/deployment.json python3 examples/k3s-p1/observe.py \ --plan /tmp/k3s-p1-receipts/YOUR_FULL_LEASE_SHA256.json \ --namespace /tmp/t3-p1-private/namespace.json \ --pvc /tmp/t3-p1-private/pvc.json \ + --deployment /tmp/t3-p1-private/deployment.json \ > /tmp/t3-p1-private/binding.json ``` -For a subsequent observation, **refresh both JSON snapshots from the same +For a subsequent observation, **refresh all three Kubernetes JSON snapshots from the same approved context** and use the **same pinned image receipt**, then pass -`--baseline /tmp/t3-p1-private/binding.json`. A Namespace/PVC UID or PV +`--baseline /tmp/t3-p1-private/binding.json`. A Namespace/PVC/Deployment UID, desired image or PV binding change now fails. A missing/unbound PVC or lease/retention mismatch also fails; nothing is automatically repaired or deleted. Store the baseline safely and never overwrite it merely to make a failing comparison pass. **Still required separately:** verify the server's persisted T3 -`environmentId` and actual live Pod UID/image/Service/EndpointSlice, +`environmentId` and actual live Pod UID/image/Service/EndpointSlice (independent from the desired +Deployment spec), private TLS and negative ACL tests, consented MCP OAuth link, exact pinned repository/delegated task result, cancel/restart/TTL behavior and deliberate PVC retention. An old or expired plan is not a live reservation. Consult From 1854c87505910ae67fd492d0894455ee2f6af37f Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:12:17 -0500 Subject: [PATCH 16/84] docs(k3s-p1): require desired Deployment observation v3 --- examples/k3s-p1/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index 3d8f275f4ca7e..89145130ce4a6 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -18,9 +18,10 @@ running a headless T3 server, ClusterIP Service, private Tailscale Ingress and an ingress-only NetworkPolicy. The separate [offline UID observation helper](OBSERVATION.md) checks a -reported Kubernetes Namespace/PVC binding **and immutable image-reference -fingerprint** against the plan and an earlier v2 baseline. Legacy UID-only -baselines are rejected. It never contacts a cluster or proves a live T3 worker. +reported Kubernetes Namespace/PVC/Deployment binding **and immutable +image-reference fingerprint** against the plan and an earlier v3 baseline. +It checks the Deployment's desired template image and PVC/SA mounting, +not the image of a live Pod. Legacy v1/v2 baselines are rejected. It never contacts a cluster or proves a live T3 worker. The local private mode-0600 receipt is create-once. A retry with identical arguments renders identical YAML. Reusing the key with changed image/expiry From 012080ef76a2c2c5537d9696c9ceef156b8f6592 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:21:39 -0500 Subject: [PATCH 17/84] test(k3s-p1): supply deployment fixture to deletion regression --- examples/k3s-p1/test_observe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/k3s-p1/test_observe.py b/examples/k3s-p1/test_observe.py index 099451206359c..91ac775e5193d 100644 --- a/examples/k3s-p1/test_observe.py +++ b/examples/k3s-p1/test_observe.py @@ -107,7 +107,7 @@ def test_namespace_annotation_or_policy_drift_fails(self): deleted = copy.deepcopy(NS) deleted['metadata']['deletionTimestamp'] = '2026-10-28T01:00:00Z' with self.assertRaises(p.ObservationError): - p.inspect(PLAN, deleted, PVC) + p.inspect(PLAN, deleted, PVC, DEPLOYMENT) def test_changed_pinned_image_invalidates_existing_binding(self): first = p.inspect(PLAN, NS, PVC, DEPLOYMENT) From db4356e7b6283dcd218018f3f08d454a6180563a Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:10:30 -0500 Subject: [PATCH 18/84] fix(k3s-p1): reject unsafe desired-worker Pod spec drift --- examples/k3s-p1/observe.py | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/examples/k3s-p1/observe.py b/examples/k3s-p1/observe.py index 41065010c0f55..0a45a6e8aa45c 100644 --- a/examples/k3s-p1/observe.py +++ b/examples/k3s-p1/observe.py @@ -82,6 +82,49 @@ def inspect(plan, namespace, pvc, deployment): container = containers[0] _require(container.get('name'), 't3', 'deployment container') _require(container.get('image'), image, 'observed deployment image') + # Fail closed if the observed desired Pod loosens its isolation while + # retaining the same Deployment UID, image, ServiceAccount and PVC. + # These checks cover spec drift, not admission/CNI enforcement or live Pods. + _require(pod.get('hostname'), 't3-worker', 'deployment hostname') + _require(pod.get('terminationGracePeriodSeconds'), 90, 'deployment termination grace') + _require(pod.get('securityContext'), { + 'runAsNonRoot': True, 'runAsUser': 10001, 'runAsGroup': 10001, + 'fsGroup': 10001, 'seccompProfile': {'type': 'RuntimeDefault'}, + }, 'deployment Pod security') + for field in ('hostNetwork', 'hostPID', 'hostIPC', 'shareProcessNamespace'): + if pod.get(field) not in (None, False): + raise ObservationError('deployment host isolation drift') + for field in ('initContainers', 'ephemeralContainers', 'imagePullSecrets', + 'hostAliases'): + if pod.get(field): + raise ObservationError('deployment unexpected Pod credential or container path') + _require(container.get('workingDir'), '/data', 'deployment working directory') + _require(container.get('imagePullPolicy'), 'IfNotPresent', 'deployment pull policy') + _require(container.get('ports'), [{'name': 'http', 'containerPort': 3773}], + 'deployment HTTP port') + _require(container.get('resources'), { + 'requests': {'cpu': '500m', 'memory': '1Gi', 'ephemeral-storage': '512Mi'}, + 'limits': {'cpu': '2', 'memory': '4Gi', 'ephemeral-storage': '2Gi'}, + }, 'deployment resource bounds') + security = container.get('securityContext') + if not isinstance(security, dict): + raise ObservationError('deployment missing container security context') + for field, value in { + 'allowPrivilegeEscalation': False, 'readOnlyRootFilesystem': True, + 'capabilities': {'drop': ['ALL']}, + }.items(): + _require(security.get(field), value, 'deployment container security') + if security.get('privileged') not in (None, False): + raise ObservationError('deployment privileged container') + for field in ('env', 'envFrom', 'command', 'args', 'lifecycle', 'volumeDevices'): + if container.get(field): + raise ObservationError('deployment unexpected injected container configuration') + _require(container.get('startupProbe'), { + 'tcpSocket': {'port': 'http'}, 'periodSeconds': 5, 'failureThreshold': 60, + }, 'deployment startup probe') + _require(container.get('readinessProbe'), { + 'tcpSocket': {'port': 'http'}, 'periodSeconds': 5, + }, 'deployment readiness probe') _require(container.get('volumeMounts'), [ {'name': 'data', 'mountPath': '/data'}, {'name': 'tmp', 'mountPath': '/tmp'}, From e7943eb96274551313ff4fb83a47e5c70c03cff2 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:10:57 -0500 Subject: [PATCH 19/84] test(k3s-p1): cover Pod privilege, resource and credential drift --- examples/k3s-p1/test_observe.py | 68 +++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/examples/k3s-p1/test_observe.py b/examples/k3s-p1/test_observe.py index 91ac775e5193d..5e875a181ebe3 100644 --- a/examples/k3s-p1/test_observe.py +++ b/examples/k3s-p1/test_observe.py @@ -54,6 +54,31 @@ {'name': 'tmp', 'emptyDir': {'sizeLimit': '1Gi'}}]}}}} +_POD = DEPLOYMENT['spec']['template']['spec'] +_POD.update({ + 'hostname': 't3-worker', 'terminationGracePeriodSeconds': 90, + 'securityContext': { + 'runAsNonRoot': True, 'runAsUser': 10001, 'runAsGroup': 10001, + 'fsGroup': 10001, 'seccompProfile': {'type': 'RuntimeDefault'}, + }, +}) +_CONTAINER = _POD['containers'][0] +_CONTAINER.update({ + 'workingDir': '/data', 'imagePullPolicy': 'IfNotPresent', + 'ports': [{'name': 'http', 'containerPort': 3773}], + 'resources': { + 'requests': {'cpu': '500m', 'memory': '1Gi', 'ephemeral-storage': '512Mi'}, + 'limits': {'cpu': '2', 'memory': '4Gi', 'ephemeral-storage': '2Gi'}, + }, + 'securityContext': {'allowPrivilegeEscalation': False, 'readOnlyRootFilesystem': True, + 'capabilities': {'drop': ['ALL']}}, + 'startupProbe': {'tcpSocket': {'port': 'http'}, 'periodSeconds': 5, + 'failureThreshold': 60}, + 'readinessProbe': {'tcpSocket': {'port': 'http'}, 'periodSeconds': 5}, +}) + + + class ObserveTests(unittest.TestCase): def test_stable_namespace_and_pvc_uid(self): initial = p.inspect(PLAN, NS, PVC, DEPLOYMENT) @@ -181,6 +206,49 @@ def test_deployment_identity_and_pvc_mount_drift_fail_closed(self): p.compare(p.inspect(PLAN, NS, PVC, replaced), p.inspect(PLAN, NS, PVC, DEPLOYMENT)) + def test_privilege_network_and_credential_injection_fail_closed(self): + changes = [ + ('host-network', ('hostNetwork',), True), + ('host-pid', ('hostPID',), True), + ('host-ipc', ('hostIPC',), True), + ('sidecar', ('initContainers',), [{'name': 'other', 'image': PLAN['image']}]), + ('registry-secret', ('imagePullSecrets',), [{'name': 'secret'}]), + ('hostname', ('hostname',), 'other'), + ('pod-security', ('securityContext', 'runAsNonRoot'), False), + ('pod-seccomp', ('securityContext', 'seccompProfile'), {'type': 'Unconfined'}), + ('env-secret', ('containers', 0, 'envFrom'), [{'secretRef': {'name': 'secret'}}]), + ('shell-override', ('containers', 0, 'command'), ['/bin/sh']), + ('privileged', ('containers', 0, 'securityContext', 'privileged'), True), + ('escalation', ('containers', 0, 'securityContext', 'allowPrivilegeEscalation'), True), + ('filesystem', ('containers', 0, 'securityContext', 'readOnlyRootFilesystem'), False), + ('caps', ('containers', 0, 'securityContext', 'capabilities'), {'drop': []}), + ('cpu', ('containers', 0, 'resources', 'limits', 'cpu'), '8'), + ('readiness', ('containers', 0, 'readinessProbe', 'tcpSocket', 'port'), 9999), + ] + for label, path, value in changes: + bad = copy.deepcopy(DEPLOYMENT) + target = bad['spec']['template']['spec'] + for segment in path[:-1]: + target = target[segment] + target[path[-1]] = value + with self.subTest(label=label), self.assertRaises(p.ObservationError): + p.inspect(PLAN, NS, PVC, bad) + + def test_missing_required_security_fields_fail_closed(self): + for path in [ + ('securityContext',), + ('containers', 0, 'securityContext'), + ('containers', 0, 'resources'), + ('containers', 0, 'startupProbe'), + ]: + bad = copy.deepcopy(DEPLOYMENT) + target = bad['spec']['template']['spec'] + for segment in path[:-1]: + target = target[segment] + del target[path[-1]] + with self.subTest(path=path), self.assertRaises(p.ObservationError): + p.inspect(PLAN, NS, PVC, bad) + def test_cli_rejects_invalid_binding_without_emitting_ids(self): with tempfile.TemporaryDirectory() as directory: paths = [Path(directory) / name for name in ('plan.json', 'ns.json', 'pvc.json', 'deployment.json')] From 09354c451ce784223b2d45fa775997d714149aa6 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:11:37 -0500 Subject: [PATCH 20/84] docs(k3s-p1): describe fail-closed Pod template contract --- examples/k3s-p1/OBSERVATION.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/k3s-p1/OBSERVATION.md b/examples/k3s-p1/OBSERVATION.md index 533974e0b6db5..65f2b71007128 100644 --- a/examples/k3s-p1/OBSERVATION.md +++ b/examples/k3s-p1/OBSERVATION.md @@ -6,7 +6,11 @@ Namespace, PVC and Deployment, then extracts their stable Kubernetes UIDs. A pre binding can be replay-checked so a replaced Namespace, PVC, Deployment **or pinned image** **fails closed** rather than looking like the same worker lease. The v3 receipt contains a SHA-256 fingerprint of the full immutable OCI image reference, never the raw registry/image name, and checks the observed -Deployment's **desired Pod template** image equals the plan. Old v1/v2 binding +Deployment's **desired Pod template** image equals the plan. It also requires the +planned non-root Pod/container security, resource bounds, probes and no unexpected +host-network/PID/IPC access, injected init containers, registry credentials, +environment secrets or shell overrides. An unexpected admission mutation fails +closed for human review; Kubernetes may default other fields. Old v1/v2 binding baselines cannot be reused: recollect a trusted v3 baseline from one approved context. This does **not** check the image actually running in any Pod. @@ -25,10 +29,10 @@ python3 -m unittest discover -s examples/k3s-p1 -p 'test_*.py' -v python3 -m py_compile examples/k3s-p1/observe.py ``` -The ten focused binding tests check stable UIDs, lease/namespace mismatch, +The twelve focused binding tests check stable UIDs, lease/namespace mismatch, PVC replacement, invalid/unbound/deleting storage, retention/annotation drift, immutable image reference drift/format, rejection of a legacy v1 baseline, -desired Deployment image/ServiceAccount/volume/selector drift, and CLI failure +desired Deployment image/ServiceAccount/volume/selector and privilege/credential/resource drift, and CLI failure without printing private IDs. ## Human-gated future Kubernetes observation (NOT RUN) From 1ed4d9777ddae3df00cee31d45773c6215173bf7 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:11:40 -0500 Subject: [PATCH 21/84] docs(k3s-p1): qualify desired-template isolation checks --- examples/k3s-p1/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index 89145130ce4a6..256663bc435d8 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -20,7 +20,9 @@ an ingress-only NetworkPolicy. The separate [offline UID observation helper](OBSERVATION.md) checks a reported Kubernetes Namespace/PVC/Deployment binding **and immutable image-reference fingerprint** against the plan and an earlier v3 baseline. -It checks the Deployment's desired template image and PVC/SA mounting, +It checks the Deployment's desired template image, PVC/SA mounting, +non-root Pod/container security, resources and probes, and rejects host access, +additional containers and injected credentials/command overrides, not the image of a live Pod. Legacy v1/v2 baselines are rejected. It never contacts a cluster or proves a live T3 worker. The local private mode-0600 receipt is create-once. A retry with identical From 9629093889b76e1555317a74cdb3a87db4fb67bf Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:12:21 -0500 Subject: [PATCH 22/84] fix(k3s-p1): accept Kubernetes-defaulted TCP and probe fields safely --- examples/k3s-p1/observe.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/examples/k3s-p1/observe.py b/examples/k3s-p1/observe.py index 0a45a6e8aa45c..1475bdbf65cfb 100644 --- a/examples/k3s-p1/observe.py +++ b/examples/k3s-p1/observe.py @@ -100,8 +100,16 @@ def inspect(plan, namespace, pvc, deployment): raise ObservationError('deployment unexpected Pod credential or container path') _require(container.get('workingDir'), '/data', 'deployment working directory') _require(container.get('imagePullPolicy'), 'IfNotPresent', 'deployment pull policy') - _require(container.get('ports'), [{'name': 'http', 'containerPort': 3773}], - 'deployment HTTP port') + # Kubernetes defaults protocol=TCP in the observed object, even when + # omitted from the manifest. Accept that default but never hostPorts. + ports = container.get('ports') + if not isinstance(ports, list) or len(ports) != 1 or not isinstance(ports[0], dict): + raise ObservationError('deployment HTTP port shape changed') + _require(ports[0].get('name'), 'http', 'deployment HTTP port name') + _require(ports[0].get('containerPort'), 3773, 'deployment HTTP port') + _require(ports[0].get('protocol', 'TCP'), 'TCP', 'deployment HTTP protocol') + if ports[0].get('hostPort') not in (None, 0) or ports[0].get('hostIP'): + raise ObservationError('deployment host port is forbidden') _require(container.get('resources'), { 'requests': {'cpu': '500m', 'memory': '1Gi', 'ephemeral-storage': '512Mi'}, 'limits': {'cpu': '2', 'memory': '4Gi', 'ephemeral-storage': '2Gi'}, @@ -119,12 +127,22 @@ def inspect(plan, namespace, pvc, deployment): for field in ('env', 'envFrom', 'command', 'args', 'lifecycle', 'volumeDevices'): if container.get(field): raise ObservationError('deployment unexpected injected container configuration') - _require(container.get('startupProbe'), { - 'tcpSocket': {'port': 'http'}, 'periodSeconds': 5, 'failureThreshold': 60, - }, 'deployment startup probe') - _require(container.get('readinessProbe'), { - 'tcpSocket': {'port': 'http'}, 'periodSeconds': 5, - }, 'deployment readiness probe') + # Kubernetes may add probe defaults to fetched API JSON. + for probe_name, failures in (('startupProbe', 60), ('readinessProbe', 3)): + probe = container.get(probe_name) + if not isinstance(probe, dict): + raise ObservationError('deployment probe missing') + _require(probe.get('tcpSocket'), {'port': 'http'}, 'deployment probe target') + _require(probe.get('periodSeconds'), 5, 'deployment probe frequency') + _require(probe.get('failureThreshold', 3), failures, 'deployment probe failures') + _require(probe.get('timeoutSeconds', 1), 1, 'deployment probe timeout') + _require(probe.get('successThreshold', 1), 1, 'deployment probe successes') + _require(probe.get('initialDelaySeconds', 0), 0, 'deployment probe initial delay') + if any(key not in { + 'tcpSocket', 'periodSeconds', 'failureThreshold', 'timeoutSeconds', + 'successThreshold', 'initialDelaySeconds', + } for key in probe): + raise ObservationError('deployment unexpected probe configuration') _require(container.get('volumeMounts'), [ {'name': 'data', 'mountPath': '/data'}, {'name': 'tmp', 'mountPath': '/tmp'}, From e2062e202d552d4aa2f53b69eb1a71a02d904702 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:12:30 -0500 Subject: [PATCH 23/84] test(k3s-p1): allow safe Kubernetes probe defaults, forbid hostPort --- examples/k3s-p1/test_observe.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/examples/k3s-p1/test_observe.py b/examples/k3s-p1/test_observe.py index 5e875a181ebe3..32982e8d563ba 100644 --- a/examples/k3s-p1/test_observe.py +++ b/examples/k3s-p1/test_observe.py @@ -206,6 +206,21 @@ def test_deployment_identity_and_pvc_mount_drift_fail_closed(self): p.compare(p.inspect(PLAN, NS, PVC, replaced), p.inspect(PLAN, NS, PVC, DEPLOYMENT)) + def test_expected_kubernetes_api_defaults_do_not_false_reject(self): + defaulted = copy.deepcopy(DEPLOYMENT) + container = defaulted['spec']['template']['spec']['containers'][0] + container['ports'][0]['protocol'] = 'TCP' + for probe_name, failure_threshold in (('startupProbe', 60), ('readinessProbe', 3)): + container[probe_name].update({ + 'failureThreshold': failure_threshold, 'timeoutSeconds': 1, + 'successThreshold': 1, 'initialDelaySeconds': 0, + }) + p.compare(p.inspect(PLAN, NS, PVC, defaulted), + p.inspect(PLAN, NS, PVC, DEPLOYMENT)) + container['ports'][0]['hostPort'] = 3773 + with self.assertRaises(p.ObservationError): + p.inspect(PLAN, NS, PVC, defaulted) + def test_privilege_network_and_credential_injection_fail_closed(self): changes = [ ('host-network', ('hostNetwork',), True), From ee1be8989768cc1648e1f0bbe21c2eb1fd515cd2 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:14:07 -0500 Subject: [PATCH 24/84] fix(k3s-p1): reject container-level security overrides --- examples/k3s-p1/observe.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/examples/k3s-p1/observe.py b/examples/k3s-p1/observe.py index 1475bdbf65cfb..52f441a1788e6 100644 --- a/examples/k3s-p1/observe.py +++ b/examples/k3s-p1/observe.py @@ -124,6 +124,19 @@ def inspect(plan, namespace, pvc, deployment): _require(security.get(field), value, 'deployment container security') if security.get('privileged') not in (None, False): raise ObservationError('deployment privileged container') + # Container-level overrides take precedence over Pod securityContext. + # Only allow safe equivalent defaults; unknown keys fail closed. + allowed = {'allowPrivilegeEscalation', 'readOnlyRootFilesystem', + 'capabilities', 'privileged', 'runAsNonRoot', 'runAsUser', + 'runAsGroup', 'seccompProfile'} + if set(security) - allowed: + raise ObservationError('deployment unexpected container security override') + for field, expected in { + 'runAsNonRoot': True, 'runAsUser': 10001, 'runAsGroup': 10001, + 'seccompProfile': {'type': 'RuntimeDefault'}, + }.items(): + if field in security: + _require(security[field], expected, 'deployment container security override') for field in ('env', 'envFrom', 'command', 'args', 'lifecycle', 'volumeDevices'): if container.get(field): raise ObservationError('deployment unexpected injected container configuration') From f4e29de5e8f8b91b31c54175144423e82bb55bac Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:14:14 -0500 Subject: [PATCH 25/84] test(k3s-p1): reject root, seccomp and procMount overrides --- examples/k3s-p1/test_observe.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/k3s-p1/test_observe.py b/examples/k3s-p1/test_observe.py index 32982e8d563ba..4a092c40243b2 100644 --- a/examples/k3s-p1/test_observe.py +++ b/examples/k3s-p1/test_observe.py @@ -234,6 +234,11 @@ def test_privilege_network_and_credential_injection_fail_closed(self): ('env-secret', ('containers', 0, 'envFrom'), [{'secretRef': {'name': 'secret'}}]), ('shell-override', ('containers', 0, 'command'), ['/bin/sh']), ('privileged', ('containers', 0, 'securityContext', 'privileged'), True), + ('container-root', ('containers', 0, 'securityContext', 'runAsUser'), 0), + ('container-seccomp', ('containers', 0, 'securityContext', 'seccompProfile'), + {'type': 'Unconfined'}), + ('container-procmount', ('containers', 0, 'securityContext', 'procMount'), + 'Unmasked'), ('escalation', ('containers', 0, 'securityContext', 'allowPrivilegeEscalation'), True), ('filesystem', ('containers', 0, 'securityContext', 'readOnlyRootFilesystem'), False), ('caps', ('containers', 0, 'securityContext', 'capabilities'), {'drop': []}), From bb74f13fdffc20b58895e9a1b202fbf9b714eee5 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 00:17:14 -0500 Subject: [PATCH 26/84] fix(k3s-p1): fail closed on malformed observation JSON and dates --- examples/k3s-p1/observe.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/examples/k3s-p1/observe.py b/examples/k3s-p1/observe.py index 52f441a1788e6..468ebedc104ba 100644 --- a/examples/k3s-p1/observe.py +++ b/examples/k3s-p1/observe.py @@ -9,6 +9,7 @@ import hashlib import json from pathlib import Path +from datetime import datetime import re @@ -28,7 +29,7 @@ def _uid(obj, label): return value -def inspect(plan, namespace, pvc, deployment): +def _inspect(plan, namespace, pvc, deployment): """Verify immutable lease identity and bound storage from API JSON objects. Inputs must be collected from one explicitly approved Kubernetes context. @@ -47,6 +48,12 @@ def inspect(plan, namespace, pvc, deployment): expiry = plan.get('expiresAt') if not isinstance(expiry, str) or not re.fullmatch(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z', expiry): raise ObservationError('invalid lease expiry') + try: + expiry_date = datetime.strptime(expiry, '%Y-%m-%dT%H:%M:%SZ') + except ValueError as exc: + raise ObservationError('invalid lease expiration calendar date') from exc + if expiry_date.year < 2020 or expiry_date.year > 2100: + raise ObservationError('lease expiration is outside planning bounds') # A stable Namespace/PVC is not enough: the same key must not silently # rebind to a different image. Hash the *whole* pinned reference so the # private binding receipt never exposes the registry or image name. @@ -201,6 +208,16 @@ def inspect(plan, namespace, pvc, deployment): 'expiresAt': expiry, 'retention': 'retain'} +def inspect(plan, namespace, pvc, deployment): + """Treat malformed API JSON as a failed binding, without leaking details.""" + try: + return _inspect(plan, namespace, pvc, deployment) + except (AttributeError, KeyError, TypeError) as exc: + # Fetched Kubernetes objects may contain unexpected null/list values. + # Never let a malformed snapshot escape as an unhandled traceback. + raise ObservationError('malformed JSON object in P1 binding observation') from exc + + def compare(current, baseline): if not isinstance(baseline, dict) or current != baseline: raise ObservationError('binding changed: refuse to resume or accept a worker') From d9a9b5e9e936a3d98b1721a51237bbe8e440e9b2 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 00:17:32 -0500 Subject: [PATCH 27/84] test(k3s-p1): reject malformed API JSON and invalid lease calendar dates --- examples/k3s-p1/test_observe.py | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/examples/k3s-p1/test_observe.py b/examples/k3s-p1/test_observe.py index 4a092c40243b2..330512c3a4003 100644 --- a/examples/k3s-p1/test_observe.py +++ b/examples/k3s-p1/test_observe.py @@ -294,5 +294,50 @@ def test_cli_rejects_invalid_binding_without_emitting_ids(self): self.assertNotIn(altered['metadata']['uid'], rejected.stderr) + def test_invalid_calendar_dates_and_planning_bounds_fail_closed(self): + for expiry in ('2026-02-30T00:00:00Z', '2026-13-01T00:00:00Z', + '2019-12-31T23:59:59Z', '2101-01-01T00:00:00Z'): + bad = copy.deepcopy(PLAN) + bad['expiresAt'] = expiry + with self.subTest(expiry=expiry), self.assertRaises(p.ObservationError): + p.inspect(bad, NS, PVC, DEPLOYMENT) + + def test_malformed_nested_api_json_fails_without_raw_traceback(self): + cases = [ + ('namespace-metadata', 1, ('metadata',), None), + ('namespace-status', 1, ('status',), []), + ('pvc-spec', 2, ('spec',), None), + ('pvc-requests', 2, ('spec', 'resources', 'requests'), []), + ('deployment-spec', 3, ('spec',), None), + ('deployment-selector', 3, ('spec', 'selector'), None), + ('deployment-pod', 3, ('spec', 'template', 'spec'), []), + ] + for label, index, path, value in cases: + parts = [copy.deepcopy(x) for x in (PLAN, NS, PVC, DEPLOYMENT)] + target = parts[index] + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + with self.subTest(label=label), self.assertRaisesRegex( + p.ObservationError, 'malformed JSON object'): + p.inspect(*parts) + + with tempfile.TemporaryDirectory() as directory: + documents = [copy.deepcopy(x) for x in (PLAN, NS, PVC, DEPLOYMENT)] + documents[1]['metadata'] = None + paths = [Path(directory) / name for name in + ('plan.json', 'namespace.json', 'pvc.json', 'deployment.json')] + for path, doc in zip(paths, documents): + path.write_text(json.dumps(doc), encoding='utf-8') + args = [sys.executable, str(HERE / 'observe.py')] + for key, path in zip(('plan', 'namespace', 'pvc', 'deployment'), paths): + args.extend((f'--{key}', str(path))) + result = subprocess.run(args, capture_output=True, text=True, check=False) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, '') + self.assertNotIn('Traceback', result.stderr) + self.assertNotIn(str(paths[1]), result.stderr) + + if __name__ == '__main__': unittest.main() From baa0762d0deb05efaab7791557674fbfc6686d5b Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 00:17:44 -0500 Subject: [PATCH 28/84] docs(k3s-p1): document fail-closed observation input validation --- examples/k3s-p1/OBSERVATION.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/k3s-p1/OBSERVATION.md b/examples/k3s-p1/OBSERVATION.md index 65f2b71007128..8a64f9c070500 100644 --- a/examples/k3s-p1/OBSERVATION.md +++ b/examples/k3s-p1/OBSERVATION.md @@ -29,8 +29,9 @@ python3 -m unittest discover -s examples/k3s-p1 -p 'test_*.py' -v python3 -m py_compile examples/k3s-p1/observe.py ``` -The twelve focused binding tests check stable UIDs, lease/namespace mismatch, -PVC replacement, invalid/unbound/deleting storage, retention/annotation drift, +The focused binding tests check stable UIDs, lease/namespace mismatch, +PVC replacement, invalid/unbound/deleting storage, malformed API JSON and +invalid lease calendar dates (without a raw CLI traceback), retention/annotation drift, immutable image reference drift/format, rejection of a legacy v1 baseline, desired Deployment image/ServiceAccount/volume/selector and privilege/credential/resource drift, and CLI failure without printing private IDs. From 622a11e1600483e97246eace35a1e268e02dd9d9 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:11:45 -0500 Subject: [PATCH 29/84] fix(k3s-p1): atomically publish local lease plan receipt --- examples/k3s-p1/plan.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/examples/k3s-p1/plan.py b/examples/k3s-p1/plan.py index 8af66bd933be3..c68f2b9072de2 100644 --- a/examples/k3s-p1/plan.py +++ b/examples/k3s-p1/plan.py @@ -13,6 +13,7 @@ import os from pathlib import Path import re +import secrets import stat import sys @@ -130,18 +131,35 @@ def lock(state_dir: Path, plan: dict) -> bool: raise PlanError('state directory must be owned by caller with mode 0700') filename = plan['leaseHash'] + '.json' data = (json.dumps(plan, sort_keys=True, separators=(',', ':')) + '\n').encode('utf-8') + # Never publish a receipt until its bytes are fully written and synced. + # O_EXCL on the final name exposes an empty/partial file to concurrent + # retries; a hard link from a private temporary inode publishes atomically. + pending = f'.{filename}.{secrets.token_hex(16)}.pending' file_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_NOFOLLOW', 0) + created = False try: - file = os.open(filename, file_flags, 0o600, dir_fd=directory) - except FileExistsError: - file = None - except OSError as exc: - raise PlanError('cannot establish local lease receipt') from exc - if file is not None: + file = os.open(pending, file_flags, 0o600, dir_fd=directory) with os.fdopen(file, 'wb') as output: output.write(data) output.flush() os.fsync(output.fileno()) + try: + os.link(pending, filename, src_dir_fd=directory, + dst_dir_fd=directory, follow_symlinks=False) + except FileExistsError: + # Another caller published first. Verify its exact bytes below. + pass + else: + os.fsync(directory) + created = True + except OSError as exc: + raise PlanError('cannot establish local lease receipt') from exc + finally: + try: + os.unlink(pending, dir_fd=directory) + except FileNotFoundError: + pass + if created: return True read_flags = os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) try: From b6caa1fb7c9b19847d02047ed0599dc4eed42755 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:11:48 -0500 Subject: [PATCH 30/84] test(k3s-p1): cover concurrent receipt publish and failure --- examples/k3s-p1/test_plan.py | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/examples/k3s-p1/test_plan.py b/examples/k3s-p1/test_plan.py index fd407acfd99b4..0254e06adaa01 100644 --- a/examples/k3s-p1/test_plan.py +++ b/examples/k3s-p1/test_plan.py @@ -5,6 +5,7 @@ import os import tempfile import unittest +from unittest import mock import yaml @@ -92,6 +93,42 @@ def test_local_receipt_is_0600_and_conflicts_fail(self): with self.assertRaisesRegex(p.PlanError, 'unsafe'): p.lock(base, plan) + def test_concurrent_identical_receipt_publish_is_atomic(self): + with tempfile.TemporaryDirectory() as temp: + base = Path(temp) / 'private' + base.mkdir(mode=0o700) + plan, _ = p.build('race:1', IMAGE, EXPIRY) + original_link = os.link + entered = False + + def publish_with_competitor(*args, **kwargs): + nonlocal entered + if not entered: + entered = True + # The first publisher has written its private temp file + # but has not exposed a final receipt. A competing retry + # can publish a whole receipt without seeing partial bytes. + self.assertTrue(p.lock(base, plan)) + return original_link(*args, **kwargs) + + with mock.patch.object(p.os, 'link', side_effect=publish_with_competitor): + self.assertFalse(p.lock(base, plan)) + self.assertTrue(entered) + self.assertEqual(len(list(base.iterdir())), 1) + self.assertEqual(len(list(base.glob('*.json'))), 1) + self.assertFalse(p.lock(base, plan)) + + def test_failed_atomic_publish_leaves_no_conflicting_receipt(self): + with tempfile.TemporaryDirectory() as temp: + base = Path(temp) / 'private' + base.mkdir(mode=0o700) + plan, _ = p.build('write:1', IMAGE, EXPIRY) + with mock.patch.object(p.os, 'link', side_effect=OSError('simulated I/O failure')): + with self.assertRaisesRegex(p.PlanError, 'cannot establish'): + p.lock(base, plan) + self.assertEqual(list(base.iterdir()), []) + self.assertTrue(p.lock(base, plan)) + def test_rejects_symlinked_or_public_plan_directory(self): with tempfile.TemporaryDirectory() as temp: private = Path(temp) / 'private' From 3d28bccb61378e266a49618e310a5171e3dff33d Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:11:52 -0500 Subject: [PATCH 31/84] docs(k3s-p1): clarify atomic local receipt semantics --- examples/k3s-p1/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index 256663bc435d8..1498d155bdcb9 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -25,7 +25,7 @@ non-root Pod/container security, resources and probes, and rejects host access, additional containers and injected credentials/command overrides, not the image of a live Pod. Legacy v1/v2 baselines are rejected. It never contacts a cluster or proves a live T3 worker. -The local private mode-0600 receipt is create-once. A retry with identical +The local private mode-0600 receipt is create-once and atomically published only after a synced write. Concurrent identical requests reuse the complete receipt; a failed publish leaves no half-written final record. A retry with identical arguments renders identical YAML. Reusing the key with changed image/expiry fails closed rather than silently rebinding a purported lease. The receipt is **not** a distributed lock or a cluster reservation. Expiration is advisory From 018738657ea6e367aa99ae0c717dd25d678f638d Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:13:36 -0500 Subject: [PATCH 32/84] feat(k3s-p1): add offline private route drift guard --- examples/k3s-p1/route_check.py | 105 +++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 examples/k3s-p1/route_check.py diff --git a/examples/k3s-p1/route_check.py b/examples/k3s-p1/route_check.py new file mode 100644 index 0000000000000..b639058893274 --- /dev/null +++ b/examples/k3s-p1/route_check.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Offline P1 Service/Ingress/NetworkPolicy route contract; never calls Kubernetes. + +It checks only desired API objects, not the effective CNI, operator, TLS, +DNS, tailnet ACL, peer OAuth grant, or a running T3 environment. +""" +import argparse +import json +from pathlib import Path +import re + + +class RouteError(ValueError): + """A saved Kubernetes route does not match the immutable P1 plan.""" + + +def _same(actual, expected, field): + if actual != expected: + raise RouteError(f'{field} differs from private P1 plan') + + +def _meta(obj, kind, name, namespace): + if not isinstance(obj, dict) or obj.get('kind') != kind: + raise RouteError(f'{kind} object is missing or malformed') + meta = obj.get('metadata') + if not isinstance(meta, dict) or meta.get('name') != name or meta.get('namespace') != namespace: + raise RouteError(f'{kind} identity differs from P1 plan') + if meta.get('deletionTimestamp'): + raise RouteError(f'{kind} is deleting') + return meta + + +def verify(plan, service, ingress, policy): + """Fail closed for route/spec drift; no cluster writes or readiness claims.""" + if not isinstance(plan, dict) or plan.get('format') != 't3-k3s-p1-plan-v1': + raise RouteError('invalid P1 plan format') + lease = plan.get('leaseHash') + if not isinstance(lease, str) or not re.fullmatch(r'[0-9a-f]{64}', lease): + raise RouteError('invalid P1 lease hash') + ns = 't3-p1-' + lease[:24] + _same(plan.get('namespace'), ns, 'namespace') + _meta(service, 'Service', 't3-worker', ns) + _meta(ingress, 'Ingress', 't3-worker', ns) + _meta(policy, 'NetworkPolicy', 'worker-ingress', ns) + + spec = service.get('spec') + if not isinstance(spec, dict): + raise RouteError('invalid Service spec') + _same(spec.get('type', 'ClusterIP'), 'ClusterIP', 'Service type') + _same(spec.get('selector'), {'app': 't3-worker'}, 'Service selector') + if spec.get('externalIPs') or spec.get('externalName') or spec.get('loadBalancerIP'): + raise RouteError('Service must not expose public/external endpoints') + ports = spec.get('ports') + if not isinstance(ports, list) or len(ports) != 1 or not isinstance(ports[0], dict): + raise RouteError('Service ports differ from P1 plan') + port = ports[0] + for key, expected in {'name': 'http', 'port': 80, 'targetPort': 'http', 'protocol': 'TCP'}.items(): + _same(port.get(key, 'TCP' if key == 'protocol' else None), expected, 'Service port') + if port.get('nodePort') not in (None, 0): + raise RouteError('Service NodePort forbidden') + + imeta = ingress['metadata'] + annotations = imeta.get('annotations') or {} + if not isinstance(annotations, dict) or set(annotations) - {'kubectl.kubernetes.io/last-applied-configuration'}: + raise RouteError('unexpected ingress annotations require review') + ispec = ingress.get('spec') + if not isinstance(ispec, dict): + raise RouteError('invalid Ingress spec') + _same(ispec.get('ingressClassName'), 'tailscale', 'private IngressClass') + if ispec.get('rules') or ispec.get('defaultBackend') != { + 'service': {'name': 't3-worker', 'port': {'name': 'http'}} + }: + raise RouteError('Ingress backend or rules differ from private plan') + _same(ispec.get('tls'), [{'hosts': ['t3-p1-' + lease[:24]]}], 'private Ingress hostname') + + pspec = policy.get('spec') + if not isinstance(pspec, dict): + raise RouteError('invalid NetworkPolicy spec') + _same(pspec.get('podSelector'), {'matchLabels': {'app': 't3-worker'}}, 'policy selector') + _same(pspec.get('policyTypes'), ['Ingress'], 'policy types') + _same(pspec.get('ingress'), [{'from': [{'namespaceSelector': {'matchLabels': { + 'kubernetes.io/metadata.name': 'tailscale'}}}], + 'ports': [{'protocol': 'TCP', 'port': 3773}]}], 'private ingress allowlist') + if pspec.get('egress'): + raise RouteError('unexpected policy egress needs separate review') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for flag in ('plan', 'service', 'ingress', 'network-policy'): + parser.add_argument('--' + flag, type=Path, required=True) + args = parser.parse_args() + try: + values = [json.loads(getattr(args, x).read_text(encoding='utf-8')) + for x in ('plan', 'service', 'ingress', 'network_policy')] + verify(*values) + except (OSError, ValueError, TypeError, AttributeError, KeyError) as exc: + # Do not repeat private kubeconfig paths, DNS names, or JSON in errors. + parser.exit(1, f'FAIL: P1 private route specification not verified ({type(exc).__name__})\n') + print('PASS: P1 desired route specification matches plan (offline only)') + print('NOT RUN: DNS, private TLS, Tailscale ACL, NetworkPolicy enforcement, OAuth or live delegation') + + +if __name__ == '__main__': + main() From d4823533090b95b412ec3f485c083c2337433332 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:13:55 -0500 Subject: [PATCH 33/84] test(k3s-p1): reject public route and policy drift offline --- examples/k3s-p1/test_route_check.py | 101 ++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 examples/k3s-p1/test_route_check.py diff --git a/examples/k3s-p1/test_route_check.py b/examples/k3s-p1/test_route_check.py new file mode 100644 index 0000000000000..337e641f4078a --- /dev/null +++ b/examples/k3s-p1/test_route_check.py @@ -0,0 +1,101 @@ +"""Offline malicious-route regression cases; no cluster/network required.""" +import copy +import importlib.util +from pathlib import Path +import json +import subprocess +import sys +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('route_check', HERE / 'route_check.py') +assert spec and spec.loader +rc = importlib.util.module_from_spec(spec) +spec.loader.exec_module(rc) + +H = 'a' * 64 +N = 't3-p1-' + H[:24] +PLAN = {'format': 't3-k3s-p1-plan-v1', 'leaseHash': H, 'namespace': N} +S = {'kind': 'Service', 'metadata': {'name': 't3-worker', 'namespace': N}, + 'spec': {'type': 'ClusterIP', 'selector': {'app': 't3-worker'}, + 'ports': [{'name': 'http', 'port': 80, 'targetPort': 'http'}]}} +I = {'kind': 'Ingress', 'metadata': {'name': 't3-worker', 'namespace': N}, + 'spec': {'ingressClassName': 'tailscale', 'tls': [{'hosts': [N]}], + 'defaultBackend': {'service': {'name': 't3-worker', 'port': {'name': 'http'}}}}} +P = {'kind': 'NetworkPolicy', 'metadata': {'name': 'worker-ingress', 'namespace': N}, + 'spec': {'podSelector': {'matchLabels': {'app': 't3-worker'}}, 'policyTypes': ['Ingress'], + 'ingress': [{'from': [{'namespaceSelector': {'matchLabels': { + 'kubernetes.io/metadata.name': 'tailscale'}}}], + 'ports': [{'protocol': 'TCP', 'port': 3773}]}]}} + + +class RouteTests(unittest.TestCase): + def test_matching_private_route(self): + rc.verify(PLAN, S, I, P) + s = copy.deepcopy(S) + s['spec']['ports'][0]['protocol'] = 'TCP' # normal API default + i = copy.deepcopy(I) + i['metadata']['annotations'] = {'kubectl.kubernetes.io/last-applied-configuration': '...'} + rc.verify(PLAN, s, i, P) + + def test_reject_public_service_and_route_drift(self): + changes = [ + (1, ('spec', 'type'), 'LoadBalancer'), + (1, ('spec', 'type'), 'NodePort'), + (1, ('spec', 'externalIPs'), ['8.8.8.8']), + (1, ('spec', 'ports'), [{'name': 'http', 'port': 80, 'targetPort': 'http', 'nodePort': 30080}]), + (1, ('spec', 'selector'), {'app': 'other'}), + (2, ('spec', 'ingressClassName'), 'nginx'), + (2, ('spec', 'rules'), [{'host': 'public.example.net'}]), + (2, ('spec', 'tls'), [{'hosts': ['public.example.net']}]), + (2, ('metadata', 'annotations'), {'tailscale.com/funnel': 'true'}), + (2, ('spec', 'defaultBackend'), {'service': {'name': 'other', 'port': {'name': 'http'}}}), + (3, ('spec', 'ingress'), [{'from': [{'ipBlock': {'cidr': '0.0.0.0/0'}}]}]), + (3, ('spec', 'policyTypes'), ['Ingress', 'Egress']), + (3, ('spec', 'podSelector'), {}), + ] + for idx, keys, value in changes: + with self.subTest(idx=idx, key=keys): + args = copy.deepcopy([PLAN, S, I, P]) + cursor = args[idx] + for key in keys[:-1]: + cursor = cursor[key] + cursor[keys[-1]] = value + with self.assertRaises(rc.RouteError): + rc.verify(*args) + + def test_reject_invalid_namespace_or_malformed_api_snapshots(self): + for plan in ({**PLAN, 'namespace': 'other'}, {**PLAN, 'leaseHash': 'not-hex'}, + {**PLAN, 'format': 'other'}): + with self.subTest(plan=plan), self.assertRaises(rc.RouteError): + rc.verify(plan, S, I, P) + for idx in (1, 2, 3): + for invalid in (None, [], {'kind': 'other'}, {'kind': 'Service', 'metadata': None}): + args = copy.deepcopy([PLAN, S, I, P]) + args[idx] = invalid + with self.subTest(idx=idx, invalid=invalid), self.assertRaises(rc.RouteError): + rc.verify(*args) + + def test_cli_no_network_and_no_private_details(self): + with tempfile.TemporaryDirectory() as d: + paths = [Path(d) / x for x in ('plan.json', 'service.json', 'ingress.json', 'policy.json')] + for path, obj in zip(paths, (PLAN, S, I, P)): + path.write_text(json.dumps(obj)) + args = [sys.executable, str(HERE / 'route_check.py'), '--plan', str(paths[0]), + '--service', str(paths[1]), '--ingress', str(paths[2]), + '--network-policy', str(paths[3])] + good = subprocess.run(args, capture_output=True, text=True, check=False) + self.assertEqual(good.returncode, 0, good.stderr) + self.assertIn('offline only', good.stdout) + bad_i = copy.deepcopy(I) + bad_i['metadata']['annotations'] = {'tailscale.com/funnel': 'true'} + paths[2].write_text(json.dumps(bad_i)) + bad = subprocess.run(args, capture_output=True, text=True, check=False) + self.assertNotEqual(bad.returncode, 0) + self.assertNotIn(N, bad.stderr) + self.assertNotIn('tailscale.com/funnel', bad.stderr) + + +if __name__ == '__main__': + unittest.main() From 8714042b65546da23006f4049837d2c1be1c5371 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:14:13 -0500 Subject: [PATCH 34/84] docs(k3s-p1): describe read-only route acceptance and physical gates --- examples/k3s-p1/README.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index 1498d155bdcb9..2ae2f72b62d02 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -85,3 +85,38 @@ launch workers or alter the current deployment. Canonical decisions: [fork #45](https://github.com/kvnloo/t3code/issues/45). No upstream writes and no READY status. + + +## Offline private-route drift check (no cluster calls) + +`route_check.py` compares a private P1 plan receipt with saved Kubernetes +**Service, Ingress and NetworkPolicy JSON**. It rejects a public +LoadBalancer/NodePort, external IP, wrong backend, non-Tailscale IngressClass, +Funnel/unreviewed ingress annotations, extra ingress rules, changed tailnet +hostname, or a widened NetworkPolicy. The checked-in +`test_route_check.py` covers those regressions. This is a **static desired +spec check**: a matching snapshot does **not** prove a trusted API response, +correct CNI enforcement, private Tailscale TLS/ACLs, OAuth permission or +a working pod. Do not connect or deploy a worker based on this alone. + +Only **after P0 real disposable-cluster acceptance** and separate owner +approval for a P1 sandbox, save the read-only API snapshots privately: + +```sh +umask 077 +mkdir -p /tmp/t3-p1-private +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get service t3-worker -o json > /tmp/t3-p1-private/service.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get ingress t3-worker -o json > /tmp/t3-p1-private/ingress.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get networkpolicy worker-ingress -o json > /tmp/t3-p1-private/policy.json +python3 examples/k3s-p1/route_check.py \ + --plan /tmp/k3s-p1-receipts/YOUR_FULL_LEASE_SHA256.json \ + --service /tmp/t3-p1-private/service.json \ + --ingress /tmp/t3-p1-private/ingress.json \ + --network-policy /tmp/t3-p1-private/policy.json +``` + +A failed check requires manual review, never automatic rollback or deletion. +Unexpected admission-controller annotations intentionally fail closed. Also +check actual DNS, HTTPS certificate, peer identity, private ACL negative test +and CNI policy enforcement independently; do not treat these API objects as +proof of the data plane. From 9e476b7495d24cf8c52141b4b7652f7f274209d0 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:15:09 -0500 Subject: [PATCH 35/84] fix(k3s-p1): fail closed on additive namespace ingress policies --- examples/k3s-p1/route_check.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/examples/k3s-p1/route_check.py b/examples/k3s-p1/route_check.py index b639058893274..1a8fa72af314d 100644 --- a/examples/k3s-p1/route_check.py +++ b/examples/k3s-p1/route_check.py @@ -30,7 +30,7 @@ def _meta(obj, kind, name, namespace): return meta -def verify(plan, service, ingress, policy): +def verify(plan, service, ingress, policy, all_policies): """Fail closed for route/spec drift; no cluster writes or readiness claims.""" if not isinstance(plan, dict) or plan.get('format') != 't3-k3s-p1-plan-v1': raise RouteError('invalid P1 plan format') @@ -84,20 +84,35 @@ def verify(plan, service, ingress, policy): if pspec.get('egress'): raise RouteError('unexpected policy egress needs separate review') + # Kubernetes ingress policy is additive. A single safe named policy says + # nothing about a second namespace-wide or wildcard allow policy. + if (not isinstance(all_policies, dict) + or all_policies.get('kind') != 'NetworkPolicyList' + or not isinstance(all_policies.get('items'), list) + or len(all_policies['items']) != 1): + raise RouteError('namespace-wide NetworkPolicy inventory is missing or ambiguous') + listed = all_policies['items'][0] + listed_meta = _meta(listed, 'NetworkPolicy', 'worker-ingress', ns) + uid = policy['metadata'].get('uid') + if (not isinstance(uid, str) or not re.fullmatch(r'[a-zA-Z0-9-]{8,128}', uid) + or listed_meta.get('uid') != uid): + raise RouteError('NetworkPolicy identity differs across snapshots') + _same(listed.get('spec'), pspec, 'inventory NetworkPolicy spec') + def main(): parser = argparse.ArgumentParser(description=__doc__) - for flag in ('plan', 'service', 'ingress', 'network-policy'): + for flag in ('plan', 'service', 'ingress', 'network-policy', 'all-network-policies'): parser.add_argument('--' + flag, type=Path, required=True) args = parser.parse_args() try: values = [json.loads(getattr(args, x).read_text(encoding='utf-8')) - for x in ('plan', 'service', 'ingress', 'network_policy')] + for x in ('plan', 'service', 'ingress', 'network_policy', 'all_network_policies')] verify(*values) except (OSError, ValueError, TypeError, AttributeError, KeyError) as exc: # Do not repeat private kubeconfig paths, DNS names, or JSON in errors. parser.exit(1, f'FAIL: P1 private route specification not verified ({type(exc).__name__})\n') - print('PASS: P1 desired route specification matches plan (offline only)') + print('PASS: P1 desired private route and namespace policy inventory match (offline only)') print('NOT RUN: DNS, private TLS, Tailscale ACL, NetworkPolicy enforcement, OAuth or live delegation') From 044412dcd4550d1e72ca017ef88c8e2eff1d2725 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:15:55 -0500 Subject: [PATCH 36/84] test(k3s-p1): reject wildcard additive policies and stale inventory --- examples/k3s-p1/test_route_check.py | 66 +++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/examples/k3s-p1/test_route_check.py b/examples/k3s-p1/test_route_check.py index 337e641f4078a..add521e023eb8 100644 --- a/examples/k3s-p1/test_route_check.py +++ b/examples/k3s-p1/test_route_check.py @@ -23,21 +23,24 @@ I = {'kind': 'Ingress', 'metadata': {'name': 't3-worker', 'namespace': N}, 'spec': {'ingressClassName': 'tailscale', 'tls': [{'hosts': [N]}], 'defaultBackend': {'service': {'name': 't3-worker', 'port': {'name': 'http'}}}}} -P = {'kind': 'NetworkPolicy', 'metadata': {'name': 'worker-ingress', 'namespace': N}, +P = {'kind': 'NetworkPolicy', 'metadata': {'name': 'worker-ingress', 'namespace': N, + 'uid': '11111111-aaaa-bbbb-cccc-111111111111'}, 'spec': {'podSelector': {'matchLabels': {'app': 't3-worker'}}, 'policyTypes': ['Ingress'], 'ingress': [{'from': [{'namespaceSelector': {'matchLabels': { 'kubernetes.io/metadata.name': 'tailscale'}}}], 'ports': [{'protocol': 'TCP', 'port': 3773}]}]}} +ALL = {'kind': 'NetworkPolicyList', 'items': [copy.deepcopy(P)]} + class RouteTests(unittest.TestCase): def test_matching_private_route(self): - rc.verify(PLAN, S, I, P) + rc.verify(PLAN, S, I, P, ALL) s = copy.deepcopy(S) s['spec']['ports'][0]['protocol'] = 'TCP' # normal API default i = copy.deepcopy(I) i['metadata']['annotations'] = {'kubectl.kubernetes.io/last-applied-configuration': '...'} - rc.verify(PLAN, s, i, P) + rc.verify(PLAN, s, i, P, ALL) def test_reject_public_service_and_route_drift(self): changes = [ @@ -57,7 +60,7 @@ def test_reject_public_service_and_route_drift(self): ] for idx, keys, value in changes: with self.subTest(idx=idx, key=keys): - args = copy.deepcopy([PLAN, S, I, P]) + args = copy.deepcopy([PLAN, S, I, P, ALL]) cursor = args[idx] for key in keys[:-1]: cursor = cursor[key] @@ -69,22 +72,58 @@ def test_reject_invalid_namespace_or_malformed_api_snapshots(self): for plan in ({**PLAN, 'namespace': 'other'}, {**PLAN, 'leaseHash': 'not-hex'}, {**PLAN, 'format': 'other'}): with self.subTest(plan=plan), self.assertRaises(rc.RouteError): - rc.verify(plan, S, I, P) + rc.verify(plan, S, I, P, ALL) for idx in (1, 2, 3): for invalid in (None, [], {'kind': 'other'}, {'kind': 'Service', 'metadata': None}): - args = copy.deepcopy([PLAN, S, I, P]) + args = copy.deepcopy([PLAN, S, I, P, ALL]) args[idx] = invalid with self.subTest(idx=idx, invalid=invalid), self.assertRaises(rc.RouteError): rc.verify(*args) + def test_reject_additive_or_inconsistent_policy_inventory(self): + rc.verify(PLAN, S, I, P, ALL) + wildcard = {'kind': 'NetworkPolicy', + 'metadata': {'name': 'allow-anywhere', 'namespace': N, + 'uid': '22222222-aaaa-bbbb-cccc-222222222222'}, + 'spec': {'podSelector': {}, 'policyTypes': ['Ingress'], + 'ingress': [{}]}} + inventory_cases = [ + None, [], {}, + {'kind': 'List', 'items': [copy.deepcopy(P)]}, + {'kind': 'NetworkPolicyList', 'items': []}, + {'kind': 'NetworkPolicyList', 'items': [copy.deepcopy(P), wildcard]}, + ] + for bad in inventory_cases: + with self.subTest(bad=bad), self.assertRaises(rc.RouteError): + rc.verify(PLAN, S, I, P, bad) + for key, value in [ + ('uid', 'different-uid-xxxxxxxxx'), + ('name', 'allow-anywhere'), + ('namespace', 'other-namespace'), + ('deletionTimestamp', '2026-10-29T00:00:00Z'), + ]: + bad = copy.deepcopy(ALL) + bad['items'][0]['metadata'][key] = value + with self.subTest(field=key), self.assertRaises(rc.RouteError): + rc.verify(PLAN, S, I, P, bad) + bad = copy.deepcopy(ALL) + bad['items'][0]['spec']['ingress'] = [{}] + with self.assertRaises(rc.RouteError): + rc.verify(PLAN, S, I, P, bad) + bad = copy.deepcopy(P) + bad['metadata']['uid'] = 'different-uid-xxxxxxxxx' + with self.assertRaises(rc.RouteError): + rc.verify(PLAN, S, I, bad, ALL) + def test_cli_no_network_and_no_private_details(self): with tempfile.TemporaryDirectory() as d: - paths = [Path(d) / x for x in ('plan.json', 'service.json', 'ingress.json', 'policy.json')] - for path, obj in zip(paths, (PLAN, S, I, P)): + paths = [Path(d) / x for x in ('plan.json', 'service.json', 'ingress.json', 'policy.json', 'all-policies.json')] + for path, obj in zip(paths, (PLAN, S, I, P, ALL)): path.write_text(json.dumps(obj)) args = [sys.executable, str(HERE / 'route_check.py'), '--plan', str(paths[0]), '--service', str(paths[1]), '--ingress', str(paths[2]), - '--network-policy', str(paths[3])] + '--network-policy', str(paths[3]), + '--all-network-policies', str(paths[4])] good = subprocess.run(args, capture_output=True, text=True, check=False) self.assertEqual(good.returncode, 0, good.stderr) self.assertIn('offline only', good.stdout) @@ -95,6 +134,15 @@ def test_cli_no_network_and_no_private_details(self): self.assertNotEqual(bad.returncode, 0) self.assertNotIn(N, bad.stderr) self.assertNotIn('tailscale.com/funnel', bad.stderr) + paths[2].write_text(json.dumps(I)) + wide = copy.deepcopy(ALL) + wide['items'].append({ + 'kind': 'NetworkPolicy', 'metadata': {'name': 'allow-all', 'namespace': N}, + 'spec': {'podSelector': {}, 'policyTypes': ['Ingress'], 'ingress': [{}]}}) + paths[4].write_text(json.dumps(wide)) + missing_inventory = subprocess.run(args, capture_output=True, text=True, check=False) + self.assertNotEqual(missing_inventory.returncode, 0) + self.assertNotIn(N, missing_inventory.stderr) if __name__ == '__main__': From 62f619742687b78c0ed6d7656e18929f3d67ebd3 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:16:09 -0500 Subject: [PATCH 37/84] docs(k3s-p1): require namespace-wide read-only NetworkPolicy snapshot --- examples/k3s-p1/README.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index 2ae2f72b62d02..5c47183eada72 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -90,10 +90,13 @@ No upstream writes and no READY status. ## Offline private-route drift check (no cluster calls) `route_check.py` compares a private P1 plan receipt with saved Kubernetes -**Service, Ingress and NetworkPolicy JSON**. It rejects a public +**Service, Ingress, named NetworkPolicy and full namespace-wide NetworkPolicyList JSON**. It rejects a public LoadBalancer/NodePort, external IP, wrong backend, non-Tailscale IngressClass, Funnel/unreviewed ingress annotations, extra ingress rules, changed tailnet -hostname, or a widened NetworkPolicy. The checked-in +hostname, a widened NetworkPolicy, **or an additional wildcard-allow policy**. +Kubernetes ingress policy rules are additive; checking the named policy alone +cannot exclude another policy that exposes the worker. The inventory must +contain exactly that one named policy with the same UID and spec. The checked-in `test_route_check.py` covers those regressions. This is a **static desired spec check**: a matching snapshot does **not** prove a trusted API response, correct CNI enforcement, private Tailscale TLS/ACLs, OAuth permission or @@ -108,15 +111,19 @@ mkdir -p /tmp/t3-p1-private kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get service t3-worker -o json > /tmp/t3-p1-private/service.json kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get ingress t3-worker -o json > /tmp/t3-p1-private/ingress.json kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get networkpolicy worker-ingress -o json > /tmp/t3-p1-private/policy.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get networkpolicies -o json > /tmp/t3-p1-private/all-policies.json python3 examples/k3s-p1/route_check.py \ --plan /tmp/k3s-p1-receipts/YOUR_FULL_LEASE_SHA256.json \ --service /tmp/t3-p1-private/service.json \ --ingress /tmp/t3-p1-private/ingress.json \ - --network-policy /tmp/t3-p1-private/policy.json + --network-policy /tmp/t3-p1-private/policy.json \ + --all-network-policies /tmp/t3-p1-private/all-policies.json ``` A failed check requires manual review, never automatic rollback or deletion. -Unexpected admission-controller annotations intentionally fail closed. Also -check actual DNS, HTTPS certificate, peer identity, private ACL negative test +Unexpected admission-controller annotations and any additional network +policy fail closed. A saved inventory can be stale or collected mid-change; +repeat the approved-context read and independently inspect effective CNI rules. +Also check actual DNS, HTTPS certificate, peer identity, private ACL negative test and CNI policy enforcement independently; do not treat these API objects as proof of the data plane. From a99902c68a967bb3b2cf8e962e5e04b5e1c621de Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:14:58 -0500 Subject: [PATCH 38/84] fix(k3s-p1): reject additive namespace ingress and service exposure --- examples/k3s-p1/route_check.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/examples/k3s-p1/route_check.py b/examples/k3s-p1/route_check.py index 1a8fa72af314d..5b14cf185cbbb 100644 --- a/examples/k3s-p1/route_check.py +++ b/examples/k3s-p1/route_check.py @@ -30,7 +30,7 @@ def _meta(obj, kind, name, namespace): return meta -def verify(plan, service, ingress, policy, all_policies): +def verify(plan, service, ingress, policy, all_policies, all_ingresses, all_services): """Fail closed for route/spec drift; no cluster writes or readiness claims.""" if not isinstance(plan, dict) or plan.get('format') != 't3-k3s-p1-plan-v1': raise RouteError('invalid P1 plan format') @@ -99,20 +99,44 @@ def verify(plan, service, ingress, policy, all_policies): raise RouteError('NetworkPolicy identity differs across snapshots') _same(listed.get('spec'), pspec, 'inventory NetworkPolicy spec') + # A second Ingress or LoadBalancer/NodePort Service in this namespace can + # advertise another route to the same worker. Only inspecting the named + # private Ingress and ClusterIP Service is not a namespace-wide inventory. + # This remains a desired-spec audit, NOT proof that traffic is denied. + for kind, expected, inventory in ( + ('Service', service, all_services), + ('Ingress', ingress, all_ingresses), + ): + if (not isinstance(inventory, dict) + or inventory.get('kind') != kind + 'List' + or not isinstance(inventory.get('items'), list) + or len(inventory['items']) != 1): + raise RouteError(f'{kind} namespace inventory is missing or ambiguous') + listed = inventory['items'][0] + listed_meta = _meta(listed, kind, 't3-worker', ns) + expected_uid = expected['metadata'].get('uid') + if (not isinstance(expected_uid, str) + or not re.fullmatch(r'[a-zA-Z0-9-]{8,128}', expected_uid) + or listed_meta.get('uid') != expected_uid): + raise RouteError(f'{kind} identity differs across snapshots') + _same(listed.get('spec'), expected.get('spec'), f'inventory {kind} spec') + def main(): parser = argparse.ArgumentParser(description=__doc__) - for flag in ('plan', 'service', 'ingress', 'network-policy', 'all-network-policies'): + for flag in ('plan', 'service', 'ingress', 'network-policy', 'all-network-policies', + 'all-ingresses', 'all-services'): parser.add_argument('--' + flag, type=Path, required=True) args = parser.parse_args() try: values = [json.loads(getattr(args, x).read_text(encoding='utf-8')) - for x in ('plan', 'service', 'ingress', 'network_policy', 'all_network_policies')] + for x in ('plan', 'service', 'ingress', 'network_policy', 'all_network_policies', + 'all_ingresses', 'all_services')] verify(*values) except (OSError, ValueError, TypeError, AttributeError, KeyError) as exc: # Do not repeat private kubeconfig paths, DNS names, or JSON in errors. parser.exit(1, f'FAIL: P1 private route specification not verified ({type(exc).__name__})\n') - print('PASS: P1 desired private route and namespace policy inventory match (offline only)') + print('PASS: P1 desired private route and namespace route/policy inventories match (offline only)') print('NOT RUN: DNS, private TLS, Tailscale ACL, NetworkPolicy enforcement, OAuth or live delegation') From aa6ff714eaec2946c81b66137c4bd0b5646ace1b Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:15:27 -0500 Subject: [PATCH 39/84] test(k3s-p1): exercise additive ingress and service inventory bypasses --- examples/k3s-p1/test_route_check.py | 77 +++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 15 deletions(-) diff --git a/examples/k3s-p1/test_route_check.py b/examples/k3s-p1/test_route_check.py index add521e023eb8..d68058339bb85 100644 --- a/examples/k3s-p1/test_route_check.py +++ b/examples/k3s-p1/test_route_check.py @@ -17,10 +17,12 @@ H = 'a' * 64 N = 't3-p1-' + H[:24] PLAN = {'format': 't3-k3s-p1-plan-v1', 'leaseHash': H, 'namespace': N} -S = {'kind': 'Service', 'metadata': {'name': 't3-worker', 'namespace': N}, +S = {'kind': 'Service', 'metadata': {'name': 't3-worker', 'namespace': N, + 'uid': 'aaaaaaaa-1111-2222-3333-aaaaaaaaaaaa'}, 'spec': {'type': 'ClusterIP', 'selector': {'app': 't3-worker'}, 'ports': [{'name': 'http', 'port': 80, 'targetPort': 'http'}]}} -I = {'kind': 'Ingress', 'metadata': {'name': 't3-worker', 'namespace': N}, +I = {'kind': 'Ingress', 'metadata': {'name': 't3-worker', 'namespace': N, + 'uid': 'bbbbbbbb-1111-2222-3333-bbbbbbbbbbbb'}, 'spec': {'ingressClassName': 'tailscale', 'tls': [{'hosts': [N]}], 'defaultBackend': {'service': {'name': 't3-worker', 'port': {'name': 'http'}}}}} P = {'kind': 'NetworkPolicy', 'metadata': {'name': 'worker-ingress', 'namespace': N, @@ -31,16 +33,20 @@ 'ports': [{'protocol': 'TCP', 'port': 3773}]}]}} ALL = {'kind': 'NetworkPolicyList', 'items': [copy.deepcopy(P)]} +ALL_INGRESSES = {'kind': 'IngressList', 'items': [copy.deepcopy(I)]} +ALL_SERVICES = {'kind': 'ServiceList', 'items': [copy.deepcopy(S)]} class RouteTests(unittest.TestCase): def test_matching_private_route(self): - rc.verify(PLAN, S, I, P, ALL) + rc.verify(PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES) s = copy.deepcopy(S) s['spec']['ports'][0]['protocol'] = 'TCP' # normal API default i = copy.deepcopy(I) i['metadata']['annotations'] = {'kubectl.kubernetes.io/last-applied-configuration': '...'} - rc.verify(PLAN, s, i, P, ALL) + rc.verify(PLAN, s, i, P, ALL, + {'kind': 'IngressList', 'items': [copy.deepcopy(i)]}, + {'kind': 'ServiceList', 'items': [copy.deepcopy(s)]}) def test_reject_public_service_and_route_drift(self): changes = [ @@ -60,7 +66,7 @@ def test_reject_public_service_and_route_drift(self): ] for idx, keys, value in changes: with self.subTest(idx=idx, key=keys): - args = copy.deepcopy([PLAN, S, I, P, ALL]) + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) cursor = args[idx] for key in keys[:-1]: cursor = cursor[key] @@ -72,16 +78,16 @@ def test_reject_invalid_namespace_or_malformed_api_snapshots(self): for plan in ({**PLAN, 'namespace': 'other'}, {**PLAN, 'leaseHash': 'not-hex'}, {**PLAN, 'format': 'other'}): with self.subTest(plan=plan), self.assertRaises(rc.RouteError): - rc.verify(plan, S, I, P, ALL) + rc.verify(plan, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES) for idx in (1, 2, 3): for invalid in (None, [], {'kind': 'other'}, {'kind': 'Service', 'metadata': None}): - args = copy.deepcopy([PLAN, S, I, P, ALL]) + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) args[idx] = invalid with self.subTest(idx=idx, invalid=invalid), self.assertRaises(rc.RouteError): rc.verify(*args) def test_reject_additive_or_inconsistent_policy_inventory(self): - rc.verify(PLAN, S, I, P, ALL) + rc.verify(PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES) wildcard = {'kind': 'NetworkPolicy', 'metadata': {'name': 'allow-anywhere', 'namespace': N, 'uid': '22222222-aaaa-bbbb-cccc-222222222222'}, @@ -95,7 +101,7 @@ def test_reject_additive_or_inconsistent_policy_inventory(self): ] for bad in inventory_cases: with self.subTest(bad=bad), self.assertRaises(rc.RouteError): - rc.verify(PLAN, S, I, P, bad) + rc.verify(PLAN, S, I, P, bad, ALL_INGRESSES, ALL_SERVICES) for key, value in [ ('uid', 'different-uid-xxxxxxxxx'), ('name', 'allow-anywhere'), @@ -105,25 +111,66 @@ def test_reject_additive_or_inconsistent_policy_inventory(self): bad = copy.deepcopy(ALL) bad['items'][0]['metadata'][key] = value with self.subTest(field=key), self.assertRaises(rc.RouteError): - rc.verify(PLAN, S, I, P, bad) + rc.verify(PLAN, S, I, P, bad, ALL_INGRESSES, ALL_SERVICES) bad = copy.deepcopy(ALL) bad['items'][0]['spec']['ingress'] = [{}] with self.assertRaises(rc.RouteError): - rc.verify(PLAN, S, I, P, bad) + rc.verify(PLAN, S, I, P, bad, ALL_INGRESSES, ALL_SERVICES) bad = copy.deepcopy(P) bad['metadata']['uid'] = 'different-uid-xxxxxxxxx' with self.assertRaises(rc.RouteError): - rc.verify(PLAN, S, I, bad, ALL) + rc.verify(PLAN, S, I, bad, ALL, ALL_INGRESSES, ALL_SERVICES) + + def test_reject_additional_namespace_public_routes(self): + # Each named resource is still private; extra objects can advertise + # the same worker on another controller or through a public Service. + public_ingress = {'kind': 'Ingress', 'metadata': { + 'name': 'expose-worker', 'namespace': N, 'uid': 'cccccccc-1111-2222-3333-cccccccccccc'}, + 'spec': {'ingressClassName': 'nginx', + 'defaultBackend': {'service': {'name': 't3-worker', 'port': {'name': 'http'}}}}} + public_service = {'kind': 'Service', 'metadata': { + 'name': 'expose-worker', 'namespace': N, 'uid': 'dddddddd-1111-2222-3333-dddddddddddd'}, + 'spec': {'type': 'LoadBalancer', 'selector': {'app': 't3-worker'}, + 'ports': [{'port': 80, 'targetPort': 3773}]}} + for index, extra in [(5, public_ingress), (6, public_service)]: + with self.subTest(index=index): + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + args[index]['items'].append(extra) + with self.assertRaises(rc.RouteError): + rc.verify(*args) + + for index in (5, 6): + for bad in (None, {}, [], {'kind': 'List', 'items': []}, + {'kind': 'IngressList' if index == 5 else 'ServiceList', 'items': []}): + with self.subTest(index=index, bad=bad): + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + args[index] = bad + with self.assertRaises(rc.RouteError): + rc.verify(*args) + for field, value in [ + ('uid', 'different-uid-xxxxxxxxx'), ('namespace', 'wrong'), + ('deletionTimestamp', '2026-10-29T00:00:00Z'), + ]: + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + args[index]['items'][0]['metadata'][field] = value + with self.subTest(index=index, field=field), self.assertRaises(rc.RouteError): + rc.verify(*args) + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + args[index]['items'][0]['spec']['type' if index == 6 else 'ingressClassName'] = 'LoadBalancer' if index == 6 else 'nginx' + with self.subTest(index=index, field='spec'), self.assertRaises(rc.RouteError): + rc.verify(*args) def test_cli_no_network_and_no_private_details(self): with tempfile.TemporaryDirectory() as d: - paths = [Path(d) / x for x in ('plan.json', 'service.json', 'ingress.json', 'policy.json', 'all-policies.json')] - for path, obj in zip(paths, (PLAN, S, I, P, ALL)): + paths = [Path(d) / x for x in ('plan.json', 'service.json', 'ingress.json', 'policy.json', + 'all-policies.json', 'all-ingresses.json', 'all-services.json')] + for path, obj in zip(paths, (PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES)): path.write_text(json.dumps(obj)) args = [sys.executable, str(HERE / 'route_check.py'), '--plan', str(paths[0]), '--service', str(paths[1]), '--ingress', str(paths[2]), '--network-policy', str(paths[3]), - '--all-network-policies', str(paths[4])] + '--all-network-policies', str(paths[4]), + '--all-ingresses', str(paths[5]), '--all-services', str(paths[6])] good = subprocess.run(args, capture_output=True, text=True, check=False) self.assertEqual(good.returncode, 0, good.stderr) self.assertIn('offline only', good.stdout) From 68098e4a91b89e814da9d6eba6d8f71a6e2528bf Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:15:57 -0500 Subject: [PATCH 40/84] docs(k3s-p1): capture complete namespace route inventories --- examples/k3s-p1/README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index 5c47183eada72..4dbf1c3b1d7d6 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -90,13 +90,13 @@ No upstream writes and no READY status. ## Offline private-route drift check (no cluster calls) `route_check.py` compares a private P1 plan receipt with saved Kubernetes -**Service, Ingress, named NetworkPolicy and full namespace-wide NetworkPolicyList JSON**. It rejects a public +**Service, Ingress, named NetworkPolicy and complete namespace-wide ServiceList, IngressList and NetworkPolicyList JSON**. It rejects a public LoadBalancer/NodePort, external IP, wrong backend, non-Tailscale IngressClass, Funnel/unreviewed ingress annotations, extra ingress rules, changed tailnet -hostname, a widened NetworkPolicy, **or an additional wildcard-allow policy**. +hostname, a widened NetworkPolicy, **an additional public Ingress or LoadBalancer/NodePort Service, or an additional wildcard-allow policy**. Kubernetes ingress policy rules are additive; checking the named policy alone cannot exclude another policy that exposes the worker. The inventory must -contain exactly that one named policy with the same UID and spec. The checked-in +contain exactly the named policy with the same UID and spec, and exactly the named private Ingress and ClusterIP Service in their respective inventories. The checked-in `test_route_check.py` covers those regressions. This is a **static desired spec check**: a matching snapshot does **not** prove a trusted API response, correct CNI enforcement, private Tailscale TLS/ACLs, OAuth permission or @@ -112,17 +112,21 @@ kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get serv kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get ingress t3-worker -o json > /tmp/t3-p1-private/ingress.json kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get networkpolicy worker-ingress -o json > /tmp/t3-p1-private/policy.json kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get networkpolicies -o json > /tmp/t3-p1-private/all-policies.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get ingresses -o json > /tmp/t3-p1-private/all-ingresses.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get services -o json > /tmp/t3-p1-private/all-services.json python3 examples/k3s-p1/route_check.py \ --plan /tmp/k3s-p1-receipts/YOUR_FULL_LEASE_SHA256.json \ --service /tmp/t3-p1-private/service.json \ --ingress /tmp/t3-p1-private/ingress.json \ --network-policy /tmp/t3-p1-private/policy.json \ - --all-network-policies /tmp/t3-p1-private/all-policies.json + --all-network-policies /tmp/t3-p1-private/all-policies.json \ + --all-ingresses /tmp/t3-p1-private/all-ingresses.json \ + --all-services /tmp/t3-p1-private/all-services.json ``` A failed check requires manual review, never automatic rollback or deletion. Unexpected admission-controller annotations and any additional network -policy fail closed. A saved inventory can be stale or collected mid-change; +policy fail closed. A saved inventory can be stale or collected mid-change; this does not enumerate routes in other namespaces or non-Kubernetes host networking. repeat the approved-context read and independently inspect effective CNI rules. Also check actual DNS, HTTPS certificate, peer identity, private ACL negative test and CNI policy enforcement independently; do not treat these API objects as From 6bbe77f259c9bcd09afa0c01ba6d75340511849c Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:16:44 -0500 Subject: [PATCH 41/84] docs(k3s-p1): clarify scope and freshness of namespace inventory --- examples/k3s-p1/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index 4dbf1c3b1d7d6..acc252b6c961d 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -126,8 +126,9 @@ python3 examples/k3s-p1/route_check.py \ A failed check requires manual review, never automatic rollback or deletion. Unexpected admission-controller annotations and any additional network -policy fail closed. A saved inventory can be stale or collected mid-change; this does not enumerate routes in other namespaces or non-Kubernetes host networking. -repeat the approved-context read and independently inspect effective CNI rules. +policy fail closed. A saved inventory can be stale or collected mid-change. It does not enumerate +routes in other namespaces or non-Kubernetes host networking. Repeat the +approved-context read and independently inspect effective CNI rules. Also check actual DNS, HTTPS certificate, peer identity, private ACL negative test and CNI policy enforcement independently; do not treat these API objects as proof of the data plane. From daa9a7454924bb73c5c9165def45a39536b65387 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:12:05 -0500 Subject: [PATCH 42/84] fix(k3s-p1): reject API-advertised public routes in offline preview --- examples/k3s-p1/route_check.py | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/examples/k3s-p1/route_check.py b/examples/k3s-p1/route_check.py index 5b14cf185cbbb..d8c3c514f0601 100644 --- a/examples/k3s-p1/route_check.py +++ b/examples/k3s-p1/route_check.py @@ -30,6 +30,35 @@ def _meta(obj, kind, name, namespace): return meta +def _route_status(obj, kind, ns): + """Fail closed on an API-advertised public/extra route, even if spec is private. + + An empty status is valid for an unapplied desired spec, NOT evidence of + HTTPS reachability or a trustworthy tailnet identity. + """ + status = obj.get('status', {}) + if not isinstance(status, dict): + raise RouteError(f'{kind} has malformed route status') + lb = status.get('loadBalancer', {}) + if not isinstance(lb, dict): + raise RouteError(f'{kind} has malformed load-balancer status') + routes = lb.get('ingress', []) + if not isinstance(routes, list): + raise RouteError(f'{kind} advertises malformed routes') + if kind == 'Service': + if routes: + raise RouteError('ClusterIP Service advertises an unexpected external route') + return + if not routes: + return # Unscheduled ingress: desired spec only, not an HTTPS proof. + if len(routes) != 1 or not isinstance(routes[0], dict): + raise RouteError('Ingress advertises unexpected extra routes') + entry = routes[0] + if (set(entry) != {'hostname'} or not isinstance(entry['hostname'], str) + or not re.fullmatch(re.escape(ns) + r'\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.ts\.net', entry['hostname'])): + raise RouteError('Ingress does not advertise the private expected hostname') + + def verify(plan, service, ingress, policy, all_policies, all_ingresses, all_services): """Fail closed for route/spec drift; no cluster writes or readiness claims.""" if not isinstance(plan, dict) or plan.get('format') != 't3-k3s-p1-plan-v1': @@ -43,6 +72,9 @@ def verify(plan, service, ingress, policy, all_policies, all_ingresses, all_serv _meta(ingress, 'Ingress', 't3-worker', ns) _meta(policy, 'NetworkPolicy', 'worker-ingress', ns) + _route_status(service, 'Service', ns) + _route_status(ingress, 'Ingress', ns) + spec = service.get('spec') if not isinstance(spec, dict): raise RouteError('invalid Service spec') @@ -120,6 +152,9 @@ def verify(plan, service, ingress, policy, all_policies, all_ingresses, all_serv or listed_meta.get('uid') != expected_uid): raise RouteError(f'{kind} identity differs across snapshots') _same(listed.get('spec'), expected.get('spec'), f'inventory {kind} spec') + _route_status(listed, kind, ns) + _same(listed.get('status', {}), expected.get('status', {}), + f'inventory {kind} advertised route status') def main(): From 73d6a23568b42b3abbdd2c9344f76ce96f14337a Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:12:22 -0500 Subject: [PATCH 43/84] test(k3s-p1): cover advertised route and inventory drift --- examples/k3s-p1/test_route_check.py | 37 +++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/examples/k3s-p1/test_route_check.py b/examples/k3s-p1/test_route_check.py index d68058339bb85..772689d7302ec 100644 --- a/examples/k3s-p1/test_route_check.py +++ b/examples/k3s-p1/test_route_check.py @@ -160,6 +160,43 @@ def test_reject_additional_namespace_public_routes(self): with self.subTest(index=index, field='spec'), self.assertRaises(rc.RouteError): rc.verify(*args) + def test_private_advertised_ingress_status_and_empty_status(self): + # An unscheduled Ingress has no status; an observed status must be private. + ingress = copy.deepcopy(I) + ingress['status'] = {'loadBalancer': {'ingress': [{ + 'hostname': N + '.tailxxx.ts.net'}]}} + service = copy.deepcopy(S) + service['status'] = {'loadBalancer': {}} + rc.verify(PLAN, service, ingress, P, ALL, + {'kind': 'IngressList', 'items': [copy.deepcopy(ingress)]}, + {'kind': 'ServiceList', 'items': [copy.deepcopy(service)]}) + + def test_reject_public_or_inconsistent_advertised_status(self): + # Public ingress, extra routes and stale mismatched inventory can + # otherwise pass when only desired .spec is validated. + cases = [ + (1, [{'ip': '203.0.113.10'}]), + (2, [{'hostname': 'public.example.com'}]), + (2, [{'ip': '203.0.113.10'}]), + (2, [{'hostname': N + '.tailxxx.ts.net', 'ip': '203.0.113.10'}]), + (2, [{'hostname': N + '.tailxxx.ts.net'}] * 2), + (5, [{'hostname': 'public.example.com'}]), + (6, [{'ip': '203.0.113.10'}]), + ] + for idx, routes in cases: + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + obj = args[idx]['items'][0] if idx in (5, 6) else args[idx] + obj['status'] = {'loadBalancer': {'ingress': routes}} + with self.subTest(idx=idx, routes=routes), self.assertRaises(rc.RouteError): + rc.verify(*args) + for idx in (1, 2, 5, 6): + for invalid in (None, {'loadBalancer': None}, {'loadBalancer': {'ingress': {}}}): + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + obj = args[idx]['items'][0] if idx in (5, 6) else args[idx] + obj['status'] = invalid + with self.subTest(idx=idx, invalid=invalid), self.assertRaises(rc.RouteError): + rc.verify(*args) + def test_cli_no_network_and_no_private_details(self): with tempfile.TemporaryDirectory() as d: paths = [Path(d) / x for x in ('plan.json', 'service.json', 'ingress.json', 'policy.json', From ab4b245a0847040f394e1496eb375cec9b70255c Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:12:36 -0500 Subject: [PATCH 44/84] docs(k3s-p1): qualify advertised route checks and live proof --- examples/k3s-p1/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index acc252b6c961d..c090836867c1c 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -94,6 +94,16 @@ No upstream writes and no READY status. LoadBalancer/NodePort, external IP, wrong backend, non-Tailscale IngressClass, Funnel/unreviewed ingress annotations, extra ingress rules, changed tailnet hostname, a widened NetworkPolicy, **an additional public Ingress or LoadBalancer/NodePort Service, or an additional wildcard-allow policy**. +The helper also inspects **advertised** `Ingress.status.loadBalancer.ingress` +and `Service.status.loadBalancer.ingress` when present, including both +named-object and namespace-inventory snapshots. A public IP, external hostname, +multiple advertised endpoints, inconsistent snapshots or malformed status +fails closed even when the Kubernetes `spec` still looks private. An absent +Ingress status is permitted only for an **unapplied desired-plan check**; it +does **not** mean the route exists. The matching `*.ts.net` name does not +establish tailnet ownership or ACL enforcement. Live HTTPS identity verification +and a separate deny-path probe remain mandatory. + Kubernetes ingress policy rules are additive; checking the named policy alone cannot exclude another policy that exposes the worker. The inventory must contain exactly the named policy with the same UID and spec, and exactly the named private Ingress and ClusterIP Service in their respective inventories. The checked-in From 05436945d0e7204d48647a4c033106974d083672 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:17:04 -0500 Subject: [PATCH 45/84] fix(k3s-p1): reject annotated service exposure paths --- examples/k3s-p1/route_check.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/examples/k3s-p1/route_check.py b/examples/k3s-p1/route_check.py index d8c3c514f0601..fa895e28345d6 100644 --- a/examples/k3s-p1/route_check.py +++ b/examples/k3s-p1/route_check.py @@ -30,6 +30,16 @@ def _meta(obj, kind, name, namespace): return meta +def _safe_annotations(obj, kind): + # Controllers can act on Service annotations even for a ClusterIP. + # In particular, tailscale.com/expose enables an additional L3 proxy + # that does not appear in the ordinary Service/Ingress inventories. + annotations = obj['metadata'].get('annotations', {}) + if (not isinstance(annotations, dict) + or set(annotations) - {'kubectl.kubernetes.io/last-applied-configuration'}): + raise RouteError(f'{kind} has unexpected route-affecting annotations') + + def _route_status(obj, kind, ns): """Fail closed on an API-advertised public/extra route, even if spec is private. @@ -71,6 +81,8 @@ def verify(plan, service, ingress, policy, all_policies, all_ingresses, all_serv _meta(service, 'Service', 't3-worker', ns) _meta(ingress, 'Ingress', 't3-worker', ns) _meta(policy, 'NetworkPolicy', 'worker-ingress', ns) + _safe_annotations(service, 'Service') + _safe_annotations(ingress, 'Ingress') _route_status(service, 'Service', ns) _route_status(ingress, 'Ingress', ns) @@ -91,10 +103,6 @@ def verify(plan, service, ingress, policy, all_policies, all_ingresses, all_serv if port.get('nodePort') not in (None, 0): raise RouteError('Service NodePort forbidden') - imeta = ingress['metadata'] - annotations = imeta.get('annotations') or {} - if not isinstance(annotations, dict) or set(annotations) - {'kubectl.kubernetes.io/last-applied-configuration'}: - raise RouteError('unexpected ingress annotations require review') ispec = ingress.get('spec') if not isinstance(ispec, dict): raise RouteError('invalid Ingress spec') @@ -146,6 +154,7 @@ def verify(plan, service, ingress, policy, all_policies, all_ingresses, all_serv raise RouteError(f'{kind} namespace inventory is missing or ambiguous') listed = inventory['items'][0] listed_meta = _meta(listed, kind, 't3-worker', ns) + _safe_annotations(listed, kind) expected_uid = expected['metadata'].get('uid') if (not isinstance(expected_uid, str) or not re.fullmatch(r'[a-zA-Z0-9-]{8,128}', expected_uid) From 89e52156ac6a8d6533598f3c120f83382e1056b1 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:17:29 -0500 Subject: [PATCH 46/84] test(k3s-p1): cover annotated ClusterIP proxy bypass --- examples/k3s-p1/test_route_check.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/examples/k3s-p1/test_route_check.py b/examples/k3s-p1/test_route_check.py index 772689d7302ec..0ea5454ee1f27 100644 --- a/examples/k3s-p1/test_route_check.py +++ b/examples/k3s-p1/test_route_check.py @@ -197,6 +197,33 @@ def test_reject_public_or_inconsistent_advertised_status(self): with self.subTest(idx=idx, invalid=invalid), self.assertRaises(rc.RouteError): rc.verify(*args) + def test_reject_controller_annotation_exposure_on_named_and_inventory_routes(self): + # Tailscale creates an L3 ingress proxy for annotated ClusterIP + # Services. It need not appear as another Ingress/Service here. + for idx, annotation in ( + (1, {'tailscale.com/expose': 'true'}), + (6, {'tailscale.com/expose': 'true'}), + (1, {'external-dns.kubernetes.io/hostname': 'public.example.org'}), + (6, {'external-dns.kubernetes.io/hostname': 'public.example.org'}), + (2, {'tailscale.com/funnel': 'true'}), + (5, {'tailscale.com/funnel': 'true'}), + (1, None), + (6, []), + ): + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + obj = args[idx]['items'][0] if idx in (5, 6) else args[idx] + obj['metadata']['annotations'] = annotation + with self.subTest(idx=idx, annotation=annotation), self.assertRaises(rc.RouteError): + rc.verify(*args) + + # kubectl apply metadata is not an alternate route. + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + for idx in (1, 2, 5, 6): + obj = args[idx]['items'][0] if idx in (5, 6) else args[idx] + obj['metadata']['annotations'] = { + 'kubectl.kubernetes.io/last-applied-configuration': '{}'} + rc.verify(*args) + def test_cli_no_network_and_no_private_details(self): with tempfile.TemporaryDirectory() as d: paths = [Path(d) / x for x in ('plan.json', 'service.json', 'ingress.json', 'policy.json', From f2508e47d7cd55f678ea9ae6f827b2128afa2273 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:17:39 -0500 Subject: [PATCH 47/84] docs(k3s-p1): distinguish private ingress from annotated Service proxy --- examples/k3s-p1/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index c090836867c1c..b9e8d4e5139c2 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -17,6 +17,12 @@ ResourceQuota, no-token ServiceAccount, separate PVC, one Recreate Deployment running a headless T3 server, ClusterIP Service, private Tailscale Ingress and an ingress-only NetworkPolicy. +The offline route checker also rejects unexpected Service annotations, including +`tailscale.com/expose`, on both the named object and its namespace inventory. +A `ClusterIP` Service with that annotation can create a separate Tailscale L3 +proxy without a second Ingress. A matching snapshot is still **not** network +isolation, TLS, ACL or peer-authorization proof. + The separate [offline UID observation helper](OBSERVATION.md) checks a reported Kubernetes Namespace/PVC/Deployment binding **and immutable image-reference fingerprint** against the plan and an earlier v3 baseline. From 842f81795d3f5f3037d6079567038e21f0eb08a0 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:18:19 -0500 Subject: [PATCH 48/84] chore(k3s-p1): carry forward P0 K3s-only preflight proof --- examples/k3s-p0/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/k3s-p0/README.md b/examples/k3s-p0/README.md index 47edaa600e5ec..e52e2f2b7b4fe 100644 --- a/examples/k3s-p0/README.md +++ b/examples/k3s-p0/README.md @@ -46,7 +46,9 @@ python3 examples/k3s-p0/preflight.py --context YOUR_DISPOSABLE_K3S_CONTEXT ``` The probe uses explicit-context `kubectl get` requests only. It requires -Kubernetes 1.28+, exactly one Ready/un-tainted/schedulable node, an **absent** +a stable K3s build on Kubernetes 1.28+ (`/version.gitVersion` must end in +`+k3sN`; generic Kubernetes distributions are rejected), exactly one +Ready/un-tainted/schedulable node, an **absent** `t3-k3s-p0` namespace (protecting existing state), the `tailscale` namespace and IngressClass controller `tailscale.com/ts-ingress`, and exactly one named default StorageClass. Missing RBAC or API data fails closed; errors never copy From 018752576f5a550da1a59f091525f9301759693b Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:18:23 -0500 Subject: [PATCH 49/84] chore(k3s-p1): carry forward P0 K3s-only preflight proof --- examples/k3s-p0/preflight.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/k3s-p0/preflight.py b/examples/k3s-p0/preflight.py index f362d80927d9f..431e28f33935f 100644 --- a/examples/k3s-p0/preflight.py +++ b/examples/k3s-p0/preflight.py @@ -47,10 +47,13 @@ def read(*args, optional_absent=False): runner=runner) version = read("get", "--raw=/version") - match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)(?:[+\-].*)?", - version.get("gitVersion", "")) + git_version = version.get("gitVersion") + # Other Kubernetes distributions can pass every remaining prerequisite. + # This P0 proof must target a real, stable K3s build, not a generic cluster. + match = (re.fullmatch(r"v(\d+)\.(\d+)\.(\d+)\+k3s[1-9]\d*", git_version) + if isinstance(git_version, str) else None) if not match or tuple(map(int, match.groups()[:2])) < (1, 28): - raise PreflightError("Kubernetes server must be version 1.28 or newer") + raise PreflightError("K3s server must be version 1.28+ with a stable +k3sN build") nodes = read("get", "nodes", "-o", "json").get("items") if not isinstance(nodes, list) or len(nodes) != 1: From cd6bd2c8e9a3cf09a8b9584d49b1450506c194b3 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:18:27 -0500 Subject: [PATCH 50/84] chore(k3s-p1): carry forward P0 K3s-only preflight proof --- examples/k3s-p0/test_preflight.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/examples/k3s-p0/test_preflight.py b/examples/k3s-p0/test_preflight.py index 1efc74e96531d..4f0572dd36f95 100644 --- a/examples/k3s-p0/test_preflight.py +++ b/examples/k3s-p0/test_preflight.py @@ -50,6 +50,16 @@ class P0PreflightTests(unittest.TestCase): def test_passes_disposable_ready_single_node(self): preflight.verify("isolated-k3s", runner=fake_run(RESPONSES)) + def test_rejects_non_k3s_and_malformed_server_versions(self): + # A generic Kubernetes cluster can meet every other read-only check. + # This P0 proof is explicitly K3s-only; never report it as K3s. + for version in ("v1.37.1", "v1.37.1+k0s1", "v1.37.1+k3s", + "v1.27.9+k3s1", None, 137): + mapping = copy.deepcopy(RESPONSES) + mapping[("get", "--raw=/version")] = {"gitVersion": version} + with self.subTest(version=version), self.assertRaises(preflight.PreflightError): + preflight.verify("isolated-k3s", runner=fake_run(mapping)) + def test_rejects_missing_or_invalid_context(self): for context in ("", "-danger", " isolated-k3s", "isolated-k3s\n", "a\x00b"): with self.subTest(context=repr(context)), self.assertRaises(preflight.PreflightError): From 4cd0e6a37331f0468be8009009a55755ec432c3f Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:16:24 -0500 Subject: [PATCH 51/84] test(k3s-p1): inspect single ready Pod ownership without actuation --- examples/k3s-p1/pod_check.py | 149 +++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 examples/k3s-p1/pod_check.py diff --git a/examples/k3s-p1/pod_check.py b/examples/k3s-p1/pod_check.py new file mode 100644 index 0000000000000..32288e6137123 --- /dev/null +++ b/examples/k3s-p1/pod_check.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Read-only P1 Pod ownership/runtime snapshot check; NEVER a live attestation.""" +import argparse +import hashlib +import json +from pathlib import Path +import re + + +class PodError(ValueError): + """An untrusted Kubernetes snapshot does not support the claimed worker.""" + + +def need(condition, message): + if not condition: + raise PodError(message) + + +def meta(obj, kind, namespace): + need(isinstance(obj, dict) and obj.get('kind') == kind, 'invalid object kind') + value = obj.get('metadata') + need(isinstance(value, dict) and value.get('namespace') == namespace + and not value.get('deletionTimestamp'), 'resource identity or deletion drift') + uid = value.get('uid') + need(isinstance(uid, str) and re.fullmatch(r'[A-Za-z0-9-]{8,128}', uid), 'missing resource UID') + return value + + +def controller(obj, kind, name, uid): + refs = obj.get('metadata', {}).get('ownerReferences') + need(isinstance(refs, list) and len(refs) == 1, 'missing exclusive controller reference') + ref = refs[0] + need(isinstance(ref, dict) and ref.get('controller') is True + and (ref.get('kind'), ref.get('name'), ref.get('uid')) == (kind, name, uid), + 'controller reference drift') + + +def inspect(plan, binding, deployment, replicasets, pods): + """Check offline snapshots from one approved context, not actual connectivity.""" + need(isinstance(plan, dict) and isinstance(binding, dict), 'invalid plan/binding') + lease = plan.get('leaseHash') + need(plan.get('format') == 't3-k3s-p1-plan-v1' and isinstance(lease, str) + and re.fullmatch(r'[0-9a-f]{64}', lease), 'invalid immutable plan') + ns = 't3-p1-' + lease[:24] + image = plan.get('image') + need(isinstance(image, str) and re.fullmatch( + r'(?=.{1,255}$)[a-z0-9][a-z0-9._:/-]*/[a-z0-9][a-z0-9._/-]*@sha256:[a-f0-9]{64}', image) + and '..' not in image and '//' not in image, 'image not pinned') + image_hash = hashlib.sha256(image.encode('ascii')).hexdigest() + need(plan.get('namespace') == ns and binding.get('format') == 't3-k3s-p1-binding-v3' + and binding.get('namespace') == ns and binding.get('leaseHash') == lease + and binding.get('imageRefSha256') == image_hash, 'lease binding mismatch') + dmeta = meta(deployment, 'Deployment', ns) + need(dmeta.get('name') == 't3-worker' and dmeta['uid'] == binding.get('deploymentUID'), + 'deployment binding drift') + ds = deployment.get('spec', {}) + template = ds.get('template', {}).get('spec', {}) + need(isinstance(template, dict) and isinstance(template.get('containers'), list) + and len(template['containers']) == 1 and template['containers'][0].get('image') == image, + 'deployment image drift') + status = deployment.get('status', {}) + need(isinstance(status, dict) and isinstance(dmeta.get('generation'), int) + and status.get('observedGeneration') == dmeta['generation'] + and all(status.get(k) == 1 for k in ('replicas', 'readyReplicas', 'availableReplicas', 'updatedReplicas')), + 'deployment rollout not observed ready') + need(isinstance(replicasets, dict) and replicasets.get('kind') == 'ReplicaSetList' + and isinstance(replicasets.get('items'), list), 'incomplete ReplicaSet inventory') + active_rs = [] + for rs in replicasets['items']: + rm = meta(rs, 'ReplicaSet', ns) + controller(rs, 'Deployment', 't3-worker', dmeta['uid']) + rs_spec = rs.get('spec', {}) + rs_status = rs.get('status', {}) + need(isinstance(rs_spec, dict) and isinstance(rs_status, dict), 'malformed ReplicaSet') + if rs_spec.get('replicas', 0) or rs_status.get('replicas', 0): + active_rs.append((rs, rm)) + need(len(active_rs) == 1, 'ambiguous live ReplicaSet') + rs, rm = active_rs[0] + need(rs.get('spec', {}).get('replicas') == 1 + and rs.get('status', {}).get('readyReplicas') == 1, 'ReplicaSet not ready') + need(isinstance(pods, dict) and pods.get('kind') == 'PodList' + and isinstance(pods.get('items'), list) and len(pods['items']) == 1, + 'worker namespace does not have exactly one Pod') + pod = pods['items'][0] + pm = meta(pod, 'Pod', ns) + controller(pod, 'ReplicaSet', rm.get('name'), rm['uid']) + need(pm.get('labels', {}).get('app') == 't3-worker', 'Pod selector drift') + ps = pod.get('spec', {}) + need(isinstance(ps, dict) and ps.get('serviceAccountName') == 't3-worker' + and ps.get('automountServiceAccountToken') is False + and ps.get('securityContext') == template.get('securityContext') + and ps.get('volumes') == template.get('volumes'), 'admitted Pod security/storage drift') + for field in ('hostNetwork', 'hostPID', 'hostIPC', 'shareProcessNamespace'): + need(ps.get(field) in (None, False), 'Pod host isolation drift') + for field in ('initContainers', 'ephemeralContainers', 'imagePullSecrets', 'hostAliases'): + need(not ps.get(field), 'unexpected Pod configuration') + containers = ps.get('containers') + need(isinstance(containers, list) and len(containers) == 1 + and isinstance(containers[0], dict), 'unexpected Pod containers') + for field in ('name', 'image', 'securityContext', 'resources', 'volumeMounts'): + need(containers[0].get(field) == template['containers'][0].get(field), + 'admitted container configuration drift') + for field in ('env', 'envFrom', 'command', 'args', 'volumeDevices'): + need(not containers[0].get(field), 'injected Pod credentials or command') + pod_status = pod.get('status', {}) + need(isinstance(pod_status, dict) and pod_status.get('phase') == 'Running', 'Pod not running') + conditions = pod_status.get('conditions') + need(isinstance(conditions, list) and any(isinstance(c, dict) + and c.get('type') == 'Ready' and c.get('status') == 'True' for c in conditions), + 'Pod not reported ready') + cs = pod_status.get('containerStatuses') + need(isinstance(cs, list) and len(cs) == 1 and isinstance(cs[0], dict), + 'missing container status') + c = cs[0] + need(c.get('name') == 't3' and c.get('ready') is True + and c.get('started') is True and isinstance(c.get('state'), dict) + and isinstance(c['state'].get('running'), dict), 'T3 process not reported ready') + need(c.get('image') == image, 'running image reference differs from lease') + cid, iid = c.get('containerID'), c.get('imageID') + need(isinstance(cid, str) and len(cid) >= 16 + and isinstance(iid, str) and re.search(r'sha256:[a-f0-9]{64}', iid), + 'runtime container or image ID missing') + restarts = c.get('restartCount') + need(type(restarts) is int and restarts >= 0, 'invalid restart count') + return {'format': 't3-k3s-p1-pod-snapshot-v1', 'leaseHash': lease, + 'deploymentUID': dmeta['uid'], 'replicaSetUID': rm['uid'], + 'podUID': pm['uid'], 'imageRefSha256': image_hash, + 'imageIDSha256': hashlib.sha256(iid.encode()).hexdigest(), + 'containerIDSha256': hashlib.sha256(cid.encode()).hexdigest(), + 'restartCount': restarts} + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + for flag in ('plan', 'binding', 'deployment', 'replicasets', 'pods'): + p.add_argument('--' + flag, required=True, type=Path) + a = p.parse_args() + try: + docs = [json.loads(getattr(a, key).read_text(encoding='utf-8')) + for key in ('plan', 'binding', 'deployment', 'replicasets', 'pods')] + receipt = inspect(*docs) + except (OSError, ValueError, TypeError, KeyError, AttributeError, IndexError) as exc: + p.exit(1, f'FAIL: P1 offline Pod snapshot not verified ({type(exc).__name__})\n') + print(json.dumps(receipt, sort_keys=True)) + print('NOT RUN: trusted live cluster, actual digest attestation, T3 identity, HTTPS, OAuth, delegation') + + +if __name__ == '__main__': + main() From 7a59d513bfe22480eaefc0595a25de190f0b7258 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:16:57 -0500 Subject: [PATCH 52/84] test(k3s-p1): cover Pod controller, image, readiness and mutation drift --- examples/k3s-p1/test_pod_check.py | 158 ++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 examples/k3s-p1/test_pod_check.py diff --git a/examples/k3s-p1/test_pod_check.py b/examples/k3s-p1/test_pod_check.py new file mode 100644 index 0000000000000..a7ac88e042fd0 --- /dev/null +++ b/examples/k3s-p1/test_pod_check.py @@ -0,0 +1,158 @@ +"""Read-only adversarial snapshot regressions; no kubeconfig or credentials.""" +import copy +import hashlib +import importlib.util +from pathlib import Path +import json +import subprocess +import sys +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('pod_check', HERE / 'pod_check.py') +pc = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pc) + +HASH = 'a' * 64 +NS = 't3-p1-' + HASH[:24] +IMAGE = 'ghcr.io/example/t3@sha256:' + 'b' * 64 +DUID = '11111111-1111-4111-8111-111111111111' +RUID = '22222222-2222-4222-8222-222222222222' +PUID = '33333333-3333-4333-8333-333333333333' +PLAN = {'format': 't3-k3s-p1-plan-v1', 'leaseHash': HASH, 'namespace': NS, 'image': IMAGE} +BINDING = {'format': 't3-k3s-p1-binding-v3', 'leaseHash': HASH, 'namespace': NS, + 'deploymentUID': DUID, 'imageRefSha256': hashlib.sha256(IMAGE.encode()).hexdigest()} +CONTAINER = {'name': 't3', 'image': IMAGE, 'securityContext': {'readOnlyRootFilesystem': True}, + 'resources': {'limits': {'memory': '4Gi'}}, + 'volumeMounts': [{'name': 'data', 'mountPath': '/data'}]} +PODSPEC = {'serviceAccountName': 't3-worker', 'automountServiceAccountToken': False, + 'securityContext': {'runAsNonRoot': True}, + 'volumes': [{'name': 'data', 'persistentVolumeClaim': {'claimName': 't3-worker-data'}}], + 'containers': [copy.deepcopy(CONTAINER)]} +D = {'kind': 'Deployment', 'metadata': {'name': 't3-worker', 'uid': DUID, + 'namespace': NS, 'generation': 2}, + 'spec': {'template': {'spec': copy.deepcopy(PODSPEC)}}, + 'status': {'observedGeneration': 2, 'replicas': 1, 'readyReplicas': 1, + 'availableReplicas': 1, 'updatedReplicas': 1}} +RS = {'kind': 'ReplicaSet', 'metadata': {'name': 't3-worker-abcdef', 'uid': RUID, + 'namespace': NS, 'ownerReferences': [{'kind': 'Deployment', 'name': 't3-worker', + 'uid': DUID, 'controller': True}]}, + 'spec': {'replicas': 1}, 'status': {'readyReplicas': 1, 'replicas': 1}} +POD = {'kind': 'Pod', 'metadata': {'name': 't3-worker-abcdef-0123', 'namespace': NS, + 'uid': PUID, 'labels': {'app': 't3-worker'}, 'ownerReferences': [ + {'kind': 'ReplicaSet', 'name': RS['metadata']['name'], 'uid': RUID, + 'controller': True}]}, 'spec': copy.deepcopy(PODSPEC), + 'status': {'phase': 'Running', 'conditions': [{'type': 'Ready', 'status': 'True'}], + 'containerStatuses': [{'name': 't3', 'ready': True, 'started': True, + 'state': {'running': {'startedAt': '2026-10-29T00:00:00Z'}}, + 'image': IMAGE, 'imageID': 'docker-pullable://example@sha256:' + 'c' * 64, + 'containerID': 'containerd://' + 'd' * 64, 'restartCount': 0}]}} +RS_LIST = {'kind': 'ReplicaSetList', 'items': [RS]} +PODS = {'kind': 'PodList', 'items': [POD]} + + +def fixture(): + return copy.deepcopy([PLAN, BINDING, D, RS_LIST, PODS]) + + +class RuntimeTests(unittest.TestCase): + def test_binds_single_running_pod_without_exposing_image(self): + result = pc.inspect(*fixture()) + self.assertEqual(result['podUID'], PUID) + self.assertEqual(result['deploymentUID'], DUID) + self.assertEqual(result['restartCount'], 0) + self.assertNotIn(IMAGE, json.dumps(result)) + old = fixture() + old[3]['items'].append({'kind': 'ReplicaSet', 'metadata': {'name': 't3-worker-old', + 'uid': '44444444-4444-4444-4444-444444444444', 'namespace': NS, + 'ownerReferences': RS['metadata']['ownerReferences']}, + 'spec': {'replicas': 0}, 'status': {'replicas': 0}}) + pc.inspect(*old) # Kubernetes can retain an inactive revision. + + def test_reject_ownership_or_lease_rebinding(self): + for idx, path, value in [ + (1, ('deploymentUID',), 'ffffffff-ffff-ffff-ffff-ffffffffffff'), + (1, ('imageRefSha256',), '0' * 64), + (2, ('metadata', 'uid'), 'ffffffff-ffff-ffff-ffff-ffffffffffff'), + (3, ('items', 0, 'metadata', 'ownerReferences', 0, 'controller'), False), + (4, ('items', 0, 'metadata', 'ownerReferences', 0, 'uid'), 'fake'), + (4, ('items', 0, 'metadata', 'namespace'), 'other'), + ]: + args = fixture() + v = args[idx] + for key in path[:-1]: + v = v[key] + v[path[-1]] = value + with self.subTest(idx=idx, path=path), self.assertRaises(pc.PodError): + pc.inspect(*args) + + def test_reject_false_readiness_and_concurrent_pods(self): + cases = [ + (2, ('status', 'observedGeneration'), 1), + (2, ('status', 'readyReplicas'), 0), + (3, ('items', 0, 'status', 'readyReplicas'), 0), + (4, ('items', 0, 'status', 'phase'), 'Pending'), + (4, ('items', 0, 'status', 'conditions', 0, 'status'), 'False'), + (4, ('items', 0, 'status', 'containerStatuses', 0, 'ready'), False), + (4, ('items', 0, 'status', 'containerStatuses', 0, 'state'), {'waiting': {}}), + (4, ('items', 0, 'status', 'containerStatuses', 0, 'image'), + 'ghcr.io/example/other@sha256:' + 'b' * 64), + ] + for idx, path, value in cases: + args = fixture() + obj = args[idx] + for key in path[:-1]: + obj = obj[key] + obj[path[-1]] = value + with self.subTest(idx=idx, path=path), self.assertRaises(pc.PodError): + pc.inspect(*args) + args = fixture() + args[4]['items'].append(copy.deepcopy(POD)) + with self.assertRaises(pc.PodError): + pc.inspect(*args) + + def test_reject_mutated_pod_and_untrusted_snapshots(self): + for path, value in [ + (('spec', 'automountServiceAccountToken'), True), + (('spec', 'containers', 0, 'securityContext'), {'privileged': True}), + (('spec', 'containers', 0, 'image'), 'different'), + (('spec', 'volumes'), []), + (('spec', 'initContainers'), [{'name': 'sidecar'}]), + (('metadata', 'deletionTimestamp'), '2026-10-29T00:00:00Z'), + ]: + args = fixture() + pod = args[4]['items'][0] + for key in path[:-1]: + pod = pod[key] + pod[path[-1]] = value + with self.subTest(path=path), self.assertRaises(pc.PodError): + pc.inspect(*args) + for idx in range(5): + args = fixture() + args[idx] = None + with self.subTest(idx=idx), self.assertRaises((pc.PodError, TypeError, AttributeError)): + pc.inspect(*args) + + def test_cli_sanitizes_private_details(self): + with tempfile.TemporaryDirectory() as d: + paths = [Path(d) / name for name in ('plan', 'binding', 'deployment', 'replicasets', 'pods')] + for f, obj in zip(paths, fixture()): + f.write_text(json.dumps(obj)) + cmd = [sys.executable, str(HERE / 'pod_check.py')] + for flag, path in zip(('plan', 'binding', 'deployment', 'replicasets', 'pods'), paths): + cmd += ['--' + flag, str(path)] + good = subprocess.run(cmd, capture_output=True, text=True, check=False) + self.assertEqual(good.returncode, 0, good.stderr) + self.assertIn('NOT RUN:', good.stdout) + bad = copy.deepcopy(POD) + bad['status']['containerStatuses'][0]['image'] = 'private.example/secret' + paths[4].write_text(json.dumps({'kind': 'PodList', 'items': [bad]})) + failed = subprocess.run(cmd, capture_output=True, text=True, check=False) + self.assertNotEqual(failed.returncode, 0) + self.assertNotIn('private.example', failed.stderr) + self.assertNotIn(NS, failed.stderr) + + +if __name__ == '__main__': + unittest.main() From 1421b8f334671cab0345690b2c3f51178f6b43d1 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:17:27 -0500 Subject: [PATCH 53/84] docs(k3s-p1): add read-only Pod observation acceptance steps --- examples/k3s-p1/OBSERVATION.md | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/examples/k3s-p1/OBSERVATION.md b/examples/k3s-p1/OBSERVATION.md index 8a64f9c070500..b08939cdde1f8 100644 --- a/examples/k3s-p1/OBSERVATION.md +++ b/examples/k3s-p1/OBSERVATION.md @@ -77,3 +77,49 @@ repository/delegated task result, cancel/restart/TTL behavior and deliberate PVC retention. An old or expired plan is not a live reservation. Consult [work order #45](https://github.com/kvnloo/t3code/issues/45) and [stacked Draft #47](https://github.com/kvnloo/t3code/pull/47) for gates. + +## Optional read-only Pod runtime snapshot — NOT live proof + +After the desired-spec binding above, `pod_check.py` checks **saved JSON** +from one approved disposable K3s context. It binds Deployment UID → +ReplicaSet controller UID → the **only** Pod UID, verifies Deployment rollout +generation/ready replicas, Pod/container readiness as reported by Kubernetes, +pinned container image reference, required Pod security/storage, and hashes +the runtime image/container IDs into a private receipt. It rejects extra Pods, +stale rollout, wrong controller, changed image, extra containers/credentials and +unapproved Pod mutations. Older inactive ReplicaSets are permitted. + +These are **offline comparisons of supplied snapshots**, not authentication +of the Kubernetes API, live reachability, effective isolation, T3 persisted +`environmentId`, peer permission or accepted work. The imageID *hash* is +an observation, **not** equality proof against a multi-platform manifest's +pinned digest. Pod UID changing across restarts needs separate review; stable +Deployment/PVC identity cannot substitute for T3's persisted identity. + +Only after a separately approved P1 cluster run, save full namespace +inventories from the **same explicitly verified kubeconfig context**, close +together in time, into an owner-private directory: + +```sh +umask 077 +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE \ + get replicasets -o json > /tmp/t3-p1-private/replicasets.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE \ + get pods -o json > /tmp/t3-p1-private/pods.json +# Re-capture deployment.json alongside both inventories; keep the existing +# original, independently verified binding.json and lease plan. +python3 examples/k3s-p1/pod_check.py \ + --plan /tmp/k3s-p1-receipts/YOUR_FULL_LEASE_SHA256.json \ + --binding /tmp/t3-p1-private/binding.json \ + --deployment /tmp/t3-p1-private/deployment.json \ + --replicasets /tmp/t3-p1-private/replicasets.json \ + --pods /tmp/t3-p1-private/pods.json \ + > /tmp/t3-p1-private/pod-snapshot.json +``` + +Run `python3 -m unittest discover -s examples/k3s-p1 -p 'test_pod_check.py' -v` +for synthetic, credential-free regressions. Keep API snapshots private; they may +expose Pod labels and topology. No `kubectl apply`, no network calls from the +script and no automatic worker acceptance. After this check, **still NOT RUN**: +real T3 environment identity, private TLS/ACL, consented OAuth peer link, +verified delegated result and cancellation/restart behavior. From 6c6c8a94d2d0d3890338b3bb8deb8be1993eb04f Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:21:00 -0500 Subject: [PATCH 54/84] docs(k3s-p1): surface Pod snapshot checker and its proof boundary --- examples/k3s-p1/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index b9e8d4e5139c2..5017ae987389c 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -31,6 +31,8 @@ non-root Pod/container security, resources and probes, and rejects host access, additional containers and injected credentials/command overrides, not the image of a live Pod. Legacy v1/v2 baselines are rejected. It never contacts a cluster or proves a live T3 worker. +After the offline desired Deployment/PVC binding, the optional [Pod snapshot checker](OBSERVATION.md#optional-read-only-pod-runtime-snapshot--not-live-proof) can validate saved ReplicaSet→Pod controller UID, API-reported readiness and running image reference, and reject concurrent/mutated Pods. This is still **NOT live K3s proof**: its hashed imageID does not attest a multi-platform OCI digest, and it cannot establish persisted T3 identity, TLS, peer consent or successful delegation. + The local private mode-0600 receipt is create-once and atomically published only after a synced write. Concurrent identical requests reuse the complete receipt; a failed publish leaves no half-written final record. A retry with identical arguments renders identical YAML. Reusing the key with changed image/expiry fails closed rather than silently rebinding a purported lease. The receipt From bd89357afc94dd84137fa2d8564a6fd4c6b7e2d4 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 09:13:20 -0500 Subject: [PATCH 55/84] fix(k3s-p1): bind admitted pod startup ports and probes --- examples/k3s-p1/pod_check.py | 37 ++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/examples/k3s-p1/pod_check.py b/examples/k3s-p1/pod_check.py index 32288e6137123..b663fea036e55 100644 --- a/examples/k3s-p1/pod_check.py +++ b/examples/k3s-p1/pod_check.py @@ -100,8 +100,41 @@ def inspect(plan, binding, deployment, replicasets, pods): for field in ('name', 'image', 'securityContext', 'resources', 'volumeMounts'): need(containers[0].get(field) == template['containers'][0].get(field), 'admitted container configuration drift') - for field in ('env', 'envFrom', 'command', 'args', 'volumeDevices'): - need(not containers[0].get(field), 'injected Pod credentials or command') + for field in ('env', 'envFrom', 'command', 'args', 'volumeDevices', + 'lifecycle', 'stdin', 'stdinOnce', 'tty'): + need(not containers[0].get(field), 'injected Pod command or lifecycle drift') + # The desired Deployment snapshot is inspected separately. Bind the admitted + # Pod's startup path and port/probes to it; matching only image/resources + # misses hostPort, postStart exec and probe-command injection. + need(ps.get('hostname') == template.get('hostname'), 'Pod hostname drift') + desired = template['containers'][0] + admitted = containers[0] + for field in ('workingDir', 'imagePullPolicy'): + need(admitted.get(field) == desired.get(field), + 'admitted container startup drift') + want_ports, got_ports = desired.get('ports'), admitted.get('ports') + need(isinstance(want_ports, list) and len(want_ports) == 1 + and isinstance(want_ports[0], dict) + and isinstance(got_ports, list) and len(got_ports) == 1 + and isinstance(got_ports[0], dict), 'admitted container port drift') + wp, gp = want_ports[0], got_ports[0] + need(gp.get('name') == wp.get('name') + and gp.get('containerPort') == wp.get('containerPort') + and gp.get('protocol', 'TCP') == wp.get('protocol', 'TCP') + and gp.get('hostPort') in (None, 0) and not gp.get('hostIP'), + 'admitted container port or host exposure drift') + probe_defaults = {'periodSeconds': 10, 'failureThreshold': 3, + 'timeoutSeconds': 1, 'successThreshold': 1, + 'initialDelaySeconds': 0} + for name in ('startupProbe', 'readinessProbe'): + expected_probe, actual_probe = desired.get(name), admitted.get(name) + need(isinstance(expected_probe, dict) and isinstance(actual_probe, dict) + and set(actual_probe) <= set(probe_defaults) | {'tcpSocket'} + and actual_probe.get('tcpSocket') == expected_probe.get('tcpSocket'), + 'admitted container probe drift') + for field, default in probe_defaults.items(): + need(actual_probe.get(field, default) == expected_probe.get(field, default), + 'admitted container probe timing drift') pod_status = pod.get('status', {}) need(isinstance(pod_status, dict) and pod_status.get('phase') == 'Running', 'Pod not running') conditions = pod_status.get('conditions') From f60797b1de657cbe2bcb76d3e90405cdb5341d31 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 09:13:24 -0500 Subject: [PATCH 56/84] test(k3s-p1): cover port lifecycle and probe drift --- examples/k3s-p1/test_pod_check.py | 40 ++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/examples/k3s-p1/test_pod_check.py b/examples/k3s-p1/test_pod_check.py index a7ac88e042fd0..404e1a80ebf95 100644 --- a/examples/k3s-p1/test_pod_check.py +++ b/examples/k3s-p1/test_pod_check.py @@ -23,7 +23,13 @@ PLAN = {'format': 't3-k3s-p1-plan-v1', 'leaseHash': HASH, 'namespace': NS, 'image': IMAGE} BINDING = {'format': 't3-k3s-p1-binding-v3', 'leaseHash': HASH, 'namespace': NS, 'deploymentUID': DUID, 'imageRefSha256': hashlib.sha256(IMAGE.encode()).hexdigest()} -CONTAINER = {'name': 't3', 'image': IMAGE, 'securityContext': {'readOnlyRootFilesystem': True}, +CONTAINER = {'name': 't3', 'image': IMAGE, 'workingDir': '/data', + 'imagePullPolicy': 'IfNotPresent', + 'ports': [{'name': 'http', 'containerPort': 3773}], + 'startupProbe': {'tcpSocket': {'port': 'http'}, 'periodSeconds': 5, + 'failureThreshold': 60}, + 'readinessProbe': {'tcpSocket': {'port': 'http'}, 'periodSeconds': 5}, + 'securityContext': {'readOnlyRootFilesystem': True}, 'resources': {'limits': {'memory': '4Gi'}}, 'volumeMounts': [{'name': 'data', 'mountPath': '/data'}]} PODSPEC = {'serviceAccountName': 't3-worker', 'automountServiceAccountToken': False, @@ -134,6 +140,38 @@ def test_reject_mutated_pod_and_untrusted_snapshots(self): with self.subTest(idx=idx), self.assertRaises((pc.PodError, TypeError, AttributeError)): pc.inspect(*args) + def test_reject_admitted_startup_and_network_drift(self): + for path, value in [ + (('containers', 0, 'lifecycle'), + {'postStart': {'exec': {'command': ['/bin/sh', '-c', 'true']}}}), + (('containers', 0, 'ports', 0, 'hostPort'), 3773), + (('containers', 0, 'ports', 0, 'containerPort'), 9876), + (('containers', 0, 'startupProbe'), + {'exec': {'command': ['/bin/sh']}, 'periodSeconds': 5, + 'failureThreshold': 60}), + (('containers', 0, 'readinessProbe', 'tcpSocket', 'port'), 9876), + (('containers', 0, 'workingDir'), '/tmp'), + (('containers', 0, 'imagePullPolicy'), 'Always'), + (('hostname',), 'unexpected-host'), + ]: + args = fixture() + obj = args[4]['items'][0]['spec'] + for key in path[:-1]: + obj = obj[key] + obj[path[-1]] = value + with self.subTest(path=path), self.assertRaises(pc.PodError): + pc.inspect(*args) + + args = fixture() + admitted = args[4]['items'][0]['spec']['containers'][0] + # Kubernetes fills omitted probe defaults and TCP protocol. + admitted['ports'][0]['protocol'] = 'TCP' + for probe in ('startupProbe', 'readinessProbe'): + admitted[probe].update({'timeoutSeconds': 1, 'successThreshold': 1, + 'initialDelaySeconds': 0}) + admitted['readinessProbe']['failureThreshold'] = 3 + pc.inspect(*args) + def test_cli_sanitizes_private_details(self): with tempfile.TemporaryDirectory() as d: paths = [Path(d) / name for name in ('plan', 'binding', 'deployment', 'replicasets', 'pods')] From e24e209f74436eff5292d4cd1946e176034ec53d Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 09:13:27 -0500 Subject: [PATCH 57/84] docs(k3s-p1): note admitted runtime drift gate --- examples/k3s-p1/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index 5017ae987389c..dbfa207c58a1b 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -31,7 +31,7 @@ non-root Pod/container security, resources and probes, and rejects host access, additional containers and injected credentials/command overrides, not the image of a live Pod. Legacy v1/v2 baselines are rejected. It never contacts a cluster or proves a live T3 worker. -After the offline desired Deployment/PVC binding, the optional [Pod snapshot checker](OBSERVATION.md#optional-read-only-pod-runtime-snapshot--not-live-proof) can validate saved ReplicaSet→Pod controller UID, API-reported readiness and running image reference, and reject concurrent/mutated Pods. This is still **NOT live K3s proof**: its hashed imageID does not attest a multi-platform OCI digest, and it cannot establish persisted T3 identity, TLS, peer consent or successful delegation. +After the offline desired Deployment/PVC binding, the optional [Pod snapshot checker](OBSERVATION.md#optional-read-only-pod-runtime-snapshot--not-live-proof) can validate saved ReplicaSet→Pod controller UID, API-reported readiness and running image reference, and reject concurrent/mutated Pods, unexpected host ports, changed startup/probe behavior or injected post-start hooks. This is still **NOT live K3s proof**: its hashed imageID does not attest a multi-platform OCI digest, and it cannot establish persisted T3 identity, TLS, peer consent or successful delegation. The local private mode-0600 receipt is create-once and atomically published only after a synced write. Concurrent identical requests reuse the complete receipt; a failed publish leaves no half-written final record. A retry with identical arguments renders identical YAML. Reusing the key with changed image/expiry From 63c7aae662a69a6f83dd4eee9ec0b7c979bfe539 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:15:20 -0500 Subject: [PATCH 58/84] fix(k3s-p1): bind observed deployment generation to lease receipt --- examples/k3s-p1/observe.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/k3s-p1/observe.py b/examples/k3s-p1/observe.py index 468ebedc104ba..ab3028b6511e8 100644 --- a/examples/k3s-p1/observe.py +++ b/examples/k3s-p1/observe.py @@ -73,6 +73,9 @@ def _inspect(plan, namespace, pvc, deployment): if dmeta.get('deletionTimestamp'): raise ObservationError('deployment is deleting') deployment_uid = _uid(deployment, 'deployment') + generation = dmeta.get('generation') + if type(generation) is not int or generation < 1: + raise ObservationError('deployment lacks a valid generation') dspec = deployment.get('spec', {}) _require(dspec.get('replicas'), 1, 'deployment replicas') _require(dspec.get('strategy', {}).get('type'), 'Recreate', 'deployment strategy') @@ -200,10 +203,10 @@ def _inspect(plan, namespace, pvc, deployment): if not isinstance(volume, str) or not volume or len(volume) > 253: raise ObservationError('PVC has no bound PersistentVolume name') - return {'format': 't3-k3s-p1-binding-v3', 'leaseHash': lease_hash, + return {'format': 't3-k3s-p1-binding-v4', 'leaseHash': lease_hash, 'namespace': name, 'namespaceUID': _uid(namespace, 'namespace'), 'pvcUID': _uid(pvc, 'PVC'), 'deploymentUID': deployment_uid, - 'volumeName': volume, + 'deploymentGeneration': generation, 'volumeName': volume, 'imageRefSha256': hashlib.sha256(image.encode('ascii')).hexdigest(), 'expiresAt': expiry, 'retention': 'retain'} From 73eea63ac179a148501f9ff5fdda241e1da8f858 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:15:23 -0500 Subject: [PATCH 59/84] test(k3s-p1): reject stale deployment generation bindings --- examples/k3s-p1/test_observe.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/examples/k3s-p1/test_observe.py b/examples/k3s-p1/test_observe.py index 330512c3a4003..f204800823cdf 100644 --- a/examples/k3s-p1/test_observe.py +++ b/examples/k3s-p1/test_observe.py @@ -38,7 +38,7 @@ DEPLOYMENT = {'kind': 'Deployment', 'metadata': { 'name': 't3-worker', 'namespace': NAMESPACE, - 'uid': '44444444-aaaa-bbbb-cccc-444444444444'}, + 'uid': '44444444-aaaa-bbbb-cccc-444444444444', 'generation': 2}, 'spec': {'replicas': 1, 'strategy': {'type': 'Recreate'}, 'selector': {'matchLabels': {'app': 't3-worker'}}, 'template': {'metadata': {'labels': {'app': 't3-worker'}}, @@ -87,6 +87,20 @@ def test_stable_namespace_and_pvc_uid(self): self.assertNotIn('image', initial) self.assertNotIn('credentials', json.dumps(initial)) + def test_generation_blocks_stale_same_uid_binding(self): + original = p.inspect(PLAN, NS, PVC, DEPLOYMENT) + self.assertEqual(original['deploymentGeneration'], 2) + changed = copy.deepcopy(DEPLOYMENT) + changed['metadata']['generation'] = 3 # Kubernetes spec mutation preserves UID. + with self.assertRaisesRegex(p.ObservationError, 'binding changed'): + p.compare(p.inspect(PLAN, NS, PVC, changed), original) + for invalid in (None, 0, -1, True, '2'): + bad = copy.deepcopy(DEPLOYMENT) + bad['metadata']['generation'] = invalid + with self.subTest(invalid=invalid), self.assertRaisesRegex( + p.ObservationError, 'generation'): + p.inspect(PLAN, NS, PVC, bad) + def test_wrong_lease_identity_or_namespace_is_rejected(self): for item, change in [('ns', {'metadata': {'name': 'other'}}), ('plan', {'namespace': 'different'}), @@ -136,7 +150,7 @@ def test_namespace_annotation_or_policy_drift_fails(self): def test_changed_pinned_image_invalidates_existing_binding(self): first = p.inspect(PLAN, NS, PVC, DEPLOYMENT) - self.assertEqual(first['format'], 't3-k3s-p1-binding-v3') + self.assertEqual(first['format'], 't3-k3s-p1-binding-v4') self.assertEqual(len(first['imageRefSha256']), 64) self.assertNotIn(PLAN['image'], json.dumps(first)) changed = copy.deepcopy(PLAN) From cfea4fc6e0588d7013875c5e6af0f4e569e0a8d4 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:15:42 -0500 Subject: [PATCH 60/84] fix(k3s-p1): reject Pod verification after deployment revision --- examples/k3s-p1/pod_check.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/k3s-p1/pod_check.py b/examples/k3s-p1/pod_check.py index b663fea036e55..f8ce74f5082bd 100644 --- a/examples/k3s-p1/pod_check.py +++ b/examples/k3s-p1/pod_check.py @@ -47,12 +47,17 @@ def inspect(plan, binding, deployment, replicasets, pods): r'(?=.{1,255}$)[a-z0-9][a-z0-9._:/-]*/[a-z0-9][a-z0-9._/-]*@sha256:[a-f0-9]{64}', image) and '..' not in image and '//' not in image, 'image not pinned') image_hash = hashlib.sha256(image.encode('ascii')).hexdigest() - need(plan.get('namespace') == ns and binding.get('format') == 't3-k3s-p1-binding-v3' + need(plan.get('namespace') == ns and binding.get('format') == 't3-k3s-p1-binding-v4' and binding.get('namespace') == ns and binding.get('leaseHash') == lease and binding.get('imageRefSha256') == image_hash, 'lease binding mismatch') dmeta = meta(deployment, 'Deployment', ns) need(dmeta.get('name') == 't3-worker' and dmeta['uid'] == binding.get('deploymentUID'), 'deployment binding drift') + # A Deployment may keep its UID while its desired spec is changed. The + # read-only binding must be refreshed and re-reviewed for any new generation. + need(type(dmeta.get('generation')) is int and dmeta['generation'] > 0 + and binding.get('deploymentGeneration') == dmeta['generation'], + 'deployment changed since verified binding') ds = deployment.get('spec', {}) template = ds.get('template', {}).get('spec', {}) need(isinstance(template, dict) and isinstance(template.get('containers'), list) From abed115d4d1470fb411fb25f1b32ed1b7e8d6c57 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:15:45 -0500 Subject: [PATCH 61/84] test(k3s-p1): cover changed-generation Pod snapshot --- examples/k3s-p1/test_pod_check.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/examples/k3s-p1/test_pod_check.py b/examples/k3s-p1/test_pod_check.py index 404e1a80ebf95..2a80547486eb6 100644 --- a/examples/k3s-p1/test_pod_check.py +++ b/examples/k3s-p1/test_pod_check.py @@ -21,8 +21,8 @@ RUID = '22222222-2222-4222-8222-222222222222' PUID = '33333333-3333-4333-8333-333333333333' PLAN = {'format': 't3-k3s-p1-plan-v1', 'leaseHash': HASH, 'namespace': NS, 'image': IMAGE} -BINDING = {'format': 't3-k3s-p1-binding-v3', 'leaseHash': HASH, 'namespace': NS, - 'deploymentUID': DUID, 'imageRefSha256': hashlib.sha256(IMAGE.encode()).hexdigest()} +BINDING = {'format': 't3-k3s-p1-binding-v4', 'leaseHash': HASH, 'namespace': NS, + 'deploymentUID': DUID, 'deploymentGeneration': 2, 'imageRefSha256': hashlib.sha256(IMAGE.encode()).hexdigest()} CONTAINER = {'name': 't3', 'image': IMAGE, 'workingDir': '/data', 'imagePullPolicy': 'IfNotPresent', 'ports': [{'name': 'http', 'containerPort': 3773}], @@ -79,6 +79,7 @@ def test_binds_single_running_pod_without_exposing_image(self): def test_reject_ownership_or_lease_rebinding(self): for idx, path, value in [ (1, ('deploymentUID',), 'ffffffff-ffff-ffff-ffff-ffffffffffff'), + (1, ('deploymentGeneration',), 1), (1, ('imageRefSha256',), '0' * 64), (2, ('metadata', 'uid'), 'ffffffff-ffff-ffff-ffff-ffffffffffff'), (3, ('items', 0, 'metadata', 'ownerReferences', 0, 'controller'), False), @@ -93,6 +94,19 @@ def test_reject_ownership_or_lease_rebinding(self): with self.subTest(idx=idx, path=path), self.assertRaises(pc.PodError): pc.inspect(*args) + def test_reject_spec_change_between_binding_and_pod_read(self): + args = fixture() + args[2]['metadata']['generation'] = 3 + args[2]['status']['observedGeneration'] = 3 + # Same UID, image and ready replicas do not imply unchanged spec. + with self.assertRaisesRegex(pc.PodError, 'deployment changed'): + pc.inspect(*args) + for bad in (None, 0, True, '2'): + args = fixture() + args[1]['deploymentGeneration'] = bad + with self.subTest(bad=bad), self.assertRaises(pc.PodError): + pc.inspect(*args) + def test_reject_false_readiness_and_concurrent_pods(self): cases = [ (2, ('status', 'observedGeneration'), 1), From bfe5ea13f3bde15df76fba60b49e06f73d229443 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:16:04 -0500 Subject: [PATCH 62/84] docs(k3s-p1): require re-review on deployment generation changes --- examples/k3s-p1/OBSERVATION.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/examples/k3s-p1/OBSERVATION.md b/examples/k3s-p1/OBSERVATION.md index b08939cdde1f8..86eb79f23432a 100644 --- a/examples/k3s-p1/OBSERVATION.md +++ b/examples/k3s-p1/OBSERVATION.md @@ -4,14 +4,14 @@ `observe.py` helper compares one immutable P1 plan with API JSON for its Namespace, PVC and Deployment, then extracts their stable Kubernetes UIDs. A previous binding can be replay-checked so a replaced Namespace, PVC, Deployment **or pinned image** -**fails closed** rather than looking like the same worker lease. The v3 receipt contains a SHA-256 fingerprint of the full immutable OCI image +**fails closed** rather than looking like the same worker lease. The v4 receipt contains a SHA-256 fingerprint of the full immutable OCI image reference, never the raw registry/image name, and checks the observed Deployment's **desired Pod template** image equals the plan. It also requires the planned non-root Pod/container security, resource bounds, probes and no unexpected host-network/PID/IPC access, injected init containers, registry credentials, environment secrets or shell overrides. An unexpected admission mutation fails closed for human review; Kubernetes may default other fields. Old v1/v2 binding -baselines cannot be reused: recollect a trusted v3 baseline from one approved +or v3 baselines cannot be reused: recollect a trusted v4 baseline from one approved context. This does **not** check the image actually running in any Pod. This is **not** a provisioner, cluster lease, service/Pod readiness probe, @@ -65,7 +65,7 @@ python3 examples/k3s-p1/observe.py \ For a subsequent observation, **refresh all three Kubernetes JSON snapshots from the same approved context** and use the **same pinned image receipt**, then pass `--baseline /tmp/t3-p1-private/binding.json`. A Namespace/PVC/Deployment UID, desired image or PV -binding change now fails. A missing/unbound PVC or lease/retention mismatch +binding or Deployment generation change now fails. A missing/unbound PVC or lease/retention mismatch also fails; nothing is automatically repaired or deleted. Store the baseline safely and never overwrite it merely to make a failing comparison pass. @@ -85,7 +85,9 @@ from one approved disposable K3s context. It binds Deployment UID → ReplicaSet controller UID → the **only** Pod UID, verifies Deployment rollout generation/ready replicas, Pod/container readiness as reported by Kubernetes, pinned container image reference, required Pod security/storage, and hashes -the runtime image/container IDs into a private receipt. It rejects extra Pods, +the runtime image/container IDs into a private receipt. The v4 binding also +pins the Deployment's observed generation, so a changed spec on the same UID +cannot be mistaken for the originally reviewed Deployment. It rejects extra Pods, stale rollout, wrong controller, changed image, extra containers/credentials and unapproved Pod mutations. Older inactive ReplicaSets are permitted. @@ -117,6 +119,11 @@ python3 examples/k3s-p1/pod_check.py \ > /tmp/t3-p1-private/pod-snapshot.json ``` +If the Deployment generation changes, stop, review the new desired spec and +re-run `observe.py` against fresh same-context snapshots; do **not** overwrite +the original binding merely to make the check pass. Kubernetes increments +Deployment generation on spec changes, but offline snapshots can still be forged. + Run `python3 -m unittest discover -s examples/k3s-p1 -p 'test_pod_check.py' -v` for synthetic, credential-free regressions. Keep API snapshots private; they may expose Pod labels and topology. No `kubectl apply`, no network calls from the From ee71f01c01bc6498728de9069013dbf00c170bf0 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:16:06 -0500 Subject: [PATCH 63/84] docs(k3s-p1): explain v4 binding and immutable preview gate --- examples/k3s-p1/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index dbfa207c58a1b..98451e3beacb4 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -25,11 +25,13 @@ isolation, TLS, ACL or peer-authorization proof. The separate [offline UID observation helper](OBSERVATION.md) checks a reported Kubernetes Namespace/PVC/Deployment binding **and immutable -image-reference fingerprint** against the plan and an earlier v3 baseline. +image-reference fingerprint** against the plan and an earlier v4 baseline. The v4 binding also pins +the Deployment generation, preventing a same-UID spec update from passing a +later Pod check without renewed human review. It checks the Deployment's desired template image, PVC/SA mounting, non-root Pod/container security, resources and probes, and rejects host access, additional containers and injected credentials/command overrides, -not the image of a live Pod. Legacy v1/v2 baselines are rejected. It never contacts a cluster or proves a live T3 worker. +not the image of a live Pod. Legacy v1/v2/v3 baselines are rejected. It never contacts a cluster or proves a live T3 worker. After the offline desired Deployment/PVC binding, the optional [Pod snapshot checker](OBSERVATION.md#optional-read-only-pod-runtime-snapshot--not-live-proof) can validate saved ReplicaSet→Pod controller UID, API-reported readiness and running image reference, and reject concurrent/mutated Pods, unexpected host ports, changed startup/probe behavior or injected post-start hooks. This is still **NOT live K3s proof**: its hashed imageID does not attest a multi-platform OCI digest, and it cannot establish persisted T3 identity, TLS, peer consent or successful delegation. From ae9756de35f63dcf9b9dcd784508eaf8ab04faf3 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:11:07 -0500 Subject: [PATCH 64/84] test(k3s-p1): reject active ReplicaSet template drift after Pod replacement --- examples/k3s-p1/test_pod_check.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/examples/k3s-p1/test_pod_check.py b/examples/k3s-p1/test_pod_check.py index 2a80547486eb6..b1523909732c4 100644 --- a/examples/k3s-p1/test_pod_check.py +++ b/examples/k3s-p1/test_pod_check.py @@ -44,7 +44,8 @@ RS = {'kind': 'ReplicaSet', 'metadata': {'name': 't3-worker-abcdef', 'uid': RUID, 'namespace': NS, 'ownerReferences': [{'kind': 'Deployment', 'name': 't3-worker', 'uid': DUID, 'controller': True}]}, - 'spec': {'replicas': 1}, 'status': {'readyReplicas': 1, 'replicas': 1}} + 'spec': {'replicas': 1, 'template': {'spec': copy.deepcopy(PODSPEC)}}, + 'status': {'readyReplicas': 1, 'replicas': 1}} POD = {'kind': 'Pod', 'metadata': {'name': 't3-worker-abcdef-0123', 'namespace': NS, 'uid': PUID, 'labels': {'app': 't3-worker'}, 'ownerReferences': [ {'kind': 'ReplicaSet', 'name': RS['metadata']['name'], 'uid': RUID, @@ -107,6 +108,30 @@ def test_reject_spec_change_between_binding_and_pod_read(self): with self.subTest(bad=bad), self.assertRaises(pc.PodError): pc.inspect(*args) + def test_reject_active_replicaset_template_drift(self): + # A clean Pod can be replaced from an altered ReplicaSet template. + baseline = fixture() + baseline[3]['items'][0]['spec']['template']['metadata'] = { + 'labels': {'pod-template-hash': 'abcdef'}} + pc.inspect(*baseline) # Controller-added template metadata is expected. + for change in ('missing', 'sidecar', 'image', 'service-account', 'volume'): + args = fixture() + spec = args[3]['items'][0]['spec'] + if change == 'missing': + del spec['template'] + elif change == 'sidecar': + spec['template']['spec']['containers'].append( + {'name': 'injected', 'image': IMAGE}) + elif change == 'image': + spec['template']['spec']['containers'][0]['image'] = 't3:latest' + elif change == 'service-account': + spec['template']['spec']['serviceAccountName'] = 'default' + elif change == 'volume': + spec['template']['spec']['volumes'] = [{'name': 'data', 'hostPath': {'path': '/host'}}] + with self.subTest(change=change), self.assertRaisesRegex( + pc.PodError, 'ReplicaSet template'): + pc.inspect(*args) + def test_reject_false_readiness_and_concurrent_pods(self): cases = [ (2, ('status', 'observedGeneration'), 1), From 28ee025b59e7fa9615792acb45aad91d8d2c0f9c Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:11:21 -0500 Subject: [PATCH 65/84] fix(k3s-p1): bind active ReplicaSet respawn template to approved Deployment --- examples/k3s-p1/pod_check.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/k3s-p1/pod_check.py b/examples/k3s-p1/pod_check.py index f8ce74f5082bd..1b86b4c78df97 100644 --- a/examples/k3s-p1/pod_check.py +++ b/examples/k3s-p1/pod_check.py @@ -83,6 +83,13 @@ def inspect(plan, binding, deployment, replicasets, pods): rs, rm = active_rs[0] need(rs.get('spec', {}).get('replicas') == 1 and rs.get('status', {}).get('readyReplicas') == 1, 'ReplicaSet not ready') + # ReplicaSet is the template used for the NEXT Pod after restart/eviction. + # A clean, Ready Pod is insufficient if its controller was subsequently + # changed to launch a different image, credential or unsafe container. + # Compare spec only: Deployment adds pod-template-hash to RS template labels. + rs_template = rs.get('spec', {}).get('template') + need(isinstance(rs_template, dict) and isinstance(rs_template.get('spec'), dict) + and rs_template['spec'] == template, 'active ReplicaSet template drift') need(isinstance(pods, dict) and pods.get('kind') == 'PodList' and isinstance(pods.get('items'), list) and len(pods['items']) == 1, 'worker namespace does not have exactly one Pod') From 4ec3e6486580269ca691a61a828c319942956b02 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:13:32 -0500 Subject: [PATCH 66/84] docs(k3s-p1): document ReplicaSet replacement drift and current remote seam --- examples/k3s-p1/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index 98451e3beacb4..d350f51fb9fce 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -33,7 +33,7 @@ non-root Pod/container security, resources and probes, and rejects host access, additional containers and injected credentials/command overrides, not the image of a live Pod. Legacy v1/v2/v3 baselines are rejected. It never contacts a cluster or proves a live T3 worker. -After the offline desired Deployment/PVC binding, the optional [Pod snapshot checker](OBSERVATION.md#optional-read-only-pod-runtime-snapshot--not-live-proof) can validate saved ReplicaSet→Pod controller UID, API-reported readiness and running image reference, and reject concurrent/mutated Pods, unexpected host ports, changed startup/probe behavior or injected post-start hooks. This is still **NOT live K3s proof**: its hashed imageID does not attest a multi-platform OCI digest, and it cannot establish persisted T3 identity, TLS, peer consent or successful delegation. +After the offline desired Deployment/PVC binding, the optional [Pod snapshot checker](OBSERVATION.md#optional-read-only-pod-runtime-snapshot--not-live-proof) can validate saved ReplicaSet→Pod controller UID, API-reported readiness and running image reference, and reject concurrent/mutated Pods, unexpected host ports, changed startup/probe behavior or injected post-start hooks. It also compares the active ReplicaSet's replacement-Pod spec with the approved Deployment template, rejecting drift that could affect a future Pod after restart/eviction. This is still **NOT live K3s proof**: its hashed imageID does not attest a multi-platform OCI digest, and it cannot establish persisted T3 identity, TLS, peer consent or successful delegation. The local private mode-0600 receipt is create-once and atomically published only after a synced write. Concurrent identical requests reuse the complete receipt; a failed publish leaves no half-written final record. A retry with identical arguments renders identical YAML. Reusing the key with changed image/expiry @@ -89,7 +89,10 @@ fails instead of overwriting it. assume worker and owner share a filesystem, provider secrets or GPUs. The original P0 image is pinned to T3 release v0.1.3; latest-main beta -compatibility is **unverified**. This preview does not install tools, +compatibility is **unverified**. At upstream main `960d5761c13f68a2f10635bcd26489a5fa404ec1` +(2026-10-29), linked-environment `RemoteDelegation.ts` exists but still launches a +remote child before recording the owning task ([#17051](https://github.com/pingdotgg/t3code/issues/17051)); +its restart durability is not proven. This preview does not install tools, clone repositories, contact Kubernetes, link agents, grant credentials, launch workers or alter the current deployment. From 2ecae97d59c4bef72640c3e0545978a2c2ea09d4 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:17:38 -0500 Subject: [PATCH 67/84] fix(k3s-p1): bind replacement Pod metadata and ReplicaSet selector --- examples/k3s-p1/README.md | 2 +- examples/k3s-p1/observe.py | 9 ++++++- examples/k3s-p1/pod_check.py | 29 ++++++++++++++++++++- examples/k3s-p1/test_observe.py | 18 +++++++++++++ examples/k3s-p1/test_pod_check.py | 42 ++++++++++++++++++++++++++----- 5 files changed, 91 insertions(+), 9 deletions(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index d350f51fb9fce..ed25cc8ace45d 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -33,7 +33,7 @@ non-root Pod/container security, resources and probes, and rejects host access, additional containers and injected credentials/command overrides, not the image of a live Pod. Legacy v1/v2/v3 baselines are rejected. It never contacts a cluster or proves a live T3 worker. -After the offline desired Deployment/PVC binding, the optional [Pod snapshot checker](OBSERVATION.md#optional-read-only-pod-runtime-snapshot--not-live-proof) can validate saved ReplicaSet→Pod controller UID, API-reported readiness and running image reference, and reject concurrent/mutated Pods, unexpected host ports, changed startup/probe behavior or injected post-start hooks. It also compares the active ReplicaSet's replacement-Pod spec with the approved Deployment template, rejecting drift that could affect a future Pod after restart/eviction. This is still **NOT live K3s proof**: its hashed imageID does not attest a multi-platform OCI digest, and it cannot establish persisted T3 identity, TLS, peer consent or successful delegation. +After the offline desired Deployment/PVC binding, the optional [Pod snapshot checker](OBSERVATION.md#optional-read-only-pod-runtime-snapshot--not-live-proof) can validate saved ReplicaSet→Pod controller UID, API-reported readiness and running image reference, and reject concurrent/mutated Pods, unexpected host ports, changed startup/probe behavior or injected post-start hooks. It also compares the active ReplicaSet's replacement-Pod spec, template labels/annotations and selector with the approved Deployment, rejecting injection or routing drift that could affect a future Pod after restart/eviction. This is still **NOT live K3s proof**: its hashed imageID does not attest a multi-platform OCI digest, and it cannot establish persisted T3 identity, TLS, peer consent or successful delegation. The local private mode-0600 receipt is create-once and atomically published only after a synced write. Concurrent identical requests reuse the complete receipt; a failed publish leaves no half-written final record. A retry with identical arguments renders identical YAML. Reusing the key with changed image/expiry diff --git a/examples/k3s-p1/observe.py b/examples/k3s-p1/observe.py index ab3028b6511e8..87ebdb6f93f48 100644 --- a/examples/k3s-p1/observe.py +++ b/examples/k3s-p1/observe.py @@ -82,7 +82,14 @@ def _inspect(plan, namespace, pvc, deployment): expected_labels = {'app': 't3-worker'} _require(dspec.get('selector', {}).get('matchLabels'), expected_labels, 'deployment selector') template = dspec.get('template', {}) - _require(template.get('metadata', {}).get('labels'), expected_labels, 'deployment Pod labels') + template_meta = template.get('metadata', {}) + _require(template_meta.get('labels'), expected_labels, 'deployment Pod labels') + # Admission-triggering metadata can change a future replacement Pod even + # when the approved image, ServiceAccount and persistent volume match. + if (set(template_meta) - {'labels', 'annotations', 'creationTimestamp'} + or template_meta.get('creationTimestamp') is not None + or template_meta.get('annotations')): + raise ObservationError('deployment unexpected Pod template metadata') pod = template.get('spec', {}) _require(pod.get('serviceAccountName'), 't3-worker', 'deployment ServiceAccount') _require(pod.get('automountServiceAccountToken'), False, 'deployment API token') diff --git a/examples/k3s-p1/pod_check.py b/examples/k3s-p1/pod_check.py index 1b86b4c78df97..20324adfffc0a 100644 --- a/examples/k3s-p1/pod_check.py +++ b/examples/k3s-p1/pod_check.py @@ -90,13 +90,40 @@ def inspect(plan, binding, deployment, replicasets, pods): rs_template = rs.get('spec', {}).get('template') need(isinstance(rs_template, dict) and isinstance(rs_template.get('spec'), dict) and rs_template['spec'] == template, 'active ReplicaSet template drift') + # Spec equality alone misses an edited replacement template's labels, + # selector or annotations. Those can change routing or trigger admission + # sidecars when the ReplicaSet creates a replacement Pod. + desired_meta = ds.get('template', {}).get('metadata') + replacement_meta = rs_template.get('metadata') + need(isinstance(desired_meta, dict) and isinstance(replacement_meta, dict) + and set(desired_meta) <= {'labels', 'annotations', 'creationTimestamp'} + and set(replacement_meta) <= {'labels', 'annotations', 'creationTimestamp'} + and desired_meta.get('creationTimestamp') is None + and replacement_meta.get('creationTimestamp') is None + and desired_meta.get('labels') == {'app': 't3-worker'} + and not desired_meta.get('annotations') + and not replacement_meta.get('annotations'), + 'replacement Pod template metadata drift') + replacement_labels = replacement_meta.get('labels') + need(isinstance(replacement_labels, dict) + and set(replacement_labels) == {'app', 'pod-template-hash'} + and replacement_labels.get('app') == 't3-worker' + and isinstance(replacement_labels.get('pod-template-hash'), str) + and re.fullmatch(r'[a-z0-9-]{1,63}', replacement_labels['pod-template-hash']), + 'replacement Pod template labels drift') + selector = rs.get('spec', {}).get('selector') + need(isinstance(selector, dict) + and selector.get('matchLabels') == replacement_labels + and not selector.get('matchExpressions') + and set(selector) <= {'matchLabels', 'matchExpressions'}, + 'active ReplicaSet selector drift') need(isinstance(pods, dict) and pods.get('kind') == 'PodList' and isinstance(pods.get('items'), list) and len(pods['items']) == 1, 'worker namespace does not have exactly one Pod') pod = pods['items'][0] pm = meta(pod, 'Pod', ns) controller(pod, 'ReplicaSet', rm.get('name'), rm['uid']) - need(pm.get('labels', {}).get('app') == 't3-worker', 'Pod selector drift') + need(pm.get('labels') == replacement_labels, 'Pod selector drift') ps = pod.get('spec', {}) need(isinstance(ps, dict) and ps.get('serviceAccountName') == 't3-worker' and ps.get('automountServiceAccountToken') is False diff --git a/examples/k3s-p1/test_observe.py b/examples/k3s-p1/test_observe.py index f204800823cdf..fc69ae3b1297d 100644 --- a/examples/k3s-p1/test_observe.py +++ b/examples/k3s-p1/test_observe.py @@ -220,6 +220,24 @@ def test_deployment_identity_and_pvc_mount_drift_fail_closed(self): p.compare(p.inspect(PLAN, NS, PVC, replaced), p.inspect(PLAN, NS, PVC, DEPLOYMENT)) + def test_template_metadata_cannot_trigger_admission_on_restart(self): + for change in ('annotation', 'extra-field', 'timestamp'): + bad = copy.deepcopy(DEPLOYMENT) + meta = bad['spec']['template']['metadata'] + if change == 'annotation': + meta['annotations'] = {'sidecar.istio.io/inject': 'true'} + elif change == 'extra-field': + meta['finalizers'] = ['unexpected.example/finalizer'] + else: + meta['creationTimestamp'] = '2026-10-29T00:00:00Z' + with self.subTest(change=change), self.assertRaisesRegex( + p.ObservationError, 'template metadata'): + p.inspect(PLAN, NS, PVC, bad) + defaulted = copy.deepcopy(DEPLOYMENT) + defaulted['spec']['template']['metadata'].update( + {'annotations': {}, 'creationTimestamp': None}) + p.inspect(PLAN, NS, PVC, defaulted) + def test_expected_kubernetes_api_defaults_do_not_false_reject(self): defaulted = copy.deepcopy(DEPLOYMENT) container = defaulted['spec']['template']['spec']['containers'][0] diff --git a/examples/k3s-p1/test_pod_check.py b/examples/k3s-p1/test_pod_check.py index b1523909732c4..f6a9809c4d0dc 100644 --- a/examples/k3s-p1/test_pod_check.py +++ b/examples/k3s-p1/test_pod_check.py @@ -38,16 +38,21 @@ 'containers': [copy.deepcopy(CONTAINER)]} D = {'kind': 'Deployment', 'metadata': {'name': 't3-worker', 'uid': DUID, 'namespace': NS, 'generation': 2}, - 'spec': {'template': {'spec': copy.deepcopy(PODSPEC)}}, + 'spec': {'template': {'metadata': {'labels': {'app': 't3-worker'}}, + 'spec': copy.deepcopy(PODSPEC)}}, 'status': {'observedGeneration': 2, 'replicas': 1, 'readyReplicas': 1, 'availableReplicas': 1, 'updatedReplicas': 1}} RS = {'kind': 'ReplicaSet', 'metadata': {'name': 't3-worker-abcdef', 'uid': RUID, 'namespace': NS, 'ownerReferences': [{'kind': 'Deployment', 'name': 't3-worker', 'uid': DUID, 'controller': True}]}, - 'spec': {'replicas': 1, 'template': {'spec': copy.deepcopy(PODSPEC)}}, + 'spec': {'replicas': 1, + 'selector': {'matchLabels': {'app': 't3-worker', 'pod-template-hash': 'abcdef'}}, + 'template': {'metadata': {'labels': {'app': 't3-worker', + 'pod-template-hash': 'abcdef'}}, + 'spec': copy.deepcopy(PODSPEC)}}, 'status': {'readyReplicas': 1, 'replicas': 1}} POD = {'kind': 'Pod', 'metadata': {'name': 't3-worker-abcdef-0123', 'namespace': NS, - 'uid': PUID, 'labels': {'app': 't3-worker'}, 'ownerReferences': [ + 'uid': PUID, 'labels': {'app': 't3-worker', 'pod-template-hash': 'abcdef'}, 'ownerReferences': [ {'kind': 'ReplicaSet', 'name': RS['metadata']['name'], 'uid': RUID, 'controller': True}]}, 'spec': copy.deepcopy(PODSPEC), 'status': {'phase': 'Running', 'conditions': [{'type': 'Ready', 'status': 'True'}], @@ -111,9 +116,9 @@ def test_reject_spec_change_between_binding_and_pod_read(self): def test_reject_active_replicaset_template_drift(self): # A clean Pod can be replaced from an altered ReplicaSet template. baseline = fixture() - baseline[3]['items'][0]['spec']['template']['metadata'] = { - 'labels': {'pod-template-hash': 'abcdef'}} - pc.inspect(*baseline) # Controller-added template metadata is expected. + baseline[2]['spec']['template']['metadata']['creationTimestamp'] = None + baseline[3]['items'][0]['spec']['template']['metadata']['creationTimestamp'] = None + pc.inspect(*baseline) # Kubernetes's harmless null timestamp is expected. for change in ('missing', 'sidecar', 'image', 'service-account', 'volume'): args = fixture() spec = args[3]['items'][0]['spec'] @@ -132,6 +137,31 @@ def test_reject_active_replicaset_template_drift(self): pc.PodError, 'ReplicaSet template'): pc.inspect(*args) + def test_reject_replacement_metadata_or_selector_drift(self): + for change in ('sidecar-annotation', 'extra-label', 'wrong-hash', + 'wrong-selector', 'extra-selector', 'pod-label', 'desired-annotation'): + args = fixture() + desired = args[2]['spec']['template']['metadata'] + rs_spec = args[3]['items'][0]['spec'] + rs_meta = rs_spec['template']['metadata'] + if change == 'sidecar-annotation': + rs_meta['annotations'] = {'sidecar.istio.io/inject': 'true'} + elif change == 'extra-label': + rs_meta['labels']['expose'] = 'public' + elif change == 'wrong-hash': + rs_meta['labels']['pod-template-hash'] = 'wrong' + elif change == 'wrong-selector': + rs_spec['selector']['matchLabels']['app'] = 'another' + elif change == 'extra-selector': + rs_spec['selector']['matchExpressions'] = [ + {'key': 'tier', 'operator': 'Exists'}] + elif change == 'pod-label': + args[4]['items'][0]['metadata']['labels']['expose'] = 'public' + else: + desired['annotations'] = {'sidecar.istio.io/inject': 'true'} + with self.subTest(change=change), self.assertRaises(pc.PodError): + pc.inspect(*args) + def test_reject_false_readiness_and_concurrent_pods(self): cases = [ (2, ('status', 'observedGeneration'), 1), From 44be83440b88867a9f36fa297f013108e82828ea Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:15:08 -0500 Subject: [PATCH 68/84] fix(k3s-p1): reject unapproved Pod admission network and runtime overlays --- examples/k3s-p1/pod_check.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/examples/k3s-p1/pod_check.py b/examples/k3s-p1/pod_check.py index 20324adfffc0a..db353a884f236 100644 --- a/examples/k3s-p1/pod_check.py +++ b/examples/k3s-p1/pod_check.py @@ -124,6 +124,9 @@ def inspect(plan, binding, deployment, replicasets, pods): pm = meta(pod, 'Pod', ns) controller(pod, 'ReplicaSet', rm.get('name'), rm['uid']) need(pm.get('labels') == replacement_labels, 'Pod selector drift') + # Pod-level CNI/admission annotations may attach additional interfaces even + # when the ReplicaSet template and container specification remain intact. + need(pm.get('annotations') in (None, {}), 'unapproved Pod admission annotations') ps = pod.get('spec', {}) need(isinstance(ps, dict) and ps.get('serviceAccountName') == 't3-worker' and ps.get('automountServiceAccountToken') is False @@ -131,6 +134,22 @@ def inspect(plan, binding, deployment, replicasets, pods): and ps.get('volumes') == template.get('volumes'), 'admitted Pod security/storage drift') for field in ('hostNetwork', 'hostPID', 'hostIPC', 'shareProcessNamespace'): need(ps.get(field) in (None, False), 'Pod host isolation drift') + # Admission can change the Pod's runtime, GPU/DRA allocation, routing or + # scheduling without changing any field compared below. Only normal + # Kubernetes defaults are accepted; other overlays need new human review. + need(ps.get('runtimeClassName') in (None, '') + and not ps.get('overhead') and not ps.get('resourceClaims') + and ps.get('priorityClassName') in (None, ''), + 'unapproved Pod runtime or resource allocation') + need(not ps.get('nodeSelector') and not ps.get('affinity') + and not ps.get('topologySpreadConstraints') + and not ps.get('dnsConfig') + and ps.get('dnsPolicy', 'ClusterFirst') == 'ClusterFirst', + 'unapproved Pod DNS or placement policy') + need(ps.get('serviceAccount') in (None, 't3-worker') + and ps.get('terminationGracePeriodSeconds', 30) == + template.get('terminationGracePeriodSeconds', 30), + 'Pod service account or termination policy drift') for field in ('initContainers', 'ephemeralContainers', 'imagePullSecrets', 'hostAliases'): need(not ps.get(field), 'unexpected Pod configuration') containers = ps.get('containers') From ffc575882f199436928a6f1ec7d194218cc6feee Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:15:11 -0500 Subject: [PATCH 69/84] fix(k3s-p1): reject unapproved Pod admission network and runtime overlays --- examples/k3s-p1/test_pod_check.py | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/examples/k3s-p1/test_pod_check.py b/examples/k3s-p1/test_pod_check.py index f6a9809c4d0dc..b69d0411c192b 100644 --- a/examples/k3s-p1/test_pod_check.py +++ b/examples/k3s-p1/test_pod_check.py @@ -162,6 +162,41 @@ def test_reject_replacement_metadata_or_selector_drift(self): with self.subTest(change=change), self.assertRaises(pc.PodError): pc.inspect(*args) + def test_reject_pod_admission_network_and_runtime_overlays(self): + # A Pod can gain a Multus interface without an altered Pod spec or + # replacement ReplicaSet template, and a RuntimeClass can change GPU + # access. Neither is in the approved one-worker P1 plan. + cases = [ + (('metadata', 'annotations'), {'k8s.v1.cni.cncf.io/networks': 'extra-net'}), + (('metadata', 'annotations'), {'v1.multus-cni.io/default-network': 'extra-net'}), + (('spec', 'runtimeClassName'), 'nvidia'), + (('spec', 'overhead'), {'cpu': '100m'}), + (('spec', 'resourceClaims'), [{'name': 'gpu'}]), + (('spec', 'priorityClassName'), 'system-node-critical'), + (('spec', 'nodeSelector'), {'gpu': 'true'}), + (('spec', 'affinity'), {'nodeAffinity': {'requiredDuringSchedulingIgnoredDuringExecution': {}}}), + (('spec', 'topologySpreadConstraints'), [{'maxSkew': 1}]), + (('spec', 'dnsConfig'), {'nameservers': ['8.8.8.8']}), + (('spec', 'dnsPolicy'), 'None'), + (('spec', 'serviceAccount'), 'default'), + (('spec', 'terminationGracePeriodSeconds'), 1), + ] + for path, value in cases: + args = fixture() + obj = args[4]['items'][0] + for key in path[:-1]: + obj = obj[key] + obj[path[-1]] = value + with self.subTest(path=path), self.assertRaises(pc.PodError): + pc.inspect(*args) + args = fixture() + pod = args[4]['items'][0] + pod['metadata']['annotations'] = {} + pod['spec']['dnsPolicy'] = 'ClusterFirst' + pod['spec']['serviceAccount'] = 't3-worker' + pod['spec']['terminationGracePeriodSeconds'] = 30 + pc.inspect(*args) # Empty annotations and Kubernetes defaults remain valid. + def test_reject_false_readiness_and_concurrent_pods(self): cases = [ (2, ('status', 'observedGeneration'), 1), From e1abd2418fd817ce6713abf8a6ac58df5cc89aaa Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:15:15 -0500 Subject: [PATCH 70/84] fix(k3s-p1): reject unapproved Pod admission network and runtime overlays --- examples/k3s-p1/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index ed25cc8ace45d..05528ab882117 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -33,7 +33,7 @@ non-root Pod/container security, resources and probes, and rejects host access, additional containers and injected credentials/command overrides, not the image of a live Pod. Legacy v1/v2/v3 baselines are rejected. It never contacts a cluster or proves a live T3 worker. -After the offline desired Deployment/PVC binding, the optional [Pod snapshot checker](OBSERVATION.md#optional-read-only-pod-runtime-snapshot--not-live-proof) can validate saved ReplicaSet→Pod controller UID, API-reported readiness and running image reference, and reject concurrent/mutated Pods, unexpected host ports, changed startup/probe behavior or injected post-start hooks. It also compares the active ReplicaSet's replacement-Pod spec, template labels/annotations and selector with the approved Deployment, rejecting injection or routing drift that could affect a future Pod after restart/eviction. This is still **NOT live K3s proof**: its hashed imageID does not attest a multi-platform OCI digest, and it cannot establish persisted T3 identity, TLS, peer consent or successful delegation. +After the offline desired Deployment/PVC binding, the optional [Pod snapshot checker](OBSERVATION.md#optional-read-only-pod-runtime-snapshot--not-live-proof) can validate saved ReplicaSet→Pod controller UID, API-reported readiness and running image reference, and reject concurrent/mutated Pods, unexpected host ports, changed startup/probe behavior or injected post-start hooks. It also compares the active ReplicaSet's replacement-Pod spec, template labels/annotations and selector with the approved Deployment, rejecting injection or routing drift that could affect a future Pod after restart/eviction. This check also rejects unapproved **Pod-level annotations** (including Multus secondary-network attachments), injected runtime classes / GPU resource claims, changed DNS/placement controls, and mismatched termination/service-account fields while permitting normal Kubernetes defaults. If the cluster injects benign annotations, review them explicitly before changing the strict contract; never silently allow all annotations. This is still **NOT live K3s proof**: its hashed imageID does not attest a multi-platform OCI digest, and it cannot establish persisted T3 identity, TLS, peer consent or successful delegation. The local private mode-0600 receipt is create-once and atomically published only after a synced write. Concurrent identical requests reuse the complete receipt; a failed publish leaves no half-written final record. A retry with identical arguments renders identical YAML. Reusing the key with changed image/expiry From 7d0bf74e6c879185dab326ea3a7533c6b66ff9f5 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:22:10 -0500 Subject: [PATCH 71/84] feat(k3s-p1): capture read-only consistent private API snapshots --- examples/k3s-p1/capture.py | 157 +++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 examples/k3s-p1/capture.py diff --git a/examples/k3s-p1/capture.py b/examples/k3s-p1/capture.py new file mode 100644 index 0000000000000..bf24d97742d9b --- /dev/null +++ b/examples/k3s-p1/capture.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Read-only, fail-closed P1 Kubernetes snapshot capture; never applies resources. + +Human-approved disposable K3s context required. Snapshots are not live +attestation or peer authorization. Private local output, not a lease. +""" + +import argparse +import json +import os +from pathlib import Path +import re +import shutil +import stat +import subprocess +import tempfile + + +class CaptureError(RuntimeError): + """Cannot safely capture a coherent P1 API snapshot.""" + + +NAMES = ( + ('namespace', 'namespace', None), + ('pvc', 'pvc', 't3-worker-data'), + ('deployment', 'deployment', 't3-worker'), + ('service', 'service', 't3-worker'), + ('ingress', 'ingress', 't3-worker'), + ('policy', 'networkpolicy', 'worker-ingress'), +) +LISTS = ( + ('all-policies', 'networkpolicies', 'NetworkPolicyList'), + ('all-ingresses', 'ingresses', 'IngressList'), + ('all-services', 'services', 'ServiceList'), + ('replicasets', 'replicasets', 'ReplicaSetList'), + ('pods', 'pods', 'PodList'), +) + + +def _read(context, namespace, resource, name, runner): + command = ['kubectl', '--context', context, '--request-timeout=10s'] + if resource != 'namespace': + command += ['-n', namespace] + command += ['get', resource] + if name: + command.append(name) + elif resource == 'namespace': + command.append(namespace) + command += ['-o', 'json'] + try: + response = runner(command, capture_output=True, text=True, timeout=20, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + raise CaptureError('read-only Kubernetes API request unavailable') from exc + if response.returncode: + raise CaptureError('read-only Kubernetes API request rejected') + try: + item = json.loads(response.stdout) + except (ValueError, TypeError) as exc: + raise CaptureError('malformed Kubernetes JSON') from exc + if not isinstance(item, dict): + raise CaptureError('unexpected Kubernetes API response') + return item + + +def capture(context, plan, runner=subprocess.run): + """Collect named objects and inventories, rejecting concurrent changes. + + Re-reading cannot make Kubernetes's multi-call snapshot atomic. Recheck + context and resource identities independently before trusting any result. + """ + if (not isinstance(context, str) or not context or context != context.strip() + or context.startswith('-') or any(c in context for c in '\r\n\x00')): + raise CaptureError('explicit kubectl context required') + if not isinstance(plan, dict) or plan.get('format') != 't3-k3s-p1-plan-v1': + raise CaptureError('invalid immutable plan') + lease = plan.get('leaseHash') + if not isinstance(lease, str) or not re.fullmatch('[0-9a-f]{64}', lease): + raise CaptureError('invalid lease hash') + namespace = 't3-p1-' + lease[:24] + if plan.get('namespace') != namespace: + raise CaptureError('namespace differs from lease plan') + entries = {} + for key, resource, name in NAMES: + item = _read(context, namespace, resource, name, runner) + meta = item.get('metadata') + if not isinstance(meta, dict) or meta.get('namespace', namespace) != namespace: + raise CaptureError('named resource belongs to a different namespace') + entries[key] = item + for key, resource, kind in LISTS: + item = _read(context, namespace, resource, None, runner) + if item.get('kind') != kind or not isinstance(item.get('items'), list): + raise CaptureError('incomplete namespace inventory') + if any(not isinstance(obj, dict) or not isinstance(obj.get('metadata'), dict) + or obj['metadata'].get('namespace') != namespace for obj in item['items']): + raise CaptureError('inventory contains a foreign or malformed object') + entries[key] = item + # Read everything again: reject mid-capture mutation of named endpoints, + # Deployments, Pods, ReplicaSets or extra namespace routes/policies. + for key, resource, name in NAMES: + if entries[key] != _read(context, namespace, resource, name, runner): + raise CaptureError('named resource changed during capture') + for key, resource, kind in LISTS: + if entries[key] != _read(context, namespace, resource, None, runner): + raise CaptureError('namespace inventory changed during capture') + return entries + + +def publish(entries, output): + """Atomically expose a new private directory only after all files sync.""" + if not output.is_absolute() or output.exists() or output.is_symlink(): + raise CaptureError('output must be a new absolute directory') + parent = output.parent + try: + parent_fd = os.open(parent, os.O_RDONLY | os.O_DIRECTORY | getattr(os, 'O_NOFOLLOW', 0)) + except OSError as exc: + raise CaptureError('private output parent unavailable') from exc + try: + meta = os.fstat(parent_fd) + if not stat.S_ISDIR(meta.st_mode) or meta.st_uid != os.getuid() or meta.st_mode & 0o077: + raise CaptureError('output parent must be caller-owned and private (0700)') + temp = Path(tempfile.mkdtemp(prefix='.p1-capture-', dir=parent)) + try: + for key, item in entries.items(): + path = temp / (key + '.json') + with path.open('x', encoding='utf-8') as file: + os.chmod(path, 0o600) + json.dump(item, file, sort_keys=True) + file.write('\n') + file.flush() + os.fsync(file.fileno()) + os.rename(temp, output) + os.fsync(parent_fd) + finally: + if temp.exists(): + shutil.rmtree(temp) + finally: + os.close(parent_fd) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--context', required=True) + parser.add_argument('--plan', required=True, type=Path) + parser.add_argument('--output-dir', required=True, type=Path) + args = parser.parse_args() + try: + plan = json.loads(args.plan.read_text(encoding='utf-8')) + entries = capture(args.context, plan) + publish(entries, args.output_dir) + except (OSError, ValueError, TypeError, AttributeError, CaptureError) as exc: + parser.exit(1, f'FAIL: P1 read-only snapshot capture ({type(exc).__name__})\n') + print('PASS: 11 API JSON snapshots captured to private directory (read-only Kubernetes calls)') + print('NOT RUN: live integrity attestation, HTTPS, OAuth, linked delegation, cancellation, TTL') + + +if __name__ == '__main__': + main() From 494936a025c8245900dba2b656ac713bedba8584 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:22:28 -0500 Subject: [PATCH 72/84] test(k3s-p1): prove read-only snapshot capture and race rejection --- examples/k3s-p1/test_capture.py | 93 +++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 examples/k3s-p1/test_capture.py diff --git a/examples/k3s-p1/test_capture.py b/examples/k3s-p1/test_capture.py new file mode 100644 index 0000000000000..c8bef4ffb5e26 --- /dev/null +++ b/examples/k3s-p1/test_capture.py @@ -0,0 +1,93 @@ +"""Local-only P1 capture tests: no kubectl or cluster is contacted.""" +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location('capture', Path(__file__).with_name('capture.py')) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + +HASH = 'a' * 64 +NAMESPACE = 't3-p1-' + HASH[:24] +PLAN = {'format': 't3-k3s-p1-plan-v1', 'leaseHash': HASH, 'namespace': NAMESPACE} + + +class Stub: + def __init__(self, drift=None, broken=None): + self.calls = [] + self.drift = drift + self.broken = broken + + def __call__(self, command, **kwargs): + self.calls.append(command) + assert command[:4] == ['kubectl', '--context', 'disposable-k3s', '--request-timeout=10s'] + assert command[-2:] == ['-o', 'json'] + assert kwargs['timeout'] == 20 and kwargs['check'] is False + resource = command[command.index('get') + 1] + remainder = command[command.index('get') + 2:-2] + name = remainder[0] if remainder else None + if self.broken == resource: + return subprocess.CompletedProcess(command, 1, '', 'private kubeconfig detail') + if resource in ('networkpolicies', 'ingresses', 'services', 'replicasets', 'pods'): + obj = {'kind': dict(networkpolicies='NetworkPolicyList', ingresses='IngressList', + services='ServiceList', replicasets='ReplicaSetList', + pods='PodList')[resource], 'items': []} + else: + obj = {'kind': resource, 'metadata': {'name': name, 'namespace': NAMESPACE, + 'uid': 'a1234567', 'resourceVersion': '2'}, + 'spec': {'replicas': 1}} + if self.drift == resource and len([c for c in self.calls if resource in c]) > 1: + obj['metadata'] = {'resourceVersion': '3'} + return subprocess.CompletedProcess(command, 0, json.dumps(obj), '') + + +class CaptureTests(unittest.TestCase): + def test_read_only_explicit_context_and_stable_snapshot(self): + stub = Stub() + records = module.capture('disposable-k3s', PLAN, runner=stub) + self.assertEqual(len(records), 11) + self.assertEqual(len(stub.calls), 22) + self.assertTrue(all('get' in c and 'apply' not in c and 'delete' not in c + and 'patch' not in c for c in stub.calls)) + self.assertTrue(all('--context' in c for c in stub.calls)) + + def test_reject_stale_or_unsafe_plan(self): + for ctx in ('', '-bad', 'disposable\ncluster', ' spaced '): + with self.subTest(ctx=ctx), self.assertRaises(module.CaptureError): + module.capture(ctx, PLAN, runner=Stub()) + for altered in ({}, {**PLAN, 'namespace': 'existing-home'}, + {**PLAN, 'leaseHash': 'bad'}): + with self.subTest(plan=altered), self.assertRaises(module.CaptureError): + module.capture('disposable-k3s', altered, runner=Stub()) + + def test_detect_change_in_named_resource_or_inventory(self): + for res in ('deployment', 'ingresses', 'pods'): + with self.subTest(resource=res), self.assertRaises(module.CaptureError): + module.capture('disposable-k3s', PLAN, runner=Stub(drift=res)) + + def test_redacted_rejected_api_read(self): + with self.assertRaisesRegex(module.CaptureError, 'request rejected') as ctx: + module.capture('disposable-k3s', PLAN, runner=Stub(broken='pods')) + self.assertNotIn('private', str(ctx.exception)) + + def test_publish_private_atomic_and_no_overwrite(self): + with tempfile.TemporaryDirectory() as temp: + parent = Path(temp) / 'owner' + parent.mkdir(mode=0o700) + target = parent / 'snapshot' + module.publish({'pods': {'kind': 'PodList', 'items': []}}, target) + self.assertEqual(json.loads((target / 'pods.json').read_text())['kind'], 'PodList') + self.assertEqual((target / 'pods.json').stat().st_mode & 0o777, 0o600) + with self.assertRaises(module.CaptureError): + module.publish({'pods': {}}, target) + os.chmod(parent, 0o755) + with self.assertRaises(module.CaptureError): + module.publish({'pods': {}}, parent / 'another') + + +if __name__ == '__main__': + unittest.main() From 5aeb4d5c8b84f0fc5a4f8a9e6983d17a98031a2c Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:22:55 -0500 Subject: [PATCH 73/84] docs(k3s-p1): add safe snapshot capture acceptance steps and main seam qualification --- examples/k3s-p1/README.md | 55 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index 05528ab882117..f8ad29d12bfa7 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -155,3 +155,58 @@ approved-context read and independently inspect effective CNI rules. Also check actual DNS, HTTPS certificate, peer identity, private ACL negative test and CNI policy enforcement independently; do not treat these API objects as proof of the data plane. + + +## Optional read-only, revision-checked snapshot collection + +**NOT RUN on any cluster.** Only after a human has separately approved a +*disposable* P1 K3s worker and its exact context, the new \`capture.py\` +reduces mistakes in the manual observation commands above. It uses only +explicit-context \`kubectl get -o json\` calls with deadlines; it cannot +provision, mutate, link, cancel or delete a worker. It collects six named +resources (Namespace, PVC, Deployment, Service, Ingress, NetworkPolicy) and +five complete namespace inventories (policies, ingresses, services, +ReplicaSets, Pods). Each is read twice and mismatches fail closed **before +any output is published**. This catches a subset of mid-capture drift, +but Kubernetes offers **no atomic cross-resource snapshot** here; a +matching pair is not freshness, route integrity or runtime attestation. + +Create a new path inside a private, caller-owned 0700 directory. +Snapshot JSON may contain private addresses, labels, topology and paths, +so **never commit or share the output**. + +\`\`\`sh +umask 077 +mkdir -m 700 /tmp/t3-p1-captures +python3 examples/k3s-p1/capture.py \ + --context YOUR_APPROVED_DISPOSABLE_CONTEXT \ + --plan /tmp/k3s-p1-receipts/YOUR_FULL_LEASE_SHA256.json \ + --output-dir /tmp/t3-p1-captures/first +\`\`\` + +The command refuses existing destination paths, non-private parents, +invalid plans or inconsistent repeated reads. It writes each JSON as 0600 +and atomically publishes the directory after all writes are synced. Run +existing \`observe.py\`, \`pod_check.py\` and \`route_check.py\` on these +**saved files**, using \`first/namespace.json\`, \`first/pvc.json\`, +\`first/deployment.json\`, \`first/service.json\`, +\`first/ingress.json\`, \`first/policy.json\`, +\`first/all-policies.json\`, \`first/all-ingresses.json\`, +\`first/all-services.json\`, \`first/replicasets.json\`, and +\`first/pods.json\`. Never overwrite a previously accepted baseline. +Repeat capture into a *new* private directory when observing recovery. + +For credential-free development tests (does **not** contact kubectl): + +\`\`\`sh +python3 -m unittest discover -s examples/k3s-p1 -p 'test_capture.py' -v +python3 -m py_compile examples/k3s-p1/capture.py examples/k3s-p1/test_capture.py +\`\`\` + +Upstream \`main@08a463e29976617134de01e9341f898bc881d0eb\` (2026-10-29) +documents linked-environment \`delegate_task\` and permissioned peer linking +in \`docs/user/remote-access.md\` and +\`docs/internals/remote.md\`; this is source-level availability on +\`main\`, **not** verified compatibility of the P0 \`v0.1.3\` image. +Upstream record-before-launch issue #17051 remains open. Human-approved +live P0 persistence/route/client proof and P1 OAuth/delegation are still HOLD. From 20f8df92a4cb419b22fbc7b6d019c4cf855e85a6 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:23:59 -0500 Subject: [PATCH 74/84] docs(k3s-p1): render runnable capture commands as code --- examples/k3s-p1/README.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index f8ad29d12bfa7..acf95211ac5bf 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -160,9 +160,9 @@ proof of the data plane. ## Optional read-only, revision-checked snapshot collection **NOT RUN on any cluster.** Only after a human has separately approved a -*disposable* P1 K3s worker and its exact context, the new \`capture.py\` +*disposable* P1 K3s worker and its exact context, the new `capture.py` reduces mistakes in the manual observation commands above. It uses only -explicit-context \`kubectl get -o json\` calls with deadlines; it cannot +explicit-context `kubectl get -o json` calls with deadlines; it cannot provision, mutate, link, cancel or delete a worker. It collects six named resources (Namespace, PVC, Deployment, Service, Ingress, NetworkPolicy) and five complete namespace inventories (policies, ingresses, services, @@ -175,38 +175,38 @@ Create a new path inside a private, caller-owned 0700 directory. Snapshot JSON may contain private addresses, labels, topology and paths, so **never commit or share the output**. -\`\`\`sh +```sh umask 077 mkdir -m 700 /tmp/t3-p1-captures python3 examples/k3s-p1/capture.py \ --context YOUR_APPROVED_DISPOSABLE_CONTEXT \ --plan /tmp/k3s-p1-receipts/YOUR_FULL_LEASE_SHA256.json \ --output-dir /tmp/t3-p1-captures/first -\`\`\` +``` The command refuses existing destination paths, non-private parents, invalid plans or inconsistent repeated reads. It writes each JSON as 0600 and atomically publishes the directory after all writes are synced. Run -existing \`observe.py\`, \`pod_check.py\` and \`route_check.py\` on these -**saved files**, using \`first/namespace.json\`, \`first/pvc.json\`, -\`first/deployment.json\`, \`first/service.json\`, -\`first/ingress.json\`, \`first/policy.json\`, -\`first/all-policies.json\`, \`first/all-ingresses.json\`, -\`first/all-services.json\`, \`first/replicasets.json\`, and -\`first/pods.json\`. Never overwrite a previously accepted baseline. +existing `observe.py`, `pod_check.py` and `route_check.py` on these +**saved files**, using `first/namespace.json`, `first/pvc.json`, +`first/deployment.json`, `first/service.json`, +`first/ingress.json`, `first/policy.json`, +`first/all-policies.json`, `first/all-ingresses.json`, +`first/all-services.json`, `first/replicasets.json`, and +`first/pods.json`. Never overwrite a previously accepted baseline. Repeat capture into a *new* private directory when observing recovery. For credential-free development tests (does **not** contact kubectl): -\`\`\`sh +```sh python3 -m unittest discover -s examples/k3s-p1 -p 'test_capture.py' -v python3 -m py_compile examples/k3s-p1/capture.py examples/k3s-p1/test_capture.py -\`\`\` +``` -Upstream \`main@08a463e29976617134de01e9341f898bc881d0eb\` (2026-10-29) -documents linked-environment \`delegate_task\` and permissioned peer linking -in \`docs/user/remote-access.md\` and -\`docs/internals/remote.md\`; this is source-level availability on -\`main\`, **not** verified compatibility of the P0 \`v0.1.3\` image. +Upstream `main@08a463e29976617134de01e9341f898bc881d0eb` (2026-10-29) +documents linked-environment `delegate_task` and permissioned peer linking +in `docs/user/remote-access.md` and +`docs/internals/remote.md`; this is source-level availability on +`main`, **not** verified compatibility of the P0 `v0.1.3` image. Upstream record-before-launch issue #17051 remains open. Human-approved live P0 persistence/route/client proof and P1 OAuth/delegation are still HOLD. From eed68fc54e01119e708f19ed2f2398f93e00fe4a Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:12:54 -0500 Subject: [PATCH 75/84] fix(k3s-p1): bind named snapshot identities and namespace inventory --- examples/k3s-p1/capture.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/examples/k3s-p1/capture.py b/examples/k3s-p1/capture.py index bf24d97742d9b..6bb0d5ebcb922 100644 --- a/examples/k3s-p1/capture.py +++ b/examples/k3s-p1/capture.py @@ -36,6 +36,15 @@ class CaptureError(RuntimeError): ('pods', 'pods', 'PodList'), ) +NAMED_KINDS = { + 'namespace': 'Namespace', + 'pvc': 'PersistentVolumeClaim', + 'deployment': 'Deployment', + 'service': 'Service', + 'ingress': 'Ingress', + 'policy': 'NetworkPolicy', +} + def _read(context, namespace, resource, name, runner): command = ['kubectl', '--context', context, '--request-timeout=10s'] @@ -83,7 +92,16 @@ def capture(context, plan, runner=subprocess.run): for key, resource, name in NAMES: item = _read(context, namespace, resource, name, runner) meta = item.get('metadata') - if not isinstance(meta, dict) or meta.get('namespace', namespace) != namespace: + expected_name = namespace if resource == 'namespace' else name + if (item.get('kind') != NAMED_KINDS[key] + or not isinstance(meta, dict) + or meta.get('name') != expected_name + or not isinstance(meta.get('uid'), str) or not meta['uid']): + raise CaptureError('named Kubernetes identity does not match the lease') + if resource == 'namespace': + if meta.get('namespace') not in (None, ''): + raise CaptureError('Namespace carries a foreign namespace') + elif meta.get('namespace') != namespace: raise CaptureError('named resource belongs to a different namespace') entries[key] = item for key, resource, kind in LISTS: @@ -94,6 +112,17 @@ def capture(context, plan, runner=subprocess.run): or obj['metadata'].get('namespace') != namespace for obj in item['items']): raise CaptureError('inventory contains a foreign or malformed object') entries[key] = item + # Separate named GET and LIST requests can observe different objects even + # when each individual request remains stable across both reads. Reject + # mixed snapshots before publishing a seemingly coherent private receipt. + for key, inventory in (('policy', 'all-policies'), + ('ingress', 'all-ingresses'), + ('service', 'all-services')): + named = entries[key] + matches = [obj for obj in entries[inventory]['items'] + if obj['metadata'].get('name') == named['metadata']['name']] + if len(matches) != 1 or matches[0] != named: + raise CaptureError('named route or policy disagrees with namespace inventory') # Read everything again: reject mid-capture mutation of named endpoints, # Deployments, Pods, ReplicaSets or extra namespace routes/policies. for key, resource, name in NAMES: From d153e893f2522cb21e0a8b7fd3b49f261f8d8957 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:12:57 -0500 Subject: [PATCH 76/84] test(k3s-p1): reject mixed GET/LIST and foreign resource identities --- examples/k3s-p1/test_capture.py | 51 +++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/examples/k3s-p1/test_capture.py b/examples/k3s-p1/test_capture.py index c8bef4ffb5e26..b552f5349c18f 100644 --- a/examples/k3s-p1/test_capture.py +++ b/examples/k3s-p1/test_capture.py @@ -17,10 +17,11 @@ class Stub: - def __init__(self, drift=None, broken=None): + def __init__(self, drift=None, broken=None, tamper=None): self.calls = [] self.drift = drift self.broken = broken + self.tamper = tamper def __call__(self, command, **kwargs): self.calls.append(command) @@ -32,14 +33,42 @@ def __call__(self, command, **kwargs): name = remainder[0] if remainder else None if self.broken == resource: return subprocess.CompletedProcess(command, 1, '', 'private kubeconfig detail') + kinds = {'namespace': 'Namespace', 'pvc': 'PersistentVolumeClaim', + 'deployment': 'Deployment', 'service': 'Service', + 'ingress': 'Ingress', 'networkpolicy': 'NetworkPolicy'} + listed_names = {'networkpolicies': ('networkpolicy', 'worker-ingress'), + 'ingresses': ('ingress', 't3-worker'), + 'services': ('service', 't3-worker')} if resource in ('networkpolicies', 'ingresses', 'services', 'replicasets', 'pods'): + items = [] + if resource in listed_names: + source, selected_name = listed_names[resource] + items = [{'kind': kinds[source], + 'metadata': {'name': selected_name, 'namespace': NAMESPACE, + 'uid': 'a1234567', 'resourceVersion': '2'}, + 'spec': {'replicas': 1}}] obj = {'kind': dict(networkpolicies='NetworkPolicyList', ingresses='IngressList', services='ServiceList', replicasets='ReplicaSetList', - pods='PodList')[resource], 'items': []} + pods='PodList')[resource], 'items': items} else: - obj = {'kind': resource, 'metadata': {'name': name, 'namespace': NAMESPACE, - 'uid': 'a1234567', 'resourceVersion': '2'}, - 'spec': {'replicas': 1}} + meta = {'name': name, 'uid': 'a1234567', 'resourceVersion': '2'} + if resource != 'namespace': + meta['namespace'] = NAMESPACE + obj = {'kind': kinds[resource], 'metadata': meta, 'spec': {'replicas': 1}} + if self.tamper is not None and self.tamper[0] == resource: + flaw = self.tamper[1] + if flaw == 'kind': + obj['kind'] = 'Other' + elif flaw == 'name': + obj['metadata']['name'] = 'unrelated' + elif flaw == 'namespace': + obj['metadata'].pop('namespace', None) + elif flaw == 'uid': + obj['metadata']['uid'] = None + elif flaw == 'inventory_missing': + obj['items'] = [] + elif flaw == 'inventory_drift': + obj['items'][0]['spec']['replicas'] = 2 if self.drift == resource and len([c for c in self.calls if resource in c]) > 1: obj['metadata'] = {'resourceVersion': '3'} return subprocess.CompletedProcess(command, 0, json.dumps(obj), '') @@ -69,6 +98,18 @@ def test_detect_change_in_named_resource_or_inventory(self): with self.subTest(resource=res), self.assertRaises(module.CaptureError): module.capture('disposable-k3s', PLAN, runner=Stub(drift=res)) + def test_named_identity_and_mixed_inventory_fail_closed(self): + cases = ( + ('deployment', 'kind'), ('deployment', 'name'), + ('deployment', 'namespace'), ('deployment', 'uid'), + ('namespace', 'name'), ('services', 'inventory_missing'), + ('services', 'inventory_drift'), ('ingresses', 'inventory_missing'), + ('networkpolicies', 'inventory_drift'), + ) + for case in cases: + with self.subTest(mutation=case), self.assertRaises(module.CaptureError): + module.capture('disposable-k3s', PLAN, runner=Stub(tamper=case)) + def test_redacted_rejected_api_read(self): with self.assertRaisesRegex(module.CaptureError, 'request rejected') as ctx: module.capture('disposable-k3s', PLAN, runner=Stub(broken='pods')) From 721fcb3ae953c1154e69400f8b67f8138b6dc8a0 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:14:45 -0500 Subject: [PATCH 77/84] docs(k3s-p1): explain fail-closed mixed GET/LIST snapshot checks --- examples/k3s-p1/README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index acf95211ac5bf..928e9a9ad297d 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -167,9 +167,13 @@ provision, mutate, link, cancel or delete a worker. It collects six named resources (Namespace, PVC, Deployment, Service, Ingress, NetworkPolicy) and five complete namespace inventories (policies, ingresses, services, ReplicaSets, Pods). Each is read twice and mismatches fail closed **before -any output is published**. This catches a subset of mid-capture drift, -but Kubernetes offers **no atomic cross-resource snapshot** here; a -matching pair is not freshness, route integrity or runtime attestation. +any output is published**. Named resources must have the planned Kubernetes +kind/name/namespace and a UID; the named Service, Ingress and NetworkPolicy +must also exactly match their corresponding namespace-inventory entry. +This rejects mixed GET/LIST views even when each view is independently stable. +It catches only a subset of mid-capture drift: Kubernetes offers **no atomic +cross-resource snapshot** here, and matching objects do not prove freshness, +route integrity or runtime attestation. Create a new path inside a private, caller-owned 0700 directory. Snapshot JSON may contain private addresses, labels, topology and paths, From 37991a886f64ed8c36578f43cca107ecb12378c4 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:12:01 -0500 Subject: [PATCH 78/84] fix(k3s-p1): publish private snapshots without replacing racing writer --- examples/k3s-p1/capture.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/examples/k3s-p1/capture.py b/examples/k3s-p1/capture.py index 6bb0d5ebcb922..b56a65226b1ee 100644 --- a/examples/k3s-p1/capture.py +++ b/examples/k3s-p1/capture.py @@ -6,6 +6,8 @@ """ import argparse +import ctypes +import errno import json import os from pathlib import Path @@ -134,6 +136,25 @@ def capture(context, plan, runner=subprocess.run): return entries + +def _rename_directory_exclusive(source: Path, target: Path, parent_fd: int): + """Publish a directory atomically without clobbering another writer. + + This is a Linux-only acceptance helper. Without renameat2's + RENAME_NOREPLACE support, fail closed rather than overwriting evidence. + """ + renameat2 = getattr(ctypes.CDLL(None, use_errno=True), 'renameat2', None) + if renameat2 is None: + raise CaptureError('atomic no-replace directory publication unavailable') + renameat2.argtypes = (ctypes.c_int, ctypes.c_char_p, + ctypes.c_int, ctypes.c_char_p, ctypes.c_uint) + renameat2.restype = ctypes.c_int + if renameat2(parent_fd, os.fsencode(source.name), + parent_fd, os.fsencode(target.name), 1): # RENAME_NOREPLACE + if ctypes.get_errno() == errno.EEXIST: + raise CaptureError('output directory already exists') + raise CaptureError('atomic no-replace directory publication unavailable') + def publish(entries, output): """Atomically expose a new private directory only after all files sync.""" if not output.is_absolute() or output.exists() or output.is_symlink(): @@ -157,7 +178,7 @@ def publish(entries, output): file.write('\n') file.flush() os.fsync(file.fileno()) - os.rename(temp, output) + _rename_directory_exclusive(temp, output, parent_fd) os.fsync(parent_fd) finally: if temp.exists(): From 377df30a11e3d9eb94d0f4039fdab69a14a477b7 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:12:14 -0500 Subject: [PATCH 79/84] test(k3s-p1): preserve racing empty snapshot destination --- examples/k3s-p1/test_capture.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/examples/k3s-p1/test_capture.py b/examples/k3s-p1/test_capture.py index b552f5349c18f..f4e69022d4b77 100644 --- a/examples/k3s-p1/test_capture.py +++ b/examples/k3s-p1/test_capture.py @@ -6,6 +6,7 @@ import subprocess import tempfile import unittest +from unittest import mock spec = importlib.util.spec_from_file_location('capture', Path(__file__).with_name('capture.py')) module = importlib.util.module_from_spec(spec) @@ -115,6 +116,29 @@ def test_redacted_rejected_api_read(self): module.capture('disposable-k3s', PLAN, runner=Stub(broken='pods')) self.assertNotIn('private', str(ctx.exception)) + def test_concurrent_empty_destination_cannot_replace_existing_evidence(self): + # Reproduce a writer creating the final directory after publish()'s + # initial absence check. Plain os.rename() would replace that empty + # directory; renameat2(RENAME_NOREPLACE) must preserve its inode. + with tempfile.TemporaryDirectory() as temp: + parent = Path(temp) / 'owner' + parent.mkdir(mode=0o700) + target = parent / 'snapshot' + make_temp = module.tempfile.mkdtemp + competing_inode = [] + + def race(*args, **kwargs): + staging = make_temp(*args, **kwargs) + target.mkdir() + competing_inode.append(target.stat().st_ino) + return staging + + with mock.patch.object(module.tempfile, 'mkdtemp', side_effect=race): + with self.assertRaises(module.CaptureError): + module.publish({'pods': {'kind': 'PodList', 'items': []}}, target) + self.assertEqual(target.stat().st_ino, competing_inode[0]) + self.assertEqual(list(parent.iterdir()), [target]) + def test_publish_private_atomic_and_no_overwrite(self): with tempfile.TemporaryDirectory() as temp: parent = Path(temp) / 'owner' From e2119e3f1a9573c54c4112265f5534dbfb342337 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:12:25 -0500 Subject: [PATCH 80/84] docs(k3s-p1): state Linux no-clobber snapshot requirement --- examples/k3s-p1/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index 928e9a9ad297d..fb210920f1dd5 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -190,7 +190,10 @@ python3 examples/k3s-p1/capture.py \ The command refuses existing destination paths, non-private parents, invalid plans or inconsistent repeated reads. It writes each JSON as 0600 -and atomically publishes the directory after all writes are synced. Run +and atomically publishes the directory after all writes are synced. Linux +`renameat2(RENAME_NOREPLACE)` prevents a racing writer's empty destination +directory from being replaced; unsupported kernels fail closed, not silently +overwrite a previous snapshot. Run existing `observe.py`, `pod_check.py` and `route_check.py` on these **saved files**, using `first/namespace.json`, `first/pvc.json`, `first/deployment.json`, `first/service.json`, From 6cc70c543f9a29e35aaf139dca6e6165e823b1ec Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:16:08 -0500 Subject: [PATCH 81/84] fix(k3s-p1): fsync capture directory before atomic publication --- examples/k3s-p1/capture.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/k3s-p1/capture.py b/examples/k3s-p1/capture.py index b56a65226b1ee..d3a1a6deaf28d 100644 --- a/examples/k3s-p1/capture.py +++ b/examples/k3s-p1/capture.py @@ -178,6 +178,13 @@ def publish(entries, output): file.write('\n') file.flush() os.fsync(file.fileno()) + # Sync staging directory entries before the parent rename. File + # fsync alone does not durably commit the file names inside it. + temp_fd = os.open(temp, os.O_RDONLY | os.O_DIRECTORY | getattr(os, 'O_NOFOLLOW', 0)) + try: + os.fsync(temp_fd) + finally: + os.close(temp_fd) _rename_directory_exclusive(temp, output, parent_fd) os.fsync(parent_fd) finally: From 3ce8a75f4ba2d3e4e846d19a3b184c36a8950540 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:16:23 -0500 Subject: [PATCH 82/84] test(k3s-p1): assert directory fsync ordering and fail-closed publication --- examples/k3s-p1/test_capture.py | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/examples/k3s-p1/test_capture.py b/examples/k3s-p1/test_capture.py index f4e69022d4b77..1c0fe7b411885 100644 --- a/examples/k3s-p1/test_capture.py +++ b/examples/k3s-p1/test_capture.py @@ -2,6 +2,7 @@ import importlib.util import json import os +import stat from pathlib import Path import subprocess import tempfile @@ -139,6 +140,46 @@ def race(*args, **kwargs): self.assertEqual(target.stat().st_ino, competing_inode[0]) self.assertEqual(list(parent.iterdir()), [target]) + def test_staging_directory_fsync_precedes_parent_publication(self): + # A file's fsync does not make its *directory entry* crash durable. + # The staging directory must be synced before renameat2 exposes it. + with tempfile.TemporaryDirectory() as temp: + parent = Path(temp) / 'owner' + parent.mkdir(mode=0o700) + target = parent / 'snapshot' + events = [] + original_fsync = os.fsync + + def record_fsync(fd): + events.append('directory' if stat.S_ISDIR(os.fstat(fd).st_mode) + else 'file') + return original_fsync(fd) + + with mock.patch.object(module.os, 'fsync', side_effect=record_fsync): + module.publish({'pods': {'kind': 'PodList', 'items': []}}, target) + self.assertEqual(events, ['file', 'directory', 'directory']) + self.assertEqual(json.loads((target / 'pods.json').read_text())['kind'], + 'PodList') + + def test_staging_directory_sync_failure_cannot_publish(self): + with tempfile.TemporaryDirectory() as temp: + parent = Path(temp) / 'owner' + parent.mkdir(mode=0o700) + target = parent / 'snapshot' + original_fsync = os.fsync + + def reject_directory_fsync(fd): + if stat.S_ISDIR(os.fstat(fd).st_mode): + raise OSError('injected staging-directory fsync failure') + return original_fsync(fd) + + with mock.patch.object(module.os, 'fsync', + side_effect=reject_directory_fsync): + with self.assertRaises(OSError): + module.publish({'pods': {'kind': 'PodList', 'items': []}}, target) + self.assertFalse(target.exists()) + self.assertEqual(list(parent.iterdir()), []) + def test_publish_private_atomic_and_no_overwrite(self): with tempfile.TemporaryDirectory() as temp: parent = Path(temp) / 'owner' From df875c4d3cb1f83052ebee03dbabb72b77f142fb Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:18:13 -0500 Subject: [PATCH 83/84] test(k3s-p1): include service account and quota in read-only capture --- examples/k3s-p1/README.md | 12 ++++++++---- examples/k3s-p1/capture.py | 6 +++++- examples/k3s-p1/test_capture.py | 19 +++++++++++++++---- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index fb210920f1dd5..4f862b1f5485c 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -163,14 +163,17 @@ proof of the data plane. *disposable* P1 K3s worker and its exact context, the new `capture.py` reduces mistakes in the manual observation commands above. It uses only explicit-context `kubectl get -o json` calls with deadlines; it cannot -provision, mutate, link, cancel or delete a worker. It collects six named -resources (Namespace, PVC, Deployment, Service, Ingress, NetworkPolicy) and -five complete namespace inventories (policies, ingresses, services, +provision, mutate, link, cancel or delete a worker. It collects eight named +resources (Namespace, ResourceQuota, ServiceAccount, PVC, Deployment, Service, +Ingress, NetworkPolicy) and five complete namespace inventories (policies, ingresses, services, ReplicaSets, Pods). Each is read twice and mismatches fail closed **before any output is published**. Named resources must have the planned Kubernetes kind/name/namespace and a UID; the named Service, Ingress and NetworkPolicy must also exactly match their corresponding namespace-inventory entry. This rejects mixed GET/LIST views even when each view is independently stable. +The extra quota and no-token ServiceAccount snapshots are for *subsequent* +human/offline security review; collection alone does **not** validate their +specifications or prove admission limits and token restrictions are enforced. It catches only a subset of mid-capture drift: Kubernetes offers **no atomic cross-resource snapshot** here, and matching objects do not prove freshness, route integrity or runtime attestation. @@ -195,7 +198,8 @@ and atomically publishes the directory after all writes are synced. Linux directory from being replaced; unsupported kernels fail closed, not silently overwrite a previous snapshot. Run existing `observe.py`, `pod_check.py` and `route_check.py` on these -**saved files**, using `first/namespace.json`, `first/pvc.json`, +**saved files**, using `first/namespace.json`, `first/quota.json`, +`first/serviceaccount.json`, `first/pvc.json`, `first/deployment.json`, `first/service.json`, `first/ingress.json`, `first/policy.json`, `first/all-policies.json`, `first/all-ingresses.json`, diff --git a/examples/k3s-p1/capture.py b/examples/k3s-p1/capture.py index d3a1a6deaf28d..4552563d6224e 100644 --- a/examples/k3s-p1/capture.py +++ b/examples/k3s-p1/capture.py @@ -24,6 +24,8 @@ class CaptureError(RuntimeError): NAMES = ( ('namespace', 'namespace', None), + ('quota', 'resourcequota', 'worker-quota'), + ('serviceaccount', 'serviceaccount', 't3-worker'), ('pvc', 'pvc', 't3-worker-data'), ('deployment', 'deployment', 't3-worker'), ('service', 'service', 't3-worker'), @@ -40,6 +42,8 @@ class CaptureError(RuntimeError): NAMED_KINDS = { 'namespace': 'Namespace', + 'quota': 'ResourceQuota', + 'serviceaccount': 'ServiceAccount', 'pvc': 'PersistentVolumeClaim', 'deployment': 'Deployment', 'service': 'Service', @@ -206,7 +210,7 @@ def main(): publish(entries, args.output_dir) except (OSError, ValueError, TypeError, AttributeError, CaptureError) as exc: parser.exit(1, f'FAIL: P1 read-only snapshot capture ({type(exc).__name__})\n') - print('PASS: 11 API JSON snapshots captured to private directory (read-only Kubernetes calls)') + print('PASS: 13 API JSON snapshots captured to private directory (read-only Kubernetes calls)') print('NOT RUN: live integrity attestation, HTTPS, OAuth, linked delegation, cancellation, TTL') diff --git a/examples/k3s-p1/test_capture.py b/examples/k3s-p1/test_capture.py index 1c0fe7b411885..2ff0843a9dc82 100644 --- a/examples/k3s-p1/test_capture.py +++ b/examples/k3s-p1/test_capture.py @@ -35,7 +35,8 @@ def __call__(self, command, **kwargs): name = remainder[0] if remainder else None if self.broken == resource: return subprocess.CompletedProcess(command, 1, '', 'private kubeconfig detail') - kinds = {'namespace': 'Namespace', 'pvc': 'PersistentVolumeClaim', + kinds = {'namespace': 'Namespace', 'resourcequota': 'ResourceQuota', + 'serviceaccount': 'ServiceAccount', 'pvc': 'PersistentVolumeClaim', 'deployment': 'Deployment', 'service': 'Service', 'ingress': 'Ingress', 'networkpolicy': 'NetworkPolicy'} listed_names = {'networkpolicies': ('networkpolicy', 'worker-ingress'), @@ -80,8 +81,10 @@ class CaptureTests(unittest.TestCase): def test_read_only_explicit_context_and_stable_snapshot(self): stub = Stub() records = module.capture('disposable-k3s', PLAN, runner=stub) - self.assertEqual(len(records), 11) - self.assertEqual(len(stub.calls), 22) + self.assertEqual(len(records), 13) + self.assertEqual(len(stub.calls), 26) + self.assertEqual(records['quota']['kind'], 'ResourceQuota') + self.assertEqual(records['serviceaccount']['kind'], 'ServiceAccount') self.assertTrue(all('get' in c and 'apply' not in c and 'delete' not in c and 'patch' not in c for c in stub.calls)) self.assertTrue(all('--context' in c for c in stub.calls)) @@ -96,7 +99,7 @@ def test_reject_stale_or_unsafe_plan(self): module.capture('disposable-k3s', altered, runner=Stub()) def test_detect_change_in_named_resource_or_inventory(self): - for res in ('deployment', 'ingresses', 'pods'): + for res in ('resourcequota', 'serviceaccount', 'deployment', 'ingresses', 'pods'): with self.subTest(resource=res), self.assertRaises(module.CaptureError): module.capture('disposable-k3s', PLAN, runner=Stub(drift=res)) @@ -104,6 +107,8 @@ def test_named_identity_and_mixed_inventory_fail_closed(self): cases = ( ('deployment', 'kind'), ('deployment', 'name'), ('deployment', 'namespace'), ('deployment', 'uid'), + ('resourcequota', 'kind'), ('resourcequota', 'uid'), + ('serviceaccount', 'name'), ('serviceaccount', 'namespace'), ('namespace', 'name'), ('services', 'inventory_missing'), ('services', 'inventory_drift'), ('ingresses', 'inventory_missing'), ('networkpolicies', 'inventory_drift'), @@ -112,6 +117,12 @@ def test_named_identity_and_mixed_inventory_fail_closed(self): with self.subTest(mutation=case), self.assertRaises(module.CaptureError): module.capture('disposable-k3s', PLAN, runner=Stub(tamper=case)) + def test_missing_quota_or_service_account_fails_closed(self): + for res in ('resourcequota', 'serviceaccount'): + with self.subTest(resource=res), self.assertRaisesRegex( + module.CaptureError, 'request rejected'): + module.capture('disposable-k3s', PLAN, runner=Stub(broken=res)) + def test_redacted_rejected_api_read(self): with self.assertRaisesRegex(module.CaptureError, 'request rejected') as ctx: module.capture('disposable-k3s', PLAN, runner=Stub(broken='pods')) From a5dffeb7a742963f75157688d6b2f6510f3720bb Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:15:00 -0500 Subject: [PATCH 84/84] fix(k3s-p1): reject scoped quotas and worker service account credential drift --- examples/k3s-p1/README.md | 10 +++++++--- examples/k3s-p1/capture.py | 35 +++++++++++++++++++++++++++++++++ examples/k3s-p1/test_capture.py | 35 +++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 3 deletions(-) diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md index 4f862b1f5485c..9eb979caf50f3 100644 --- a/examples/k3s-p1/README.md +++ b/examples/k3s-p1/README.md @@ -171,9 +171,13 @@ any output is published**. Named resources must have the planned Kubernetes kind/name/namespace and a UID; the named Service, Ingress and NetworkPolicy must also exactly match their corresponding namespace-inventory entry. This rejects mixed GET/LIST views even when each view is independently stable. -The extra quota and no-token ServiceAccount snapshots are for *subsequent* -human/offline security review; collection alone does **not** validate their -specifications or prove admission limits and token restrictions are enforced. +The capture now fail-closes if the named ResourceQuota has different +hard limits, any scopes/scopeSelector, or an unreconciled `status.hard`, +and if the named ServiceAccount is deleting, permits API-token automount, +or references token/pull secrets. Matching *snapshots* do **not** prove +Kubernetes quota enforcement, RBAC permissions or absence of admission +mutation; human approval, CNI/ACL negative tests and live verification remain +mandatory. It catches only a subset of mid-capture drift: Kubernetes offers **no atomic cross-resource snapshot** here, and matching objects do not prove freshness, route integrity or runtime attestation. diff --git a/examples/k3s-p1/capture.py b/examples/k3s-p1/capture.py index 4552563d6224e..74f7bac08ed07 100644 --- a/examples/k3s-p1/capture.py +++ b/examples/k3s-p1/capture.py @@ -52,6 +52,40 @@ class CaptureError(RuntimeError): } + +# Exact P1 manifest controls from plan.py. ResourceQuota scopes can exempt a +# worker from limits; named-object identity alone is not enough for review. +# These API objects are snapshots, not proof of quota/RBAC enforcement. +EXPECTED_QUOTA_HARD = { + 'pods': '1', 'persistentvolumeclaims': '1', + 'requests.cpu': '2', 'requests.memory': '4Gi', + 'limits.cpu': '4', 'limits.memory': '8Gi', + 'requests.storage': '20Gi', + 'requests.ephemeral-storage': '2Gi', + 'limits.ephemeral-storage': '4Gi', +} + + +def _validate_controls(entries): + quota = entries['quota'] + service_account = entries['serviceaccount'] + qmeta = quota['metadata'] + smeta = service_account['metadata'] + if qmeta.get('deletionTimestamp') or smeta.get('deletionTimestamp'): + raise CaptureError('worker control resource is deleting') + spec = quota.get('spec') + status = quota.get('status') + if (not isinstance(spec, dict) or set(spec) != {'hard'} + or spec.get('hard') != EXPECTED_QUOTA_HARD + or not isinstance(status, dict) + or status.get('hard') != EXPECTED_QUOTA_HARD): + raise CaptureError('worker quota scope, bounds, or reconciliation differs from plan') + if (service_account.get('automountServiceAccountToken') is not False + or service_account.get('secrets') + or service_account.get('imagePullSecrets')): + raise CaptureError('worker ServiceAccount credential policy differs from plan') + + def _read(context, namespace, resource, name, runner): command = ['kubectl', '--context', context, '--request-timeout=10s'] if resource != 'namespace': @@ -110,6 +144,7 @@ def capture(context, plan, runner=subprocess.run): elif meta.get('namespace') != namespace: raise CaptureError('named resource belongs to a different namespace') entries[key] = item + _validate_controls(entries) for key, resource, kind in LISTS: item = _read(context, namespace, resource, None, runner) if item.get('kind') != kind or not isinstance(item.get('items'), list): diff --git a/examples/k3s-p1/test_capture.py b/examples/k3s-p1/test_capture.py index 2ff0843a9dc82..defb690866476 100644 --- a/examples/k3s-p1/test_capture.py +++ b/examples/k3s-p1/test_capture.py @@ -58,6 +58,12 @@ def __call__(self, command, **kwargs): if resource != 'namespace': meta['namespace'] = NAMESPACE obj = {'kind': kinds[resource], 'metadata': meta, 'spec': {'replicas': 1}} + if resource == 'resourcequota': + obj['spec'] = {'hard': module.EXPECTED_QUOTA_HARD.copy()} + obj['status'] = {'hard': module.EXPECTED_QUOTA_HARD.copy()} + if resource == 'serviceaccount': + obj.pop('spec') + obj['automountServiceAccountToken'] = False if self.tamper is not None and self.tamper[0] == resource: flaw = self.tamper[1] if flaw == 'kind': @@ -72,6 +78,20 @@ def __call__(self, command, **kwargs): obj['items'] = [] elif flaw == 'inventory_drift': obj['items'][0]['spec']['replicas'] = 2 + elif flaw == 'quota_scope': + obj['spec']['scopeSelector'] = {'matchExpressions': []} + elif flaw == 'quota_hard': + obj['spec']['hard']['pods'] = '3' + elif flaw == 'quota_unreconciled': + obj.pop('status', None) + elif flaw == 'sa_automount': + obj['automountServiceAccountToken'] = True + elif flaw == 'sa_secret': + obj['secrets'] = [{'name': 'unapproved-token'}] + elif flaw == 'sa_pull_secret': + obj['imagePullSecrets'] = [{'name': 'unapproved-pull'}] + elif flaw == 'deleting': + obj['metadata']['deletionTimestamp'] = '2026-10-29T00:00:00Z' if self.drift == resource and len([c for c in self.calls if resource in c]) > 1: obj['metadata'] = {'resourceVersion': '3'} return subprocess.CompletedProcess(command, 0, json.dumps(obj), '') @@ -117,6 +137,21 @@ def test_named_identity_and_mixed_inventory_fail_closed(self): with self.subTest(mutation=case), self.assertRaises(module.CaptureError): module.capture('disposable-k3s', PLAN, runner=Stub(tamper=case)) + def test_quota_scope_and_service_account_credentials_fail_closed(self): + cases = ( + ('resourcequota', 'quota_scope'), + ('resourcequota', 'quota_hard'), + ('resourcequota', 'quota_unreconciled'), + ('resourcequota', 'deleting'), + ('serviceaccount', 'sa_automount'), + ('serviceaccount', 'sa_secret'), + ('serviceaccount', 'sa_pull_secret'), + ('serviceaccount', 'deleting'), + ) + for case in cases: + with self.subTest(mutation=case), self.assertRaises(module.CaptureError): + module.capture('disposable-k3s', PLAN, runner=Stub(tamper=case)) + def test_missing_quota_or_service_account_fails_closed(self): for res in ('resourcequota', 'serviceaccount'): with self.subTest(resource=res), self.assertRaisesRegex(